authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-06-20 05:41:03-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2026-06-20 21:08:23-04:00
log8ecc942ce890768f556e870bea1d66fe7bad8936
tree06be1303df607c8c377b1ef26ba250b19e1cbf14
parent2e7bd58abaafae8a029bc1a372085ace6738a365

x86_64: implement win64 abi for bool vectors

5 files changed, 1932 insertions(+), 283 deletions(-)

src/codegen/llvm/FuncGen.zig+4
...@@ -6790,6 +6790,7 @@ const ParamTypeIterator = struct {...@@ -6790,6 +6790,7 @@ const ParamTypeIterator = struct {
6790 }6790 }
6791 },6791 },
6792 .sse,6792 .sse,
6793 .bool_vector_mask,
6793 .integer_per_element,6794 .integer_per_element,
6794 .sse_per_element,6795 .sse_per_element,
6795 .sse_sse_x87_per_qword,6796 .sse_sse_x87_per_qword,
...@@ -6874,6 +6875,7 @@ const ParamTypeIterator = struct {...@@ -6874,6 +6875,7 @@ const ParamTypeIterator = struct {
6874 .none => break,6875 .none => break,
6875 .memory => unreachable, // handled above6876 .memory => unreachable, // handled above
6876 .win_i128 => unreachable, // windows only6877 .win_i128 => unreachable, // windows only
6878 .bool_vector_mask,
6877 .integer_per_element,6879 .integer_per_element,
6878 .sse_per_element,6880 .sse_per_element,
6879 .sse_sse_x87_per_qword,6881 .sse_sse_x87_per_qword,
...@@ -7091,6 +7093,7 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err...@@ -7091,6 +7093,7 @@ fn lowerWin64FnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.Err
7091 }7093 }
7092 },7094 },
7093 .sse,7095 .sse,
7096 .bool_vector_mask,
7094 .integer_per_element,7097 .integer_per_element,
7095 .sse_per_element,7098 .sse_per_element,
7096 .sse_sse_x87_per_qword,7099 .sse_sse_x87_per_qword,
...@@ -7156,6 +7159,7 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E...@@ -7156,6 +7159,7 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
7156 .none => break,7159 .none => break,
7157 .memory => return .void,7160 .memory => return .void,
7158 .win_i128 => unreachable, // windows only7161 .win_i128 => unreachable, // windows only
7162 .bool_vector_mask,
7159 .integer_per_element,7163 .integer_per_element,
7160 .sse_per_element,7164 .sse_per_element,
7161 .sse_sse_x87_per_qword,7165 .sse_sse_x87_per_qword,
src/codegen/x86_64/CodeGen.zig+1911-277
...@@ -164,7 +164,7 @@ next_temp_index: Temp.Index = @enumFromInt(0),...@@ -164,7 +164,7 @@ next_temp_index: Temp.Index = @enumFromInt(0),
164temp_type: [Temp.Index.max]Type = undefined,164temp_type: [Temp.Index.max]Type = undefined,
165165
166const MaskInfo = packed struct {166const MaskInfo = packed struct {
167 kind: enum(u1) { sign, all },167 kind: enum(u2) { lsb, msb, zero_extend, sign_extend },
168 inverted: bool = false,168 inverted: bool = false,
169 scalar: Memory.Size,169 scalar: Memory.Size,
170};170};
...@@ -204,13 +204,15 @@ pub const MCValue = union(enum) {...@@ -204,13 +204,15 @@ pub const MCValue = union(enum) {
204 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.204 /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register.
205 register_overflow: struct { reg: Register, eflags: Condition },205 register_overflow: struct { reg: Register, eflags: Condition },
206 /// The value is a bool vector stored in a vector register with a different scalar type.206 /// The value is a bool vector stored in a vector register with a different scalar type.
207 register_mask: struct { reg: Register, info: MaskInfo },207 register_mask: Mask,
208 /// The value is in memory at a hard-coded address.208 /// The value is in memory at a hard-coded address.
209 /// If the type is a pointer, it means the pointer address is stored at this memory location.209 /// If the type is a pointer, it means the pointer address is stored at this memory location.
210 memory: u64,210 memory: u64,
211 /// The value is in memory at a constant offset from the address in a register.211 /// The value is in memory at a constant offset from the address in a register.
212 indirect: bits.RegisterOffset,212 indirect: bits.RegisterOffset,
213 indirect_load_frame: bits.FrameAddr,213 indirect_load_frame: bits.FrameAddr,
214 /// The value is a bool vector stored in memory with a different scalar type at the address in a register.
215 indirect_mask: Mask,
214 /// The value stored at an offset from a frame index.216 /// The value stored at an offset from a frame index.
215 /// Payload is a frame address.217 /// Payload is a frame address.
216 load_frame: bits.FrameAddr,218 load_frame: bits.FrameAddr,
...@@ -243,6 +245,8 @@ pub const MCValue = union(enum) {...@@ -243,6 +245,8 @@ pub const MCValue = union(enum) {
243 reserved_frame: FrameIndex,245 reserved_frame: FrameIndex,
244 air_ref: Air.Inst.Ref,246 air_ref: Air.Inst.Ref,
245247
248 const Mask = struct { reg: Register, info: MaskInfo };
249
246 fn isModifiable(mcv: MCValue) bool {250 fn isModifiable(mcv: MCValue) bool {
247 return switch (mcv) {251 return switch (mcv) {
248 .none,252 .none,
...@@ -278,6 +282,7 @@ pub const MCValue = union(enum) {...@@ -278,6 +282,7 @@ pub const MCValue = union(enum) {
278 .register_quadruple,282 .register_quadruple,
279 .memory,283 .memory,
280 .indirect,284 .indirect,
285 .indirect_mask,
281 .load_nav,286 .load_nav,
282 => true,287 => true,
283 .load_frame => |frame_addr| !frame_addr.index.isNamed(),288 .load_frame => |frame_addr| !frame_addr.index.isNamed(),
...@@ -346,9 +351,10 @@ pub const MCValue = union(enum) {...@@ -346,9 +351,10 @@ pub const MCValue = union(enum) {
346 .register_quadruple,351 .register_quadruple,
347 => |*regs| regs,352 => |*regs| regs,
348 inline .register_offset,353 inline .register_offset,
349 .indirect,
350 .register_overflow,354 .register_overflow,
351 .register_mask,355 .register_mask,
356 .indirect,
357 .indirect_mask,
352 => |*pl| (&pl.reg)[0..1],358 => |*pl| (&pl.reg)[0..1],
353 else => &.{},359 else => &.{},
354 };360 };
...@@ -384,6 +390,7 @@ pub const MCValue = union(enum) {...@@ -384,6 +390,7 @@ pub const MCValue = union(enum) {
384 .register_offset,390 .register_offset,
385 .register_overflow,391 .register_overflow,
386 .register_mask,392 .register_mask,
393 .indirect_mask,
387 .lea_frame,394 .lea_frame,
388 .lea_nav,395 .lea_nav,
389 .lea_uav,396 .lea_uav,
...@@ -427,6 +434,7 @@ pub const MCValue = union(enum) {...@@ -427,6 +434,7 @@ pub const MCValue = union(enum) {
427 .memory,434 .memory,
428 .indirect,435 .indirect,
429 .indirect_load_frame,436 .indirect_load_frame,
437 .indirect_mask,
430 .load_nav,438 .load_nav,
431 .load_uav,439 .load_uav,
432 .load_lazy_sym,440 .load_lazy_sym,
...@@ -476,6 +484,7 @@ pub const MCValue = union(enum) {...@@ -476,6 +484,7 @@ pub const MCValue = union(enum) {
476 .memory,484 .memory,
477 .indirect,485 .indirect,
478 .indirect_load_frame,486 .indirect_load_frame,
487 .indirect_mask,
479 .load_frame,488 .load_frame,
480 .load_nav,489 .load_nav,
481 .lea_nav,490 .lea_nav,
...@@ -547,6 +556,10 @@ pub const MCValue = union(enum) {...@@ -547,6 +556,10 @@ pub const MCValue = union(enum) {
547 .disp = reg_off.off + mod_rm.disp,556 .disp = reg_off.off + mod_rm.disp,
548 } },557 } },
549 },558 },
559 .indirect_mask => |reg_mask| .{
560 .base = .{ .reg = reg_mask.reg.toSize(.ptr, function.target) },
561 .mod = .{ .rm = mod_rm },
562 },
550 .load_frame => |frame_addr| .{563 .load_frame => |frame_addr| .{
551 .base = .{ .frame = frame_addr.index },564 .base = .{ .frame = frame_addr.index },
552 .mod = .{ .rm = .{565 .mod = .{ .rm = .{
...@@ -568,7 +581,6 @@ pub const MCValue = union(enum) {...@@ -568,7 +581,6 @@ pub const MCValue = union(enum) {
568 switch (mcv) {581 switch (mcv) {
569 .none, .unreach, .dead, .undef => try w.print("({s})", .{@tagName(mcv)}),582 .none, .unreach, .dead, .undef => try w.print("({s})", .{@tagName(mcv)}),
570 .immediate => |pl| try w.print("0x{x}", .{pl}),583 .immediate => |pl| try w.print("0x{x}", .{pl}),
571 .memory => |pl| try w.print("[ds:0x{x}]", .{pl}),
572 inline .eflags, .register => |pl| try w.print("{s}", .{@tagName(pl)}),584 inline .eflags, .register => |pl| try w.print("{s}", .{@tagName(pl)}),
573 .register_pair => |pl| try w.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),585 .register_pair => |pl| try w.print("{s}:{s}", .{ @tagName(pl[1]), @tagName(pl[0]) }),
574 .register_triple => |pl| try w.print("{s}:{s}:{s}", .{586 .register_triple => |pl| try w.print("{s}:{s}:{s}", .{
...@@ -582,14 +594,21 @@ pub const MCValue = union(enum) {...@@ -582,14 +594,21 @@ pub const MCValue = union(enum) {
582 @tagName(pl.eflags),594 @tagName(pl.eflags),
583 @tagName(pl.reg),595 @tagName(pl.reg),
584 }),596 }),
585 .register_mask => |pl| try w.print("mask({s},{f}):{c}{s}", .{597 .register_mask => |pl| try w.print("mask({s},{f}):{s}{s}", .{
586 @tagName(pl.info.kind),598 @tagName(pl.info.kind),
587 pl.info.scalar,599 pl.info.scalar,
588 @as(u8, if (pl.info.inverted) '!' else ' '),600 if (pl.info.inverted) "!" else "",
589 @tagName(pl.reg),601 @tagName(pl.reg),
590 }),602 }),
603 .memory => |pl| try w.print("[ds:0x{x}]", .{pl}),
591 .indirect => |pl| try w.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),604 .indirect => |pl| try w.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }),
592 .indirect_load_frame => |pl| try w.print("[[{f} + 0x{x}]]", .{ pl.index, pl.off }),605 .indirect_load_frame => |pl| try w.print("[[{f} + 0x{x}]]", .{ pl.index, pl.off }),
606 .indirect_mask => |pl| try w.print("[mask({s},{f}):{s}{s}]", .{
607 @tagName(pl.info.kind),
608 pl.info.scalar,
609 if (pl.info.inverted) "!" else "",
610 @tagName(pl.reg),
611 }),
593 .load_frame => |pl| try w.print("[{f} + 0x{x}]", .{ pl.index, pl.off }),612 .load_frame => |pl| try w.print("[{f} + 0x{x}]", .{ pl.index, pl.off }),
594 .lea_frame => |pl| try w.print("{f} + 0x{x}", .{ pl.index, pl.off }),613 .lea_frame => |pl| try w.print("{f} + 0x{x}", .{ pl.index, pl.off }),
595 .load_nav => |pl| try w.print("[nav:{d}]", .{@intFromEnum(pl)}),614 .load_nav => |pl| try w.print("[nav:{d}]", .{@intFromEnum(pl)}),
...@@ -666,6 +685,7 @@ const InstTracking = struct {...@@ -666,6 +685,7 @@ const InstTracking = struct {
666 .register_overflow,685 .register_overflow,
667 .register_mask,686 .register_mask,
668 .indirect,687 .indirect,
688 .indirect_mask,
669 => .none,689 => .none,
670 }, .short = result };690 }, .short = result };
671 }691 }
...@@ -765,6 +785,7 @@ const InstTracking = struct {...@@ -765,6 +785,7 @@ const InstTracking = struct {
765 .register_overflow,785 .register_overflow,
766 .register_mask,786 .register_mask,
767 .indirect,787 .indirect,
788 .indirect_mask,
768 .register_tee,789 .register_tee,
769 .elementwise_gpr,790 .elementwise_gpr,
770 .elementwise_sse,791 .elementwise_sse,
...@@ -72517,7 +72538,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72517,7 +72538,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72517 .{ .src = .{ .to_sse, .none, .none } },72538 .{ .src = .{ .to_sse, .none, .none } },
72518 },72539 },
72519 .extra_temps = .{72540 .extra_temps = .{
72520 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72541 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72521 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72542 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72522 .unused,72543 .unused,
72523 .unused,72544 .unused,
...@@ -72541,7 +72562,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72541,7 +72562,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72541 .{ .src = .{ .to_mut_sse, .none, .none } },72562 .{ .src = .{ .to_mut_sse, .none, .none } },
72542 },72563 },
72543 .extra_temps = .{72564 .extra_temps = .{
72544 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72545 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72546 .unused,72567 .unused,
72547 .unused,72568 .unused,
...@@ -72565,7 +72586,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72565,7 +72586,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72565 .{ .src = .{ .to_sse, .none, .none } },72586 .{ .src = .{ .to_sse, .none, .none } },
72566 },72587 },
72567 .extra_temps = .{72588 .extra_temps = .{
72568 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72589 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72569 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },72590 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
72570 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72591 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72571 .unused,72592 .unused,
...@@ -72593,7 +72614,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72593,7 +72614,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72593 .{ .src = .{ .to_sse, .none, .none } },72614 .{ .src = .{ .to_sse, .none, .none } },
72594 },72615 },
72595 .extra_temps = .{72616 .extra_temps = .{
72596 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72617 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72597 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },72618 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
72598 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72619 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72599 .unused,72620 .unused,
...@@ -72622,7 +72643,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72622,7 +72643,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72622 .{ .src = .{ .to_mut_sse, .none, .none } },72643 .{ .src = .{ .to_mut_sse, .none, .none } },
72623 },72644 },
72624 .extra_temps = .{72645 .extra_temps = .{
72625 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72646 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72626 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72647 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72627 .unused,72648 .unused,
72628 .unused,72649 .unused,
...@@ -72651,7 +72672,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72651,7 +72672,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72651 .{ .src = .{ .to_mut_sse, .none, .none } },72672 .{ .src = .{ .to_mut_sse, .none, .none } },
72652 },72673 },
72653 .extra_temps = .{72674 .extra_temps = .{
72654 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72675 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72655 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72676 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72656 .unused,72677 .unused,
72657 .unused,72678 .unused,
...@@ -72718,7 +72739,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72718,7 +72739,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72718 .{ .src = .{ .to_sse, .none, .none } },72739 .{ .src = .{ .to_sse, .none, .none } },
72719 },72740 },
72720 .extra_temps = .{72741 .extra_temps = .{
72721 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72742 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72722 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72743 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72723 .unused,72744 .unused,
72724 .unused,72745 .unused,
...@@ -72744,7 +72765,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72744,7 +72765,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72744 .{ .src = .{ .to_mut_sse, .none, .none } },72765 .{ .src = .{ .to_mut_sse, .none, .none } },
72745 },72766 },
72746 .extra_temps = .{72767 .extra_temps = .{
72747 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72768 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72748 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72769 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72749 .unused,72770 .unused,
72750 .unused,72771 .unused,
...@@ -72770,7 +72791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72770,7 +72791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72770 .{ .src = .{ .to_sse, .none, .none } },72791 .{ .src = .{ .to_sse, .none, .none } },
72771 },72792 },
72772 .extra_temps = .{72793 .extra_temps = .{
72773 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72794 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72774 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },72795 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
72775 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72796 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72776 .unused,72797 .unused,
...@@ -72800,7 +72821,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72800,7 +72821,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72800 .{ .src = .{ .to_sse, .none, .none } },72821 .{ .src = .{ .to_sse, .none, .none } },
72801 },72822 },
72802 .extra_temps = .{72823 .extra_temps = .{
72803 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72824 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72804 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },72825 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
72805 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72826 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72806 .unused,72827 .unused,
...@@ -72831,7 +72852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72831,7 +72852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72831 .{ .src = .{ .to_mut_sse, .none, .none } },72852 .{ .src = .{ .to_mut_sse, .none, .none } },
72832 },72853 },
72833 .extra_temps = .{72854 .extra_temps = .{
72834 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72855 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72835 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72856 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72836 .unused,72857 .unused,
72837 .unused,72858 .unused,
...@@ -72862,7 +72883,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72862,7 +72883,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72862 .{ .src = .{ .to_mut_sse, .none, .none } },72883 .{ .src = .{ .to_mut_sse, .none, .none } },
72863 },72884 },
72864 .extra_temps = .{72885 .extra_temps = .{
72865 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },72886 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72866 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },72887 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72867 .unused,72888 .unused,
72868 .unused,72889 .unused,
...@@ -72932,7 +72953,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72932,7 +72953,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72932 .{ .src = .{ .to_sse, .none, .none } },72953 .{ .src = .{ .to_sse, .none, .none } },
72933 },72954 },
72934 .extra_temps = .{72955 .extra_temps = .{
72935 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72956 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72936 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72957 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72937 .unused,72958 .unused,
72938 .unused,72959 .unused,
...@@ -72958,7 +72979,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72958,7 +72979,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72958 .{ .src = .{ .to_mut_sse, .none, .none } },72979 .{ .src = .{ .to_mut_sse, .none, .none } },
72959 },72980 },
72960 .extra_temps = .{72981 .extra_temps = .{
72961 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },72982 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
72962 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },72983 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
72963 .unused,72984 .unused,
72964 .unused,72985 .unused,
...@@ -72984,7 +73005,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -72984,7 +73005,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72984 .{ .src = .{ .to_sse, .none, .none } },73005 .{ .src = .{ .to_sse, .none, .none } },
72985 },73006 },
72986 .extra_temps = .{73007 .extra_temps = .{
72987 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73008 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
72988 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73009 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
72989 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73010 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72990 .unused,73011 .unused,
...@@ -73014,7 +73035,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73014,7 +73035,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73014 .{ .src = .{ .to_sse, .none, .none } },73035 .{ .src = .{ .to_sse, .none, .none } },
73015 },73036 },
73016 .extra_temps = .{73037 .extra_temps = .{
73017 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73038 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73018 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73039 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73019 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73040 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73020 .unused,73041 .unused,
...@@ -73045,7 +73066,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73045,7 +73066,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73045 .{ .src = .{ .to_mut_sse, .none, .none } },73066 .{ .src = .{ .to_mut_sse, .none, .none } },
73046 },73067 },
73047 .extra_temps = .{73068 .extra_temps = .{
73048 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73069 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73049 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73070 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73050 .unused,73071 .unused,
73051 .unused,73072 .unused,
...@@ -73076,7 +73097,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73076,7 +73097,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73076 .{ .src = .{ .to_mut_sse, .none, .none } },73097 .{ .src = .{ .to_mut_sse, .none, .none } },
73077 },73098 },
73078 .extra_temps = .{73099 .extra_temps = .{
73079 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73100 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73080 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73101 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73081 .unused,73102 .unused,
73082 .unused,73103 .unused,
...@@ -73146,8 +73167,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73146,8 +73167,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73146 .{ .src = .{ .to_sse, .none, .none } },73167 .{ .src = .{ .to_sse, .none, .none } },
73147 },73168 },
73148 .extra_temps = .{73169 .extra_temps = .{
73149 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73170 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73150 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73171 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73151 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73172 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73152 .unused,73173 .unused,
73153 .unused,73174 .unused,
...@@ -73172,8 +73193,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73172,8 +73193,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73172 .{ .src = .{ .to_mut_sse, .none, .none } },73193 .{ .src = .{ .to_mut_sse, .none, .none } },
73173 },73194 },
73174 .extra_temps = .{73195 .extra_temps = .{
73175 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73196 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73176 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73197 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73177 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73198 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73178 .unused,73199 .unused,
73179 .unused,73200 .unused,
...@@ -73198,7 +73219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73198,7 +73219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73198 .{ .src = .{ .to_mut_sse, .none, .none } },73219 .{ .src = .{ .to_mut_sse, .none, .none } },
73199 },73220 },
73200 .extra_temps = .{73221 .extra_temps = .{
73201 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73222 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73202 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73223 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73203 .unused,73224 .unused,
73204 .unused,73225 .unused,
...@@ -73224,8 +73245,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73224,8 +73245,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73224 .{ .src = .{ .to_sse, .none, .none } },73245 .{ .src = .{ .to_sse, .none, .none } },
73225 },73246 },
73226 .extra_temps = .{73247 .extra_temps = .{
73227 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73248 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73228 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73249 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73229 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73250 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73230 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73251 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73231 .unused,73252 .unused,
...@@ -73253,8 +73274,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73253,8 +73274,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73253 .{ .src = .{ .to_mut_sse, .none, .none } },73274 .{ .src = .{ .to_mut_sse, .none, .none } },
73254 },73275 },
73255 .extra_temps = .{73276 .extra_temps = .{
73256 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73277 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73257 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73278 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73258 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73279 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73259 .unused,73280 .unused,
73260 .unused,73281 .unused,
...@@ -73323,8 +73344,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73323,8 +73344,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73323 .{ .src = .{ .to_sse, .none, .none } },73344 .{ .src = .{ .to_sse, .none, .none } },
73324 },73345 },
73325 .extra_temps = .{73346 .extra_temps = .{
73326 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73347 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73327 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73348 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73328 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73349 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73329 .unused,73350 .unused,
73330 .unused,73351 .unused,
...@@ -73350,8 +73371,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73350,8 +73371,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73350 .{ .src = .{ .to_mut_sse, .none, .none } },73371 .{ .src = .{ .to_mut_sse, .none, .none } },
73351 },73372 },
73352 .extra_temps = .{73373 .extra_temps = .{
73353 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73374 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73354 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73375 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73355 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73376 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73356 .unused,73377 .unused,
73357 .unused,73378 .unused,
...@@ -73377,7 +73398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73377,7 +73398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73377 .{ .src = .{ .to_mut_sse, .none, .none } },73398 .{ .src = .{ .to_mut_sse, .none, .none } },
73378 },73399 },
73379 .extra_temps = .{73400 .extra_temps = .{
73380 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73401 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73381 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73402 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73382 .unused,73403 .unused,
73383 .unused,73404 .unused,
...@@ -73404,8 +73425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73404,8 +73425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73404 .{ .src = .{ .to_sse, .none, .none } },73425 .{ .src = .{ .to_sse, .none, .none } },
73405 },73426 },
73406 .extra_temps = .{73427 .extra_temps = .{
73407 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73408 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73429 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73409 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73430 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73410 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73431 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73411 .unused,73432 .unused,
...@@ -73435,8 +73456,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73435,8 +73456,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73435 .{ .src = .{ .to_mut_sse, .none, .none } },73456 .{ .src = .{ .to_mut_sse, .none, .none } },
73436 },73457 },
73437 .extra_temps = .{73458 .extra_temps = .{
73438 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73459 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73439 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73460 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73440 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73461 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73441 .unused,73462 .unused,
73442 .unused,73463 .unused,
...@@ -73508,8 +73529,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73508,8 +73529,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73508 .{ .src = .{ .to_sse, .none, .none } },73529 .{ .src = .{ .to_sse, .none, .none } },
73509 },73530 },
73510 .extra_temps = .{73531 .extra_temps = .{
73511 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73532 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73512 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73533 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73513 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73534 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73514 .unused,73535 .unused,
73515 .unused,73536 .unused,
...@@ -73535,8 +73556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73535,8 +73556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73535 .{ .src = .{ .to_mut_sse, .none, .none } },73556 .{ .src = .{ .to_mut_sse, .none, .none } },
73536 },73557 },
73537 .extra_temps = .{73558 .extra_temps = .{
73538 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73559 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73539 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },73560 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word } } },
73540 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73561 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73541 .unused,73562 .unused,
73542 .unused,73563 .unused,
...@@ -73562,7 +73583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73562,7 +73583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73562 .{ .src = .{ .to_mut_sse, .none, .none } },73583 .{ .src = .{ .to_mut_sse, .none, .none } },
73563 },73584 },
73564 .extra_temps = .{73585 .extra_temps = .{
73565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73586 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73587 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73567 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73588 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73568 .unused,73589 .unused,
...@@ -73590,8 +73611,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73590,8 +73611,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73590 .{ .src = .{ .to_sse, .none, .none } },73611 .{ .src = .{ .to_sse, .none, .none } },
73591 },73612 },
73592 .extra_temps = .{73613 .extra_temps = .{
73593 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73614 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73594 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73615 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73595 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73616 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73596 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73617 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73597 .unused,73618 .unused,
...@@ -73621,8 +73642,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73621,8 +73642,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73621 .{ .src = .{ .to_mut_sse, .none, .none } },73642 .{ .src = .{ .to_mut_sse, .none, .none } },
73622 },73643 },
73623 .extra_temps = .{73644 .extra_temps = .{
73624 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },73645 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .word, .smear = 8 } } },
73625 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .reverse } },73646 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73626 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },73647 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
73627 .unused,73648 .unused,
73628 .unused,73649 .unused,
...@@ -73694,8 +73715,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73694,8 +73715,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73694 .{ .src = .{ .to_sse, .none, .none } },73715 .{ .src = .{ .to_sse, .none, .none } },
73695 },73716 },
73696 .extra_temps = .{73717 .extra_temps = .{
73697 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73718 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73698 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },73719 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
73699 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73720 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73700 .unused,73721 .unused,
73701 .unused,73722 .unused,
...@@ -73720,8 +73741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73720,8 +73741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73720 .{ .src = .{ .to_mut_sse, .none, .none } },73741 .{ .src = .{ .to_mut_sse, .none, .none } },
73721 },73742 },
73722 .extra_temps = .{73743 .extra_temps = .{
73723 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73744 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73724 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },73745 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
73725 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73746 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73726 .unused,73747 .unused,
73727 .unused,73748 .unused,
...@@ -73746,7 +73767,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73746,7 +73767,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73746 .{ .src = .{ .to_mut_sse, .none, .none } },73767 .{ .src = .{ .to_mut_sse, .none, .none } },
73747 },73768 },
73748 .extra_temps = .{73769 .extra_temps = .{
73749 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73770 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73750 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73771 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73751 .unused,73772 .unused,
73752 .unused,73773 .unused,
...@@ -73773,8 +73794,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73773,8 +73794,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73773 .{ .src = .{ .to_sse, .none, .none } },73794 .{ .src = .{ .to_sse, .none, .none } },
73774 },73795 },
73775 .extra_temps = .{73796 .extra_temps = .{
73776 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },73797 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
73777 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },73798 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73778 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73799 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73779 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },73800 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
73780 .unused,73801 .unused,
...@@ -73844,8 +73865,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73844,8 +73865,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73844 .{ .src = .{ .to_sse, .none, .none } },73865 .{ .src = .{ .to_sse, .none, .none } },
73845 },73866 },
73846 .extra_temps = .{73867 .extra_temps = .{
73847 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73868 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73848 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },73869 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
73849 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73870 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73850 .unused,73871 .unused,
73851 .unused,73872 .unused,
...@@ -73871,8 +73892,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73871,8 +73892,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73871 .{ .src = .{ .to_mut_sse, .none, .none } },73892 .{ .src = .{ .to_mut_sse, .none, .none } },
73872 },73893 },
73873 .extra_temps = .{73894 .extra_temps = .{
73874 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73895 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73875 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },73896 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
73876 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73897 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73877 .unused,73898 .unused,
73878 .unused,73899 .unused,
...@@ -73898,7 +73919,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73898,7 +73919,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73898 .{ .src = .{ .to_mut_sse, .none, .none } },73919 .{ .src = .{ .to_mut_sse, .none, .none } },
73899 },73920 },
73900 .extra_temps = .{73921 .extra_temps = .{
73901 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },73922 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
73902 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },73923 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
73903 .unused,73924 .unused,
73904 .unused,73925 .unused,
...@@ -73926,8 +73947,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -73926,8 +73947,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
73926 .{ .src = .{ .to_sse, .none, .none } },73947 .{ .src = .{ .to_sse, .none, .none } },
73927 },73948 },
73928 .extra_temps = .{73949 .extra_temps = .{
73929 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },73950 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
73930 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },73951 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
73931 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },73952 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
73932 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },73953 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
73933 .unused,73954 .unused,
...@@ -74000,8 +74021,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74000,8 +74021,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74000 .{ .src = .{ .to_sse, .none, .none } },74021 .{ .src = .{ .to_sse, .none, .none } },
74001 },74022 },
74002 .extra_temps = .{74023 .extra_temps = .{
74003 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74024 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74004 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },74025 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
74005 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74026 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74006 .unused,74027 .unused,
74007 .unused,74028 .unused,
...@@ -74027,8 +74048,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74027,8 +74048,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74027 .{ .src = .{ .to_mut_sse, .none, .none } },74048 .{ .src = .{ .to_mut_sse, .none, .none } },
74028 },74049 },
74029 .extra_temps = .{74050 .extra_temps = .{
74030 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74051 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74031 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },74052 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword } } },
74032 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74053 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74033 .unused,74054 .unused,
74034 .unused,74055 .unused,
...@@ -74054,7 +74075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74054,7 +74075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74054 .{ .src = .{ .to_mut_sse, .none, .none } },74075 .{ .src = .{ .to_mut_sse, .none, .none } },
74055 },74076 },
74056 .extra_temps = .{74077 .extra_temps = .{
74057 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74078 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74058 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74079 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74059 .unused,74080 .unused,
74060 .unused,74081 .unused,
...@@ -74082,8 +74103,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74082,8 +74103,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74082 .{ .src = .{ .to_sse, .none, .none } },74103 .{ .src = .{ .to_sse, .none, .none } },
74083 },74104 },
74084 .extra_temps = .{74105 .extra_temps = .{
74085 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },74106 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .dword, .smear = 8 } } },
74086 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .reverse } },74107 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .reverse } } },
74087 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },74108 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
74088 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },74109 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
74089 .unused,74110 .unused,
...@@ -74156,8 +74177,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74156,8 +74177,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74156 .{ .src = .{ .to_sse, .none, .none } },74177 .{ .src = .{ .to_sse, .none, .none } },
74157 },74178 },
74158 .extra_temps = .{74179 .extra_temps = .{
74159 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74180 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74160 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },74181 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
74161 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74182 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74162 .unused,74183 .unused,
74163 .unused,74184 .unused,
...@@ -74182,8 +74203,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74182,8 +74203,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74182 .{ .src = .{ .to_mut_sse, .none, .none } },74203 .{ .src = .{ .to_mut_sse, .none, .none } },
74183 },74204 },
74184 .extra_temps = .{74205 .extra_temps = .{
74185 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74206 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74186 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },74207 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
74187 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74208 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74188 .unused,74209 .unused,
74189 .unused,74210 .unused,
...@@ -74208,7 +74229,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74208,7 +74229,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74208 .{ .src = .{ .to_mut_sse, .none, .none } },74229 .{ .src = .{ .to_mut_sse, .none, .none } },
74209 },74230 },
74210 .extra_temps = .{74231 .extra_temps = .{
74211 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74232 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74212 .unused,74233 .unused,
74213 .unused,74234 .unused,
74214 .unused,74235 .unused,
...@@ -74279,7 +74300,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74279,7 +74300,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74279 .{ .src = .{ .to_sse, .none, .none } },74300 .{ .src = .{ .to_sse, .none, .none } },
74280 },74301 },
74281 .extra_temps = .{74302 .extra_temps = .{
74282 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74303 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74283 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },74304 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
74284 .unused,74305 .unused,
74285 .unused,74306 .unused,
...@@ -74307,7 +74328,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74307,7 +74328,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74307 .{ .src = .{ .to_mut_sse, .none, .none } },74328 .{ .src = .{ .to_mut_sse, .none, .none } },
74308 },74329 },
74309 .extra_temps = .{74330 .extra_temps = .{
74310 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74331 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74311 .unused,74332 .unused,
74312 .unused,74333 .unused,
74313 .unused,74334 .unused,
...@@ -74381,8 +74402,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74381,8 +74402,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74381 .{ .src = .{ .to_sse, .none, .none } },74402 .{ .src = .{ .to_sse, .none, .none } },
74382 },74403 },
74383 .extra_temps = .{74404 .extra_temps = .{
74384 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74405 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74385 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },74406 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
74386 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74407 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74387 .unused,74408 .unused,
74388 .unused,74409 .unused,
...@@ -74408,8 +74429,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74408,8 +74429,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74408 .{ .src = .{ .to_mut_sse, .none, .none } },74429 .{ .src = .{ .to_mut_sse, .none, .none } },
74409 },74430 },
74410 .extra_temps = .{74431 .extra_temps = .{
74411 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74432 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74412 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },74433 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .qword } } },
74413 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74434 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74414 .unused,74435 .unused,
74415 .unused,74436 .unused,
...@@ -74435,7 +74456,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74435,7 +74456,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74435 .{ .src = .{ .to_mut_sse, .none, .none } },74456 .{ .src = .{ .to_mut_sse, .none, .none } },
74436 },74457 },
74437 .extra_temps = .{74458 .extra_temps = .{
74438 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74459 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74439 .unused,74460 .unused,
74440 .unused,74461 .unused,
74441 .unused,74462 .unused,
...@@ -74509,8 +74530,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74509,8 +74530,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74509 .{ .src = .{ .to_sse, .none, .none } },74530 .{ .src = .{ .to_sse, .none, .none } },
74510 },74531 },
74511 .extra_temps = .{74532 .extra_temps = .{
74512 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74533 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74513 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74534 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74514 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74535 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74515 .unused,74536 .unused,
74516 .unused,74537 .unused,
...@@ -74535,8 +74556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74535,8 +74556,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74535 .{ .src = .{ .to_mut_sse, .none, .none } },74556 .{ .src = .{ .to_mut_sse, .none, .none } },
74536 },74557 },
74537 .extra_temps = .{74558 .extra_temps = .{
74538 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74559 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74539 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74560 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74540 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74561 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74541 .unused,74562 .unused,
74542 .unused,74563 .unused,
...@@ -74561,8 +74582,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74561,8 +74582,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74561 .{ .src = .{ .to_sse, .none, .none } },74582 .{ .src = .{ .to_sse, .none, .none } },
74562 },74583 },
74563 .extra_temps = .{74584 .extra_temps = .{
74564 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74585 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74565 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74586 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74587 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74567 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },74588 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
74568 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },74589 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
...@@ -74594,8 +74615,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74594,8 +74615,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74594 .{ .src = .{ .to_mut_sse, .none, .none } },74615 .{ .src = .{ .to_mut_sse, .none, .none } },
74595 },74616 },
74596 .extra_temps = .{74617 .extra_temps = .{
74597 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74618 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74598 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74619 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74599 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74620 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74600 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },74621 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
74601 .unused,74622 .unused,
...@@ -74627,8 +74648,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74627,8 +74648,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74627 .{ .src = .{ .to_sse, .none, .none } },74648 .{ .src = .{ .to_sse, .none, .none } },
74628 },74649 },
74629 .extra_temps = .{74650 .extra_temps = .{
74630 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74651 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74631 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74652 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74632 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74653 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74633 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },74654 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
74634 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },74655 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
...@@ -74660,8 +74681,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74660,8 +74681,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74660 .{ .src = .{ .to_mut_sse, .none, .none } },74681 .{ .src = .{ .to_mut_sse, .none, .none } },
74661 },74682 },
74662 .extra_temps = .{74683 .extra_temps = .{
74663 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74684 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74664 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74685 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74665 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74686 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74666 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },74687 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
74667 .unused,74688 .unused,
...@@ -74693,8 +74714,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74693,8 +74714,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74693 .{ .src = .{ .to_sse, .none, .none } },74714 .{ .src = .{ .to_sse, .none, .none } },
74694 },74715 },
74695 .extra_temps = .{74716 .extra_temps = .{
74696 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },74717 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74697 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },74718 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .repeat = 2, .size = .xword } } },
74698 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74719 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74699 .unused,74720 .unused,
74700 .unused,74721 .unused,
...@@ -74720,8 +74741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74720,8 +74741,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74720 .{ .src = .{ .to_sse, .none, .none } },74741 .{ .src = .{ .to_sse, .none, .none } },
74721 },74742 },
74722 .extra_temps = .{74743 .extra_temps = .{
74723 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },74744 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74724 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74745 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74725 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74746 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74726 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },74747 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
74727 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },74748 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -74752,8 +74773,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74752,8 +74773,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74752 .extra_temps = .{74773 .extra_temps = .{
74753 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74774 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74754 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },74775 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
74755 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .forward } },74776 .{ .type = .vector_32_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74756 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },74777 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .repeat = 2, .size = .xword } } },
74757 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },74778 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
74758 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },74779 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
74759 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },74780 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
...@@ -74788,8 +74809,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -74788,8 +74809,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
74788 .extra_temps = .{74809 .extra_temps = .{
74789 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },74810 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
74790 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },74811 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
74791 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .forward } },74812 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
74792 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },74813 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .reverse, .size = .xword } } },
74793 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },74814 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
74794 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },74815 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
74795 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },74816 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -82067,7 +82088,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82067,7 +82088,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82067 .dst_temps = .{ .{ .mut_rc_mask = .{82088 .dst_temps = .{ .{ .mut_rc_mask = .{
82068 .ref = .src0,82089 .ref = .src0,
82069 .rc = .sse,82090 .rc = .sse,
82070 .info = .{ .kind = .all, .scalar = .dword },82091 .info = .{ .kind = .sign_extend, .scalar = .dword },
82071 } }, .unused },82092 } }, .unused },
82072 .each = .{ .once = &.{82093 .each = .{ .once = &.{
82073 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },82094 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
...@@ -82107,7 +82128,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82107,7 +82128,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82107 .dst_temps = .{ .{ .mut_rc_mask = .{82128 .dst_temps = .{ .{ .mut_rc_mask = .{
82108 .ref = .src0,82129 .ref = .src0,
82109 .rc = .sse,82130 .rc = .sse,
82110 .info = .{ .kind = .all, .scalar = .dword },82131 .info = .{ .kind = .sign_extend, .scalar = .dword },
82111 } }, .unused },82132 } }, .unused },
82112 .each = .{ .once = &.{82133 .each = .{ .once = &.{
82113 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },82134 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
...@@ -82147,7 +82168,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82147,7 +82168,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82147 .dst_temps = .{ .{ .mut_rc_mask = .{82168 .dst_temps = .{ .{ .mut_rc_mask = .{
82148 .ref = .src0,82169 .ref = .src0,
82149 .rc = .sse,82170 .rc = .sse,
82150 .info = .{ .kind = .all, .scalar = .dword },82171 .info = .{ .kind = .sign_extend, .scalar = .dword },
82151 } }, .unused },82172 } }, .unused },
82152 .each = .{ .once = &.{82173 .each = .{ .once = &.{
82153 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },82174 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
...@@ -82171,7 +82192,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82171,7 +82192,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82171 .dst_temps = .{ .{ .mut_rc_mask = .{82192 .dst_temps = .{ .{ .mut_rc_mask = .{
82172 .ref = .src0,82193 .ref = .src0,
82173 .rc = .sse,82194 .rc = .sse,
82174 .info = .{ .kind = .all, .scalar = .dword },82195 .info = .{ .kind = .sign_extend, .scalar = .dword },
82175 } }, .unused },82196 } }, .unused },
82176 .each = .{ .once = &.{82197 .each = .{ .once = &.{
82177 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {82198 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
...@@ -82194,7 +82215,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82194,7 +82215,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82194 .dst_temps = .{ .{ .mut_rc_mask = .{82215 .dst_temps = .{ .{ .mut_rc_mask = .{
82195 .ref = .src0,82216 .ref = .src0,
82196 .rc = .sse,82217 .rc = .sse,
82197 .info = .{ .kind = .all, .scalar = .dword },82218 .info = .{ .kind = .sign_extend, .scalar = .dword },
82198 } }, .unused },82219 } }, .unused },
82199 .each = .{ .once = &.{82220 .each = .{ .once = &.{
82200 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {82221 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
...@@ -82216,7 +82237,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82216,7 +82237,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82216 },82237 },
82217 .dst_temps = .{ .{ .ref_mask = .{82238 .dst_temps = .{ .{ .ref_mask = .{
82218 .ref = .src0,82239 .ref = .src0,
82219 .info = .{ .kind = .all, .scalar = .dword },82240 .info = .{ .kind = .sign_extend, .scalar = .dword },
82220 } }, .unused },82241 } }, .unused },
82221 .each = .{ .once = &.{82242 .each = .{ .once = &.{
82222 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {82243 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {
...@@ -82238,7 +82259,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82238,7 +82259,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82238 .dst_temps = .{ .{ .mut_rc_mask = .{82259 .dst_temps = .{ .{ .mut_rc_mask = .{
82239 .ref = .src0,82260 .ref = .src0,
82240 .rc = .sse,82261 .rc = .sse,
82241 .info = .{ .kind = .all, .scalar = .dword },82262 .info = .{ .kind = .sign_extend, .scalar = .dword },
82242 } }, .unused },82263 } }, .unused },
82243 .each = .{ .once = &.{82264 .each = .{ .once = &.{
82244 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {82265 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -82261,7 +82282,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82261,7 +82282,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82261 .dst_temps = .{ .{ .mut_rc_mask = .{82282 .dst_temps = .{ .{ .mut_rc_mask = .{
82262 .ref = .src0,82283 .ref = .src0,
82263 .rc = .sse,82284 .rc = .sse,
82264 .info = .{ .kind = .all, .scalar = .dword },82285 .info = .{ .kind = .sign_extend, .scalar = .dword },
82265 } }, .unused },82286 } }, .unused },
82266 .each = .{ .once = &.{82287 .each = .{ .once = &.{
82267 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {82288 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -82283,7 +82304,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82283,7 +82304,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82283 },82304 },
82284 .dst_temps = .{ .{ .ref_mask = .{82305 .dst_temps = .{ .{ .ref_mask = .{
82285 .ref = .src0,82306 .ref = .src0,
82286 .info = .{ .kind = .all, .scalar = .dword },82307 .info = .{ .kind = .sign_extend, .scalar = .dword },
82287 } }, .unused },82308 } }, .unused },
82288 .each = .{ .once = &.{82309 .each = .{ .once = &.{
82289 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {82310 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {
...@@ -82305,7 +82326,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82305,7 +82326,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82305 .dst_temps = .{ .{ .mut_rc_mask = .{82326 .dst_temps = .{ .{ .mut_rc_mask = .{
82306 .ref = .src0,82327 .ref = .src0,
82307 .rc = .sse,82328 .rc = .sse,
82308 .info = .{ .kind = .all, .scalar = .dword },82329 .info = .{ .kind = .sign_extend, .scalar = .dword },
82309 } }, .unused },82330 } }, .unused },
82310 .each = .{ .once = &.{82331 .each = .{ .once = &.{
82311 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {82332 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -82328,7 +82349,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82328,7 +82349,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82328 .dst_temps = .{ .{ .mut_rc_mask = .{82349 .dst_temps = .{ .{ .mut_rc_mask = .{
82329 .ref = .src0,82350 .ref = .src0,
82330 .rc = .sse,82351 .rc = .sse,
82331 .info = .{ .kind = .all, .scalar = .dword },82352 .info = .{ .kind = .sign_extend, .scalar = .dword },
82332 } }, .unused },82353 } }, .unused },
82333 .each = .{ .once = &.{82354 .each = .{ .once = &.{
82334 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {82355 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -82350,7 +82371,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82350,7 +82371,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82350 .dst_temps = .{ .{ .mut_rc_mask = .{82371 .dst_temps = .{ .{ .mut_rc_mask = .{
82351 .ref = .src0,82372 .ref = .src0,
82352 .rc = .sse,82373 .rc = .sse,
82353 .info = .{ .kind = .all, .scalar = .qword },82374 .info = .{ .kind = .sign_extend, .scalar = .qword },
82354 } }, .unused },82375 } }, .unused },
82355 .each = .{ .once = &.{82376 .each = .{ .once = &.{
82356 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {82377 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
...@@ -82373,7 +82394,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82373,7 +82394,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82373 .dst_temps = .{ .{ .mut_rc_mask = .{82394 .dst_temps = .{ .{ .mut_rc_mask = .{
82374 .ref = .src0,82395 .ref = .src0,
82375 .rc = .sse,82396 .rc = .sse,
82376 .info = .{ .kind = .all, .scalar = .qword },82397 .info = .{ .kind = .sign_extend, .scalar = .qword },
82377 } }, .unused },82398 } }, .unused },
82378 .each = .{ .once = &.{82399 .each = .{ .once = &.{
82379 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {82400 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
...@@ -82395,7 +82416,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82395,7 +82416,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82395 },82416 },
82396 .dst_temps = .{ .{ .ref_mask = .{82417 .dst_temps = .{ .{ .ref_mask = .{
82397 .ref = .src0,82418 .ref = .src0,
82398 .info = .{ .kind = .all, .scalar = .qword },82419 .info = .{ .kind = .sign_extend, .scalar = .qword },
82399 } }, .unused },82420 } }, .unused },
82400 .each = .{ .once = &.{82421 .each = .{ .once = &.{
82401 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {82422 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {
...@@ -82417,7 +82438,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82417,7 +82438,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82417 .dst_temps = .{ .{ .mut_rc_mask = .{82438 .dst_temps = .{ .{ .mut_rc_mask = .{
82418 .ref = .src0,82439 .ref = .src0,
82419 .rc = .sse,82440 .rc = .sse,
82420 .info = .{ .kind = .all, .scalar = .qword },82441 .info = .{ .kind = .sign_extend, .scalar = .qword },
82421 } }, .unused },82442 } }, .unused },
82422 .each = .{ .once = &.{82443 .each = .{ .once = &.{
82423 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {82444 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -82440,7 +82461,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82440,7 +82461,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82440 .dst_temps = .{ .{ .mut_rc_mask = .{82461 .dst_temps = .{ .{ .mut_rc_mask = .{
82441 .ref = .src0,82462 .ref = .src0,
82442 .rc = .sse,82463 .rc = .sse,
82443 .info = .{ .kind = .all, .scalar = .qword },82464 .info = .{ .kind = .sign_extend, .scalar = .qword },
82444 } }, .unused },82465 } }, .unused },
82445 .each = .{ .once = &.{82466 .each = .{ .once = &.{
82446 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {82467 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -82462,7 +82483,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82462,7 +82483,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82462 },82483 },
82463 .dst_temps = .{ .{ .ref_mask = .{82484 .dst_temps = .{ .{ .ref_mask = .{
82464 .ref = .src0,82485 .ref = .src0,
82465 .info = .{ .kind = .all, .scalar = .qword },82486 .info = .{ .kind = .sign_extend, .scalar = .qword },
82466 } }, .unused },82487 } }, .unused },
82467 .each = .{ .once = &.{82488 .each = .{ .once = &.{
82468 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {82489 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {
...@@ -82484,7 +82505,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82484,7 +82505,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82484 .dst_temps = .{ .{ .mut_rc_mask = .{82505 .dst_temps = .{ .{ .mut_rc_mask = .{
82485 .ref = .src0,82506 .ref = .src0,
82486 .rc = .sse,82507 .rc = .sse,
82487 .info = .{ .kind = .all, .scalar = .qword },82508 .info = .{ .kind = .sign_extend, .scalar = .qword },
82488 } }, .unused },82509 } }, .unused },
82489 .each = .{ .once = &.{82510 .each = .{ .once = &.{
82490 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {82511 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -82507,7 +82528,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -82507,7 +82528,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
82507 .dst_temps = .{ .{ .mut_rc_mask = .{82528 .dst_temps = .{ .{ .mut_rc_mask = .{
82508 .ref = .src0,82529 .ref = .src0,
82509 .rc = .sse,82530 .rc = .sse,
82510 .info = .{ .kind = .all, .scalar = .qword },82531 .info = .{ .kind = .sign_extend, .scalar = .qword },
82511 } }, .unused },82532 } }, .unused },
82512 .each = .{ .once = &.{82533 .each = .{ .once = &.{
82513 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {82534 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -84711,7 +84732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84711,7 +84732,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84711 .{ .src = .{ .to_sse, .to_sse, .none } },84732 .{ .src = .{ .to_sse, .to_sse, .none } },
84712 },84733 },
84713 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{84734 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84714 .kind = .all,84735 .kind = .sign_extend,
84715 .inverted = switch (cc) {84736 .inverted = switch (cc) {
84716 else => unreachable,84737 else => unreachable,
84717 .e => false,84738 .e => false,
...@@ -84735,7 +84756,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84735,7 +84756,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84735 .{ .src = .{ .to_sse, .to_sse, .none } },84756 .{ .src = .{ .to_sse, .to_sse, .none } },
84736 },84757 },
84737 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{84758 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84738 .kind = .all,84759 .kind = .sign_extend,
84739 .inverted = switch (cc) {84760 .inverted = switch (cc) {
84740 else => unreachable,84761 else => unreachable,
84741 .e => false,84762 .e => false,
...@@ -84759,7 +84780,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84759,7 +84780,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84759 .{ .src = .{ .to_sse, .to_sse, .none } },84780 .{ .src = .{ .to_sse, .to_sse, .none } },
84760 },84781 },
84761 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{84782 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84762 .kind = .all,84783 .kind = .sign_extend,
84763 .inverted = switch (cc) {84784 .inverted = switch (cc) {
84764 else => unreachable,84785 else => unreachable,
84765 .e => false,84786 .e => false,
...@@ -84783,7 +84804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84783,7 +84804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84783 .{ .src = .{ .to_sse, .to_sse, .none } },84804 .{ .src = .{ .to_sse, .to_sse, .none } },
84784 },84805 },
84785 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{84806 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84786 .kind = .all,84807 .kind = .sign_extend,
84787 .inverted = switch (cc) {84808 .inverted = switch (cc) {
84788 else => unreachable,84809 else => unreachable,
84789 .e => false,84810 .e => false,
...@@ -84807,7 +84828,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84807,7 +84828,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84807 .{ .src = .{ .to_mut_sse, .to_sse, .none } },84828 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
84808 },84829 },
84809 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84830 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84810 .kind = .all,84831 .kind = .sign_extend,
84811 .inverted = switch (cc) {84832 .inverted = switch (cc) {
84812 else => unreachable,84833 else => unreachable,
84813 .e => false,84834 .e => false,
...@@ -84831,7 +84852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84831,7 +84852,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84831 .{ .src = .{ .to_mut_sse, .to_sse, .none } },84852 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
84832 },84853 },
84833 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84854 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84834 .kind = .all,84855 .kind = .sign_extend,
84835 .inverted = switch (cc) {84856 .inverted = switch (cc) {
84836 else => unreachable,84857 else => unreachable,
84837 .e => false,84858 .e => false,
...@@ -84855,7 +84876,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84855,7 +84876,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84855 .{ .src = .{ .to_mut_sse, .to_sse, .none } },84876 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
84856 },84877 },
84857 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84878 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84858 .kind = .all,84879 .kind = .sign_extend,
84859 .inverted = switch (cc) {84880 .inverted = switch (cc) {
84860 else => unreachable,84881 else => unreachable,
84861 .e => false,84882 .e => false,
...@@ -84879,7 +84900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84879,7 +84900,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84879 .{ .src = .{ .to_mut_sse, .to_sse, .none } },84900 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
84880 },84901 },
84881 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84902 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84882 .kind = .all,84903 .kind = .sign_extend,
84883 .inverted = switch (cc) {84904 .inverted = switch (cc) {
84884 else => unreachable,84905 else => unreachable,
84885 .e => false,84906 .e => false,
...@@ -84903,7 +84924,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84903,7 +84924,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84903 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },84924 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
84904 },84925 },
84905 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84926 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84906 .kind = .all,84927 .kind = .sign_extend,
84907 .inverted = switch (cc) {84928 .inverted = switch (cc) {
84908 else => unreachable,84929 else => unreachable,
84909 .e => false,84930 .e => false,
...@@ -84927,7 +84948,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84927,7 +84948,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84927 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },84948 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
84928 },84949 },
84929 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84950 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84930 .kind = .all,84951 .kind = .sign_extend,
84931 .inverted = switch (cc) {84952 .inverted = switch (cc) {
84932 else => unreachable,84953 else => unreachable,
84933 .e => false,84954 .e => false,
...@@ -84951,7 +84972,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84951,7 +84972,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84951 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },84972 .{ .src = .{ .to_mut_mmx, .to_mmx, .none } },
84952 },84973 },
84953 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{84974 .dst_temps = .{ .{ .ref_mask = .{ .ref = .src0, .info = .{
84954 .kind = .all,84975 .kind = .sign_extend,
84955 .inverted = switch (cc) {84976 .inverted = switch (cc) {
84956 else => unreachable,84977 else => unreachable,
84957 .e => false,84978 .e => false,
...@@ -84975,7 +84996,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84975,7 +84996,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84975 .{ .src = .{ .to_sse, .to_sse, .none } },84996 .{ .src = .{ .to_sse, .to_sse, .none } },
84976 },84997 },
84977 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{84998 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
84978 .kind = .all,84999 .kind = .sign_extend,
84979 .inverted = switch (cc) {85000 .inverted = switch (cc) {
84980 else => unreachable,85001 else => unreachable,
84981 .e => false,85002 .e => false,
...@@ -84999,7 +85020,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -84999,7 +85020,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
84999 .{ .src = .{ .to_sse, .to_sse, .none } },85020 .{ .src = .{ .to_sse, .to_sse, .none } },
85000 },85021 },
85001 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{85022 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85002 .kind = .all,85023 .kind = .sign_extend,
85003 .inverted = switch (cc) {85024 .inverted = switch (cc) {
85004 else => unreachable,85025 else => unreachable,
85005 .e => false,85026 .e => false,
...@@ -85023,7 +85044,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -85023,7 +85044,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
85023 .{ .src = .{ .to_sse, .to_sse, .none } },85044 .{ .src = .{ .to_sse, .to_sse, .none } },
85024 },85045 },
85025 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{85046 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85026 .kind = .all,85047 .kind = .sign_extend,
85027 .inverted = switch (cc) {85048 .inverted = switch (cc) {
85028 else => unreachable,85049 else => unreachable,
85029 .e => false,85050 .e => false,
...@@ -85047,7 +85068,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -85047,7 +85068,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
85047 .{ .src = .{ .to_sse, .to_sse, .none } },85068 .{ .src = .{ .to_sse, .to_sse, .none } },
85048 },85069 },
85049 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{85070 .dst_temps = .{ .{ .mut_rc_mask = .{ .ref = .src0, .rc = .sse, .info = .{
85050 .kind = .all,85071 .kind = .sign_extend,
85051 .inverted = switch (cc) {85072 .inverted = switch (cc) {
85052 else => unreachable,85073 else => unreachable,
85053 .e => false,85074 .e => false,
...@@ -86699,7 +86720,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86699,7 +86720,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86699 .dst_temps = .{ .{ .mut_rc_mask = .{86720 .dst_temps = .{ .{ .mut_rc_mask = .{
86700 .ref = .src0,86721 .ref = .src0,
86701 .rc = .sse,86722 .rc = .sse,
86702 .info = .{ .kind = .all, .scalar = .dword },86723 .info = .{ .kind = .sign_extend, .scalar = .dword },
86703 } }, .unused },86724 } }, .unused },
86704 .each = .{ .once = &.{86725 .each = .{ .once = &.{
86705 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },86726 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
...@@ -86739,7 +86760,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86739,7 +86760,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86739 .dst_temps = .{ .{ .mut_rc_mask = .{86760 .dst_temps = .{ .{ .mut_rc_mask = .{
86740 .ref = .src0,86761 .ref = .src0,
86741 .rc = .sse,86762 .rc = .sse,
86742 .info = .{ .kind = .all, .scalar = .dword },86763 .info = .{ .kind = .sign_extend, .scalar = .dword },
86743 } }, .unused },86764 } }, .unused },
86744 .each = .{ .once = &.{86765 .each = .{ .once = &.{
86745 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },86766 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
...@@ -86779,7 +86800,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86779,7 +86800,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86779 .dst_temps = .{ .{ .mut_rc_mask = .{86800 .dst_temps = .{ .{ .mut_rc_mask = .{
86780 .ref = .src0,86801 .ref = .src0,
86781 .rc = .sse,86802 .rc = .sse,
86782 .info = .{ .kind = .all, .scalar = .dword },86803 .info = .{ .kind = .sign_extend, .scalar = .dword },
86783 } }, .unused },86804 } }, .unused },
86784 .each = .{ .once = &.{86805 .each = .{ .once = &.{
86785 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },86806 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
...@@ -86805,7 +86826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86805,7 +86826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86805 .dst_temps = .{ .{ .mut_rc_mask = .{86826 .dst_temps = .{ .{ .mut_rc_mask = .{
86806 .ref = .src0,86827 .ref = .src0,
86807 .rc = .sse,86828 .rc = .sse,
86808 .info = .{ .kind = .all, .scalar = .dword },86829 .info = .{ .kind = .sign_extend, .scalar = .dword },
86809 } }, .unused },86830 } }, .unused },
86810 .each = .{ .once = &.{86831 .each = .{ .once = &.{
86811 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {86832 .{ ._, .v_ss, .cmp, .dst0x, .src0x, .src1d, .vp(switch (cc) {
...@@ -86828,7 +86849,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86828,7 +86849,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86828 },86849 },
86829 .dst_temps = .{ .{ .ref_mask = .{86850 .dst_temps = .{ .{ .ref_mask = .{
86830 .ref = .src0,86851 .ref = .src0,
86831 .info = .{ .kind = .all, .scalar = .dword },86852 .info = .{ .kind = .sign_extend, .scalar = .dword },
86832 } }, .unused },86853 } }, .unused },
86833 .each = .{ .once = &.{86854 .each = .{ .once = &.{
86834 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {86855 .{ ._, ._ss, .cmp, .dst0x, .src1d, .sp(switch (cc) {
...@@ -86852,7 +86873,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86852,7 +86873,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86852 .dst_temps = .{ .{ .mut_rc_mask = .{86873 .dst_temps = .{ .{ .mut_rc_mask = .{
86853 .ref = .src0,86874 .ref = .src0,
86854 .rc = .sse,86875 .rc = .sse,
86855 .info = .{ .kind = .all, .scalar = .dword },86876 .info = .{ .kind = .sign_extend, .scalar = .dword },
86856 } }, .unused },86877 } }, .unused },
86857 .each = .{ .once = &.{86878 .each = .{ .once = &.{
86858 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {86879 .{ ._, .v_ps, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -86875,7 +86896,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86875,7 +86896,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86875 },86896 },
86876 .dst_temps = .{ .{ .ref_mask = .{86897 .dst_temps = .{ .{ .ref_mask = .{
86877 .ref = .src0,86898 .ref = .src0,
86878 .info = .{ .kind = .all, .scalar = .dword },86899 .info = .{ .kind = .sign_extend, .scalar = .dword },
86879 } }, .unused },86900 } }, .unused },
86880 .each = .{ .once = &.{86901 .each = .{ .once = &.{
86881 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {86902 .{ ._, ._ps, .cmp, .dst0x, .src1x, .sp(switch (cc) {
...@@ -86899,7 +86920,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86899,7 +86920,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86899 .dst_temps = .{ .{ .mut_rc_mask = .{86920 .dst_temps = .{ .{ .mut_rc_mask = .{
86900 .ref = .src0,86921 .ref = .src0,
86901 .rc = .sse,86922 .rc = .sse,
86902 .info = .{ .kind = .all, .scalar = .dword },86923 .info = .{ .kind = .sign_extend, .scalar = .dword },
86903 } }, .unused },86924 } }, .unused },
86904 .each = .{ .once = &.{86925 .each = .{ .once = &.{
86905 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {86926 .{ ._, .v_ps, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -86923,7 +86944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86923,7 +86944,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86923 .dst_temps = .{ .{ .mut_rc_mask = .{86944 .dst_temps = .{ .{ .mut_rc_mask = .{
86924 .ref = .src0,86945 .ref = .src0,
86925 .rc = .sse,86946 .rc = .sse,
86926 .info = .{ .kind = .all, .scalar = .qword },86947 .info = .{ .kind = .sign_extend, .scalar = .qword },
86927 } }, .unused },86948 } }, .unused },
86928 .each = .{ .once = &.{86949 .each = .{ .once = &.{
86929 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {86950 .{ ._, .v_sd, .cmp, .dst0x, .src0x, .src1q, .vp(switch (cc) {
...@@ -86946,7 +86967,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86946,7 +86967,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86946 },86967 },
86947 .dst_temps = .{ .{ .ref_mask = .{86968 .dst_temps = .{ .{ .ref_mask = .{
86948 .ref = .src0,86969 .ref = .src0,
86949 .info = .{ .kind = .all, .scalar = .qword },86970 .info = .{ .kind = .sign_extend, .scalar = .qword },
86950 } }, .unused },86971 } }, .unused },
86951 .each = .{ .once = &.{86972 .each = .{ .once = &.{
86952 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {86973 .{ ._, ._sd, .cmp, .dst0x, .src1q, .sp(switch (cc) {
...@@ -86970,7 +86991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86970,7 +86991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86970 .dst_temps = .{ .{ .mut_rc_mask = .{86991 .dst_temps = .{ .{ .mut_rc_mask = .{
86971 .ref = .src0,86992 .ref = .src0,
86972 .rc = .sse,86993 .rc = .sse,
86973 .info = .{ .kind = .all, .scalar = .qword },86994 .info = .{ .kind = .sign_extend, .scalar = .qword },
86974 } }, .unused },86995 } }, .unused },
86975 .each = .{ .once = &.{86996 .each = .{ .once = &.{
86976 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {86997 .{ ._, .v_pd, .cmp, .dst0x, .src0x, .src1x, .vp(switch (cc) {
...@@ -86993,7 +87014,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -86993,7 +87014,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
86993 },87014 },
86994 .dst_temps = .{ .{ .ref_mask = .{87015 .dst_temps = .{ .{ .ref_mask = .{
86995 .ref = .src0,87016 .ref = .src0,
86996 .info = .{ .kind = .all, .scalar = .qword },87017 .info = .{ .kind = .sign_extend, .scalar = .qword },
86997 } }, .unused },87018 } }, .unused },
86998 .each = .{ .once = &.{87019 .each = .{ .once = &.{
86999 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {87020 .{ ._, ._pd, .cmp, .dst0x, .src1x, .sp(switch (cc) {
...@@ -87017,7 +87038,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -87017,7 +87038,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
87017 .dst_temps = .{ .{ .mut_rc_mask = .{87038 .dst_temps = .{ .{ .mut_rc_mask = .{
87018 .ref = .src0,87039 .ref = .src0,
87019 .rc = .sse,87040 .rc = .sse,
87020 .info = .{ .kind = .all, .scalar = .qword },87041 .info = .{ .kind = .sign_extend, .scalar = .qword },
87021 } }, .unused },87042 } }, .unused },
87022 .each = .{ .once = &.{87043 .each = .{ .once = &.{
87023 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {87044 .{ ._, .v_pd, .cmp, .dst0y, .src0y, .src1y, .vp(switch (cc) {
...@@ -89392,6 +89413,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -89392,6 +89413,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
89392 .store, .store_safe => |air_tag| {89413 .store, .store_safe => |air_tag| {
89393 const bin_op = air_datas[@intFromEnum(inst)].bin_op;89414 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
89394 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });89415 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
89416 switch (ops[1].tracking(cg).short) {
89417 else => {},
89418 .register_mask => |src_reg_mask| {
89419 const ty = ops[1].typeOf(cg);
89420 const new_op1 = try cg.tempAllocReg(ty, abi.RegisterClass.gp);
89421 try cg.genSetReg(
89422 new_op1.tracking(cg).short.register,
89423 ty,
89424 .{ .register_mask = src_reg_mask },
89425 .{ .safety = switch (air_tag) {
89426 else => unreachable,
89427 .store => false,
89428 .store_safe => true,
89429 } },
89430 );
89431 try ops[1].die(cg);
89432 ops[1] = new_op1;
89433 },
89434 }
89395 cg.select(&.{}, &.{}, &ops, comptime &.{ .{89435 cg.select(&.{}, &.{}, &ops, comptime &.{ .{
89396 .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any },89436 .src_constraints = .{ .{ .ptr_bool_vec_elem = .byte }, .bool, .any },
89397 .patterns = &.{89437 .patterns = &.{
...@@ -127167,7 +127207,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127167,7 +127207,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127167 .dst_constraints = .{ .bool, .any },127207 .dst_constraints = .{ .bool, .any },
127168 .src_constraints = .{ .any_bool_vec, .any, .any },127208 .src_constraints = .{ .any_bool_vec, .any, .any },
127169 .patterns = &.{127209 .patterns = &.{
127170 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } },127210 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127171 },127211 },
127172 .dst_temps = .{ .{ .cc = .z }, .unused },127212 .dst_temps = .{ .{ .cc = .z }, .unused },
127173 .clobbers = .{ .eflags = true },127213 .clobbers = .{ .eflags = true },
...@@ -127179,7 +127219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127179,7 +127219,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127179 .dst_constraints = .{ .bool, .any },127219 .dst_constraints = .{ .bool, .any },
127180 .src_constraints = .{ .any_bool_vec, .any, .any },127220 .src_constraints = .{ .any_bool_vec, .any, .any },
127181 .patterns = &.{127221 .patterns = &.{
127182 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .inverted } }, .none, .none } },127222 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127183 },127223 },
127184 .dst_temps = .{ .{ .cc = .z }, .unused },127224 .dst_temps = .{ .{ .cc = .z }, .unused },
127185 .clobbers = .{ .eflags = true },127225 .clobbers = .{ .eflags = true },
...@@ -127191,7 +127231,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127191,7 +127231,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127191 .dst_constraints = .{ .bool, .any },127231 .dst_constraints = .{ .bool, .any },
127192 .src_constraints = .{ .any_bool_vec, .any, .any },127232 .src_constraints = .{ .any_bool_vec, .any, .any },
127193 .patterns = &.{127233 .patterns = &.{
127194 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },127234 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127195 },127235 },
127196 .extra_temps = .{127236 .extra_temps = .{
127197 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },127237 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -127217,7 +127257,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127217,7 +127257,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127217 .dst_constraints = .{ .bool, .any },127257 .dst_constraints = .{ .bool, .any },
127218 .src_constraints = .{ .any_bool_vec, .any, .any },127258 .src_constraints = .{ .any_bool_vec, .any, .any },
127219 .patterns = &.{127259 .patterns = &.{
127220 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },127260 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127221 },127261 },
127222 .extra_temps = .{127262 .extra_temps = .{
127223 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },127263 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -127243,7 +127283,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127243,7 +127283,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127243 .dst_constraints = .{ .bool, .any },127283 .dst_constraints = .{ .bool, .any },
127244 .src_constraints = .{ .any_bool_vec, .any, .any },127284 .src_constraints = .{ .any_bool_vec, .any, .any },
127245 .patterns = &.{127285 .patterns = &.{
127246 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },127286 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127247 },127287 },
127248 .extra_temps = .{127288 .extra_temps = .{
127249 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127289 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -127269,7 +127309,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127269,7 +127309,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127269 .dst_constraints = .{ .bool, .any },127309 .dst_constraints = .{ .bool, .any },
127270 .src_constraints = .{ .any_bool_vec, .any, .any },127310 .src_constraints = .{ .any_bool_vec, .any, .any },
127271 .patterns = &.{127311 .patterns = &.{
127272 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },127312 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127273 },127313 },
127274 .extra_temps = .{127314 .extra_temps = .{
127275 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127315 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -127295,7 +127335,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127295,7 +127335,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127295 .dst_constraints = .{ .bool, .any },127335 .dst_constraints = .{ .bool, .any },
127296 .src_constraints = .{ .any_bool_vec, .any, .any },127336 .src_constraints = .{ .any_bool_vec, .any, .any },
127297 .patterns = &.{127337 .patterns = &.{
127298 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } },127338 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127299 },127339 },
127300 .dst_temps = .{ .{ .cc = .z }, .unused },127340 .dst_temps = .{ .{ .cc = .z }, .unused },
127301 .clobbers = .{ .eflags = true },127341 .clobbers = .{ .eflags = true },
...@@ -127307,7 +127347,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127307,7 +127347,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127307 .dst_constraints = .{ .bool, .any },127347 .dst_constraints = .{ .bool, .any },
127308 .src_constraints = .{ .any_bool_vec, .any, .any },127348 .src_constraints = .{ .any_bool_vec, .any, .any },
127309 .patterns = &.{127349 .patterns = &.{
127310 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .inverted } }, .none, .none } },127350 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .inverted } }, .none, .none } },
127311 },127351 },
127312 .dst_temps = .{ .{ .cc = .z }, .unused },127352 .dst_temps = .{ .{ .cc = .z }, .unused },
127313 .clobbers = .{ .eflags = true },127353 .clobbers = .{ .eflags = true },
...@@ -127319,7 +127359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127319,7 +127359,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127319 .dst_constraints = .{ .bool, .any },127359 .dst_constraints = .{ .bool, .any },
127320 .src_constraints = .{ .any_bool_vec, .any, .any },127360 .src_constraints = .{ .any_bool_vec, .any, .any },
127321 .patterns = &.{127361 .patterns = &.{
127322 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },127362 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127323 },127363 },
127324 .extra_temps = .{127364 .extra_temps = .{
127325 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },127365 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
...@@ -127345,7 +127385,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127345,7 +127385,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127345 .dst_constraints = .{ .bool, .any },127385 .dst_constraints = .{ .bool, .any },
127346 .src_constraints = .{ .any_bool_vec, .any, .any },127386 .src_constraints = .{ .any_bool_vec, .any, .any },
127347 .patterns = &.{127387 .patterns = &.{
127348 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },127388 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127349 },127389 },
127350 .extra_temps = .{127390 .extra_temps = .{
127351 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },127391 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
...@@ -127371,7 +127411,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127371,7 +127411,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127371 .dst_constraints = .{ .bool, .any },127411 .dst_constraints = .{ .bool, .any },
127372 .src_constraints = .{ .any_bool_vec, .any, .any },127412 .src_constraints = .{ .any_bool_vec, .any, .any },
127373 .patterns = &.{127413 .patterns = &.{
127374 .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } },127414 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .msb } }, .none, .none } },
127375 },127415 },
127376 .extra_temps = .{127416 .extra_temps = .{
127377 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127417 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -127572,7 +127612,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127572,7 +127612,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127572 .dst_constraints = .{ .bool, .any },127612 .dst_constraints = .{ .bool, .any },
127573 .src_constraints = .{ .any_bool_vec, .any, .any },127613 .src_constraints = .{ .any_bool_vec, .any, .any },
127574 .patterns = &.{127614 .patterns = &.{
127575 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } },127615 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127576 },127616 },
127577 .dst_temps = .{ .{ .cc = .nz }, .unused },127617 .dst_temps = .{ .{ .cc = .nz }, .unused },
127578 .clobbers = .{ .eflags = true },127618 .clobbers = .{ .eflags = true },
...@@ -127584,7 +127624,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127584,7 +127624,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127584 .dst_constraints = .{ .bool, .any },127624 .dst_constraints = .{ .bool, .any },
127585 .src_constraints = .{ .any_bool_vec, .any, .any },127625 .src_constraints = .{ .any_bool_vec, .any, .any },
127586 .patterns = &.{127626 .patterns = &.{
127587 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword, .is = .uninverted } }, .none, .none } },127627 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127588 },127628 },
127589 .dst_temps = .{ .{ .cc = .nz }, .unused },127629 .dst_temps = .{ .{ .cc = .nz }, .unused },
127590 .clobbers = .{ .eflags = true },127630 .clobbers = .{ .eflags = true },
...@@ -127596,7 +127636,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127596,7 +127636,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127596 .dst_constraints = .{ .bool, .any },127636 .dst_constraints = .{ .bool, .any },
127597 .src_constraints = .{ .any_bool_vec, .any, .any },127637 .src_constraints = .{ .any_bool_vec, .any, .any },
127598 .patterns = &.{127638 .patterns = &.{
127599 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },127639 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127600 },127640 },
127601 .extra_temps = .{127641 .extra_temps = .{
127602 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },127642 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -127622,7 +127662,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127622,7 +127662,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127622 .dst_constraints = .{ .bool, .any },127662 .dst_constraints = .{ .bool, .any },
127623 .src_constraints = .{ .any_bool_vec, .any, .any },127663 .src_constraints = .{ .any_bool_vec, .any, .any },
127624 .patterns = &.{127664 .patterns = &.{
127625 .{ .src = .{ .{ .all_reg_mask = .{ .size = .xword } }, .none, .none } },127665 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .sign_extend } }, .none, .none } },
127626 },127666 },
127627 .extra_temps = .{127667 .extra_temps = .{
127628 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },127668 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
...@@ -127648,7 +127688,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127648,7 +127688,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127648 .dst_constraints = .{ .bool, .any },127688 .dst_constraints = .{ .bool, .any },
127649 .src_constraints = .{ .any_bool_vec, .any, .any },127689 .src_constraints = .{ .any_bool_vec, .any, .any },
127650 .patterns = &.{127690 .patterns = &.{
127651 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },127691 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127652 },127692 },
127653 .extra_temps = .{127693 .extra_temps = .{
127654 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127694 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -127674,7 +127714,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127674,7 +127714,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127674 .dst_constraints = .{ .bool, .any },127714 .dst_constraints = .{ .bool, .any },
127675 .src_constraints = .{ .any_bool_vec, .any, .any },127715 .src_constraints = .{ .any_bool_vec, .any, .any },
127676 .patterns = &.{127716 .patterns = &.{
127677 .{ .src = .{ .{ .reg_mask = .{ .size = .xword } }, .none, .none } },127717 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .msb } }, .none, .none } },
127678 },127718 },
127679 .extra_temps = .{127719 .extra_temps = .{
127680 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127720 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -127700,7 +127740,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127700,7 +127740,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127700 .dst_constraints = .{ .bool, .any },127740 .dst_constraints = .{ .bool, .any },
127701 .src_constraints = .{ .any_bool_vec, .any, .any },127741 .src_constraints = .{ .any_bool_vec, .any, .any },
127702 .patterns = &.{127742 .patterns = &.{
127703 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } },127743 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127704 },127744 },
127705 .dst_temps = .{ .{ .cc = .nz }, .unused },127745 .dst_temps = .{ .{ .cc = .nz }, .unused },
127706 .clobbers = .{ .eflags = true },127746 .clobbers = .{ .eflags = true },
...@@ -127712,7 +127752,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127712,7 +127752,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127712 .dst_constraints = .{ .bool, .any },127752 .dst_constraints = .{ .bool, .any },
127713 .src_constraints = .{ .any_bool_vec, .any, .any },127753 .src_constraints = .{ .any_bool_vec, .any, .any },
127714 .patterns = &.{127754 .patterns = &.{
127715 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword, .is = .uninverted } }, .none, .none } },127755 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend, .is = .uninverted } }, .none, .none } },
127716 },127756 },
127717 .dst_temps = .{ .{ .cc = .nz }, .unused },127757 .dst_temps = .{ .{ .cc = .nz }, .unused },
127718 .clobbers = .{ .eflags = true },127758 .clobbers = .{ .eflags = true },
...@@ -127724,7 +127764,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127724,7 +127764,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127724 .dst_constraints = .{ .bool, .any },127764 .dst_constraints = .{ .bool, .any },
127725 .src_constraints = .{ .any_bool_vec, .any, .any },127765 .src_constraints = .{ .any_bool_vec, .any, .any },
127726 .patterns = &.{127766 .patterns = &.{
127727 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },127767 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127728 },127768 },
127729 .extra_temps = .{127769 .extra_temps = .{
127730 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },127770 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
...@@ -127750,7 +127790,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127750,7 +127790,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127750 .dst_constraints = .{ .bool, .any },127790 .dst_constraints = .{ .bool, .any },
127751 .src_constraints = .{ .any_bool_vec, .any, .any },127791 .src_constraints = .{ .any_bool_vec, .any, .any },
127752 .patterns = &.{127792 .patterns = &.{
127753 .{ .src = .{ .{ .all_reg_mask = .{ .size = .yword } }, .none, .none } },127793 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .sign_extend } }, .none, .none } },
127754 },127794 },
127755 .extra_temps = .{127795 .extra_temps = .{
127756 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },127796 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
...@@ -127776,7 +127816,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -127776,7 +127816,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
127776 .dst_constraints = .{ .bool, .any },127816 .dst_constraints = .{ .bool, .any },
127777 .src_constraints = .{ .any_bool_vec, .any, .any },127817 .src_constraints = .{ .any_bool_vec, .any, .any },
127778 .patterns = &.{127818 .patterns = &.{
127779 .{ .src = .{ .{ .reg_mask = .{ .size = .yword } }, .none, .none } },127819 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .msb } }, .none, .none } },
127780 },127820 },
127781 .extra_temps = .{127821 .extra_temps = .{
127782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },127822 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
...@@ -174528,6 +174568,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE...@@ -174528,6 +174568,7 @@ fn load(self: *CodeGen, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) InnerE
174528 .register_overflow,174568 .register_overflow,
174529 .register_mask,174569 .register_mask,
174530 .indirect_load_frame,174570 .indirect_load_frame,
174571 .indirect_mask,
174531 .register_tee,174572 .register_tee,
174532 .elementwise_gpr,174573 .elementwise_gpr,
174533 .elementwise_sse,174574 .elementwise_sse,
...@@ -174586,6 +174627,7 @@ fn store(...@@ -174586,6 +174627,7 @@ fn store(
174586 .register_overflow,174627 .register_overflow,
174587 .register_mask,174628 .register_mask,
174588 .indirect_load_frame,174629 .indirect_load_frame,
174630 .indirect_mask,
174589 .register_tee,174631 .register_tee,
174590 .elementwise_gpr,174632 .elementwise_gpr,
174591 .elementwise_sse,174633 .elementwise_sse,
...@@ -174636,6 +174678,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv:...@@ -174636,6 +174678,7 @@ fn genUnOpMir(self: *CodeGen, mir_tag: Mir.Inst.FixedTag, dst_ty: Type, dst_mcv:
174636 .register_overflow,174678 .register_overflow,
174637 .register_mask,174679 .register_mask,
174638 .indirect_load_frame,174680 .indirect_load_frame,
174681 .indirect_mask,
174639 .lea_frame,174682 .lea_frame,
174640 .lea_nav,174683 .lea_nav,
174641 .lea_uav,174684 .lea_uav,
...@@ -175336,6 +175379,7 @@ fn genBinOpMir(...@@ -175336,6 +175379,7 @@ fn genBinOpMir(
175336 .lea_lazy_sym,175379 .lea_lazy_sym,
175337 .lea_extern_func,175380 .lea_extern_func,
175338 .indirect_load_frame,175381 .indirect_load_frame,
175382 .indirect_mask,
175339 .register_tee,175383 .register_tee,
175340 .elementwise_gpr,175384 .elementwise_gpr,
175341 .elementwise_sse,175385 .elementwise_sse,
...@@ -175378,6 +175422,7 @@ fn genBinOpMir(...@@ -175378,6 +175422,7 @@ fn genBinOpMir(
175378 .register_overflow,175422 .register_overflow,
175379 .register_mask,175423 .register_mask,
175380 .indirect_load_frame,175424 .indirect_load_frame,
175425 .indirect_mask,
175381 .register_tee,175426 .register_tee,
175382 .elementwise_gpr,175427 .elementwise_gpr,
175383 .elementwise_sse,175428 .elementwise_sse,
...@@ -175555,6 +175600,7 @@ fn genBinOpMir(...@@ -175555,6 +175600,7 @@ fn genBinOpMir(
175555 .register_overflow,175600 .register_overflow,
175556 .register_mask,175601 .register_mask,
175557 .indirect_load_frame,175602 .indirect_load_frame,
175603 .indirect_mask,
175558 .register_tee,175604 .register_tee,
175559 .elementwise_gpr,175605 .elementwise_gpr,
175560 .elementwise_sse,175606 .elementwise_sse,
...@@ -175664,6 +175710,7 @@ fn genBinOpMir(...@@ -175664,6 +175710,7 @@ fn genBinOpMir(
175664 .register_overflow,175710 .register_overflow,
175665 .register_mask,175711 .register_mask,
175666 .indirect_load_frame,175712 .indirect_load_frame,
175713 .indirect_mask,
175667 .register_tee,175714 .register_tee,
175668 .elementwise_gpr,175715 .elementwise_gpr,
175669 .elementwise_sse,175716 .elementwise_sse,
...@@ -175793,7 +175840,13 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -175793,7 +175840,13 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {
175793 const result: MCValue = if (cg.mod.strip and cg.liveness.isUnused(inst)) .unreach else result: {175840 const result: MCValue = if (cg.mod.strip and cg.liveness.isUnused(inst)) .unreach else result: {
175794 const arg_ty = cg.typeOfIndex(inst);175841 const arg_ty = cg.typeOfIndex(inst);
175795 switch (src_mcv) {175842 switch (src_mcv) {
175796 .register, .register_pair, .load_frame => {175843 .register,
175844 .register_pair,
175845 .register_triple,
175846 .register_quadruple,
175847 .register_mask,
175848 .load_frame,
175849 => {
175797 for (src_mcv.getRegs()) |reg| cg.register_manager.getRegAssumeFree(reg, inst);175850 for (src_mcv.getRegs()) |reg| cg.register_manager.getRegAssumeFree(reg, inst);
175798 break :result src_mcv;175851 break :result src_mcv;
175799 },175852 },
...@@ -175803,6 +175856,22 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {...@@ -175803,6 +175856,22 @@ fn airArg(cg: *CodeGen, inst: Air.Inst.Index) !void {
175803 try cg.genCopy(arg_ty, dst_mcv, src_mcv, .{});175856 try cg.genCopy(arg_ty, dst_mcv, src_mcv, .{});
175804 break :result dst_mcv;175857 break :result dst_mcv;
175805 },175858 },
175859 .indirect_mask => |reg_mask| {
175860 cg.register_manager.getRegAssumeFree(reg_mask.reg, null);
175861 const dst_reg = try cg.register_manager.allocReg(inst, abi.RegisterClass.sse);
175862 const mask_size: u32 = @intCast(
175863 @divExact(reg_mask.info.scalar.bitSize(cg.target), 8) * arg_ty.vectorLen(zcu),
175864 );
175865 try cg.asmRegisterMemory(
175866 .{ if (cg.hasFeature(.avx)) .v_dqa else ._dqa, .mov },
175867 registerAlias(dst_reg, mask_size),
175868 .{
175869 .base = .{ .reg = reg_mask.reg },
175870 .mod = .{ .rm = .{ .size = .fromSize(mask_size) } },
175871 },
175872 );
175873 break :result .{ .register_mask = .{ .reg = dst_reg, .info = reg_mask.info } };
175874 },
175806 .indirect_load_frame => |frame_addr| {175875 .indirect_load_frame => |frame_addr| {
175807 const dst_mcv = try cg.allocRegOrMem(inst, false);175876 const dst_mcv = try cg.allocRegOrMem(inst, false);
175808 const ptr_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp);175877 const ptr_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp);
...@@ -176265,6 +176334,10 @@ fn genCall(cg: *CodeGen, info: union(enum) {...@@ -176265,6 +176334,10 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176265 for (regs) |reg| try cg.register_manager.getReg(reg, null);176334 for (regs) |reg| try cg.register_manager.getReg(reg, null);
176266 try reg_locks.appendSlice(allocator, &cg.register_manager.lockRegs(regs.len, regs));176335 try reg_locks.appendSlice(allocator, &cg.register_manager.lockRegs(regs.len, regs));
176267 },176336 },
176337 .register_mask => |reg_mask| {
176338 try cg.register_manager.getReg(reg_mask.reg, null);
176339 try reg_locks.append(allocator, cg.register_manager.lockReg(reg_mask.reg));
176340 },
176268 .indirect => |reg_off| {176341 .indirect => |reg_off| {
176269 frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu));176342 frame_index.* = try cg.allocFrameIndex(.initType(arg_ty, zcu));
176270 try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts);176343 try cg.genSetMem(.{ .frame = frame_index.* }, 0, arg_ty, src_arg, opts);
...@@ -176282,6 +176355,44 @@ fn genCall(cg: *CodeGen, info: union(enum) {...@@ -176282,6 +176355,44 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176282 opts,176355 opts,
176283 );176356 );
176284 },176357 },
176358 .indirect_mask => |reg_mask| {
176359 var src = switch (src_arg) {
176360 else => try cg.tempInit(arg_ty, src_arg),
176361 .air_ref => |src_ref| try cg.tempFromOperand(src_ref, false),
176362 };
176363 var dst = try cg.tempInit(arg_ty, .{ .register_mask = .{
176364 .reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse),
176365 .info = reg_mask.info,
176366 } });
176367 dst.copyToMask(&src, cg) catch |err| switch (err) {
176368 error.SelectFailed => return cg.fail("failed to select arg {f} {f} {f}", .{
176369 arg_ty.fmt(pt),
176370 dst.tracking(cg),
176371 src.tracking(cg),
176372 }),
176373 else => |e| return e,
176374 };
176375 try src.die(cg);
176376 const mask_size: u32 = @intCast(
176377 @divExact(reg_mask.info.scalar.bitSize(cg.target), 8) * arg_ty.vectorLen(zcu),
176378 );
176379 frame_index.* = try cg.allocFrameIndex(.init(.{
176380 .size = mask_size,
176381 .alignment = .fromByteUnits(std.math.ceilPowerOfTwoAssert(u32, mask_size)),
176382 }));
176383 try cg.asmMemoryRegister(
176384 .{ if (cg.hasFeature(.avx)) .v_dqa else ._dqa, .mov },
176385 .{
176386 .base = .{ .frame = frame_index.* },
176387 .mod = .{ .rm = .{ .size = .fromSize(mask_size) } },
176388 },
176389 registerAlias(dst.tracking(cg).short.register_mask.reg, mask_size),
176390 );
176391 try dst.die(cg);
176392
176393 try cg.register_manager.getReg(reg_mask.reg, null);
176394 try reg_locks.append(allocator, cg.register_manager.lockReg(reg_mask.reg));
176395 },
176285 .load_frame => {176396 .load_frame => {
176286 try cg.genCopy(arg_ty, dst_arg, src_arg, opts);176397 try cg.genCopy(arg_ty, dst_arg, src_arg, opts);
176287 try cg.freeValue(src_arg, .{});176398 try cg.freeValue(src_arg, .{});
...@@ -176396,7 +176507,7 @@ fn genCall(cg: *CodeGen, info: union(enum) {...@@ -176396,7 +176507,7 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176396176507
176397 var frame_offset: i32 = regs_frame_addr.info.frame_off;176508 var frame_offset: i32 = regs_frame_addr.info.frame_off;
176398 var arg_offset = elem_size * param_gpr_len;176509 var arg_offset = elem_size * param_gpr_len;
176399 while (arg_size - arg_offset != 0) : ({176510 while (arg_size - arg_offset > 0) : ({
176400 frame_offset += 8;176511 frame_offset += 8;
176401 arg_offset += elem_size;176512 arg_offset += elem_size;
176402 }) try cg.genSetMem(176513 }) try cg.genSetMem(
...@@ -176458,8 +176569,28 @@ fn genCall(cg: *CodeGen, info: union(enum) {...@@ -176458,8 +176569,28 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176458 .register_triple,176569 .register_triple,
176459 .register_quadruple,176570 .register_quadruple,
176460 => try cg.genCopy(arg_ty, dst_arg, src_arg, opts),176571 => try cg.genCopy(arg_ty, dst_arg, src_arg, opts),
176461 .indirect => |reg_off| try cg.genSetReg(reg_off.reg, .usize, .{176572 .register_mask => {
176462 .lea_frame = .{ .index = frame_index, .off = -reg_off.off },176573 var src = switch (src_arg) {
176574 else => try cg.tempInit(arg_ty, src_arg),
176575 .air_ref => |src_ref| try cg.tempFromOperand(src_ref, false),
176576 };
176577 var dst = try cg.tempInit(arg_ty, dst_arg);
176578 dst.copyToMask(&src, cg) catch |err| switch (err) {
176579 error.SelectFailed => return cg.fail("failed to select arg {f} {f} {f}", .{
176580 arg_ty.fmt(pt),
176581 dst.tracking(cg),
176582 src.tracking(cg),
176583 }),
176584 else => |e| return e,
176585 };
176586 try src.die(cg);
176587 try dst.die(cg);
176588 },
176589 .indirect => |dst_reg_off| try cg.genSetReg(dst_reg_off.reg, .usize, .{
176590 .lea_frame = .{ .index = frame_index, .off = -dst_reg_off.off },
176591 }, opts),
176592 .indirect_mask => |dst_reg_mask| try cg.genSetReg(dst_reg_mask.reg, .usize, .{
176593 .lea_frame = .{ .index = frame_index },
176463 }, opts),176594 }, opts),
176464 .register_tee => |dst_regs| {176595 .register_tee => |dst_regs| {
176465 try cg.genSetReg(dst_regs[0], arg_ty, src_arg, opts);176596 try cg.genSetReg(dst_regs[0], arg_ty, src_arg, opts);
...@@ -176570,31 +176701,43 @@ fn genCall(cg: *CodeGen, info: union(enum) {...@@ -176570,31 +176701,43 @@ fn genCall(cg: *CodeGen, info: union(enum) {
176570 return call_info.return_value.short;176701 return call_info.return_value.short;
176571}176702}
176572176703
176573fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {176704fn airRet(cg: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
176574 const pt = self.pt;176705 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176575 const zcu = pt.zcu;
176576 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176577176706
176578 const ret_ty = self.fn_type.fnReturnType(zcu);176707 const ret_ty = cg.fn_type.fnReturnType(cg.pt.zcu);
176579 switch (self.ret_mcv.short) {176708 switch (cg.ret_mcv.short) {
176580 .none => {},176709 .none => {},
176581 .register => |reg| {176710 .register => |reg| {
176582 const reg_lock = self.register_manager.lockRegAssumeUnused(reg);176711 const reg_lock = cg.register_manager.lockRegAssumeUnused(reg);
176583 defer self.register_manager.unlockReg(reg_lock);176712 defer cg.register_manager.unlockReg(reg_lock);
176584 try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });176713 try cg.genCopy(ret_ty, cg.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176585 },176714 },
176586 inline .register_pair, .register_triple, .register_quadruple => |regs| {176715 inline .register_pair, .register_triple, .register_quadruple => |regs| {
176587 const reg_locks = self.register_manager.lockRegsAssumeUnused(regs.len, regs);176716 const reg_locks = cg.register_manager.lockRegsAssumeUnused(regs.len, regs);
176588 defer for (reg_locks) |reg_lock| self.register_manager.unlockReg(reg_lock);176717 defer for (reg_locks) |reg_lock| cg.register_manager.unlockReg(reg_lock);
176589 try self.genCopy(ret_ty, self.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });176718 try cg.genCopy(ret_ty, cg.ret_mcv.short, .{ .air_ref = un_op }, .{ .safety = safety });
176719 },
176720 .register_mask => {
176721 var src = try cg.tempFromOperand(un_op, true);
176722 var dst = try cg.tempInit(ret_ty, cg.ret_mcv.short);
176723 dst.copyToMask(&src, cg) catch |err| switch (err) {
176724 error.SelectFailed => return cg.fail("failed to select ret {f} {f} {f}", .{
176725 ret_ty.fmt(cg.pt),
176726 dst.tracking(cg),
176727 src.tracking(cg),
176728 }),
176729 else => |e| return e,
176730 };
176731 try src.die(cg);
176732 try dst.die(cg);
176590 },176733 },
176591 .indirect => |reg_off| {176734 .indirect => |reg_off| {
176592 try self.register_manager.getReg(reg_off.reg, null);176735 try cg.register_manager.getReg(reg_off.reg, null);
176593 const lock = self.register_manager.lockRegAssumeUnused(reg_off.reg);176736 const lock = cg.register_manager.lockRegAssumeUnused(reg_off.reg);
176594 defer self.register_manager.unlockReg(lock);176737 defer cg.register_manager.unlockReg(lock);
176595176738
176596 try self.genSetReg(reg_off.reg, .usize, self.ret_mcv.long, .{});176739 try cg.genSetReg(reg_off.reg, .usize, cg.ret_mcv.long, .{});
176597 try self.genSetMem(176740 try cg.genSetMem(
176598 .{ .reg = reg_off.reg },176741 .{ .reg = reg_off.reg },
176599 reg_off.off,176742 reg_off.off,
176600 ret_ty,176743 ret_ty,
...@@ -176604,59 +176747,78 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {...@@ -176604,59 +176747,78 @@ fn airRet(self: *CodeGen, inst: Air.Inst.Index, safety: bool) !void {
176604 },176747 },
176605 else => unreachable,176748 else => unreachable,
176606 }176749 }
176607 self.ret_mcv.liveOut(self, inst);176750 cg.ret_mcv.liveOut(cg, inst);
176608176751
176609 if (self.err_ret_trace_reg != .none) {176752 if (cg.err_ret_trace_reg != .none) {
176610 if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {176753 if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176611 if (switch (err_ret_trace.short) {176754 if (switch (err_ret_trace.short) {
176612 .register => |reg| self.err_ret_trace_reg != reg,176755 .register => |reg| cg.err_ret_trace_reg != reg,
176613 else => true,176756 else => true,
176614 }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{});176757 }) try cg.genSetReg(cg.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176615 err_ret_trace.liveOut(self, err_ret_trace_index);176758 err_ret_trace.liveOut(cg, err_ret_trace_index);
176616 }176759 }
176617 }176760 }
176618176761
176619 try self.finishAir(inst, .unreach, .{ un_op, .none, .none });176762 try cg.finishAir(inst, .unreach, .{ un_op, .none, .none });
176620176763
176621 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction176764 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
176622 // which is available if the jump is 127 bytes or less forward.176765 // which is available if the jump is 127 bytes or less forward.
176623 const jmp_reloc = try self.asmJmpReloc(undefined);176766 const jmp_reloc = try cg.asmJmpReloc(undefined);
176624 try self.epilogue_relocs.append(self.gpa, jmp_reloc);176767 try cg.epilogue_relocs.append(cg.gpa, jmp_reloc);
176625}176768}
176626176769
176627fn airRetLoad(self: *CodeGen, inst: Air.Inst.Index) !void {176770fn airRetLoad(cg: *CodeGen, inst: Air.Inst.Index) !void {
176628 const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;176771 const un_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
176629 const ptr = try self.resolveInst(un_op);176772 switch (cg.ret_mcv.short) {
176630
176631 const ptr_ty = self.typeOf(un_op);
176632 switch (self.ret_mcv.short) {
176633 .none => {},176773 .none => {},
176634 .register,176774 .register,
176635 .register_pair,176775 .register_pair,
176636 .register_triple,176776 .register_triple,
176637 .register_quadruple,176777 .register_quadruple,
176638 => try self.load(self.ret_mcv.short, ptr_ty, ptr),176778 => try cg.load(cg.ret_mcv.short, cg.typeOf(un_op), try cg.resolveInst(un_op)),
176639 .indirect => |reg_off| try self.genSetReg(reg_off.reg, ptr_ty, ptr, .{}),176779 .register_mask => {
176780 var ptr = try cg.tempFromOperand(un_op, true);
176781 const ret_ty = ptr.typeOf(cg).childType(cg.pt.zcu);
176782 var src = try ptr.load(ret_ty, .{}, cg);
176783 try ptr.die(cg);
176784 var dst = try cg.tempInit(ret_ty, cg.ret_mcv.short);
176785 dst.copyToMask(&src, cg) catch |err| switch (err) {
176786 error.SelectFailed => return cg.fail("failed to select ret_load {f} {f} {f}", .{
176787 ret_ty.fmt(cg.pt),
176788 dst.tracking(cg),
176789 src.tracking(cg),
176790 }),
176791 else => |e| return e,
176792 };
176793 try src.die(cg);
176794 try dst.die(cg);
176795 },
176796 .indirect => |dst_reg_off| try cg.genSetReg(
176797 dst_reg_off.reg,
176798 cg.typeOf(un_op),
176799 try cg.resolveInst(un_op),
176800 .{},
176801 ),
176640 else => unreachable,176802 else => unreachable,
176641 }176803 }
176642 self.ret_mcv.liveOut(self, inst);176804 cg.ret_mcv.liveOut(cg, inst);
176643176805
176644 if (self.err_ret_trace_reg != .none) {176806 if (cg.err_ret_trace_reg != .none) {
176645 if (self.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {176807 if (cg.inst_tracking.getPtr(err_ret_trace_index)) |err_ret_trace| {
176646 if (switch (err_ret_trace.short) {176808 if (switch (err_ret_trace.short) {
176647 .register => |reg| self.err_ret_trace_reg != reg,176809 .register => |reg| cg.err_ret_trace_reg != reg,
176648 else => true,176810 else => true,
176649 }) try self.genSetReg(self.err_ret_trace_reg, .usize, err_ret_trace.short, .{});176811 }) try cg.genSetReg(cg.err_ret_trace_reg, .usize, err_ret_trace.short, .{});
176650 err_ret_trace.liveOut(self, err_ret_trace_index);176812 err_ret_trace.liveOut(cg, err_ret_trace_index);
176651 }176813 }
176652 }176814 }
176653176815
176654 try self.finishAir(inst, .unreach, .{ un_op, .none, .none });176816 try cg.finishAir(inst, .unreach, .{ un_op, .none, .none });
176655176817
176656 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction176818 // TODO optimization opportunity: figure out when we can emit this as a 2 byte instruction
176657 // which is available if the jump is 127 bytes or less forward.176819 // which is available if the jump is 127 bytes or less forward.
176658 const jmp_reloc = try self.asmJmpReloc(undefined);176820 const jmp_reloc = try cg.asmJmpReloc(undefined);
176659 try self.epilogue_relocs.append(self.gpa, jmp_reloc);176821 try cg.epilogue_relocs.append(cg.gpa, jmp_reloc);
176660}176822}
176661176823
176662fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {176824fn airTry(self: *CodeGen, inst: Air.Inst.Index) !void {
...@@ -178640,6 +178802,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C...@@ -178640,6 +178802,7 @@ fn genCopy(self: *CodeGen, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: C
178640 .register_overflow,178802 .register_overflow,
178641 .register_mask,178803 .register_mask,
178642 .indirect_load_frame,178804 .indirect_load_frame,
178805 .indirect_mask,
178643 .lea_frame,178806 .lea_frame,
178644 .lea_nav,178807 .lea_nav,
178645 .lea_uav,178808 .lea_uav,
...@@ -178853,6 +179016,7 @@ fn genSetReg(...@@ -178853,6 +179016,7 @@ fn genSetReg(
178853 .unreach,179016 .unreach,
178854 .dead,179017 .dead,
178855 .indirect_load_frame,179018 .indirect_load_frame,
179019 .indirect_mask,
178856 .register_tee,179020 .register_tee,
178857 .elementwise_gpr,179021 .elementwise_gpr,
178858 .elementwise_sse,179022 .elementwise_sse,
...@@ -179282,33 +179446,105 @@ fn genSetReg(...@@ -179282,33 +179446,105 @@ fn genSetReg(
179282 const bits_lock = cg.register_manager.lockReg(bits_reg);179446 const bits_lock = cg.register_manager.lockReg(bits_reg);
179283 defer if (bits_lock) |lock| cg.register_manager.unlockReg(lock);179447 defer if (bits_lock) |lock| cg.register_manager.unlockReg(lock);
179284179448
179285 const pack_reg = switch (src_reg_mask.info.scalar) {179449 var mask_size: u32 = @intCast(
179286 else => src_reg_mask.reg,179450 @divExact(src_reg_mask.info.scalar.bitSize(cg.target), 8) * ty.vectorLen(zcu),
179287 .word => try cg.register_manager.allocReg(null, abi.RegisterClass.sse),179451 );
179452 const sign_reg = switch (src_reg_mask.info.kind) {
179453 .lsb => sign_reg: {
179454 const sign_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse);
179455 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
179456 const sign_alias = registerAlias(sign_reg, mask_size);
179457 if (cg.hasFeature(.avx2)) try cg.asmRegisterRegisterImmediate(
179458 .{ switch (src_reg_mask.info.scalar) {
179459 else => unreachable,
179460 .byte, .word => .vp_w,
179461 .dword => .vp_d,
179462 .qword => .vp_q,
179463 }, .sll },
179464 sign_alias,
179465 src_alias,
179466 .u(src_reg_mask.info.scalar.bitSize(cg.target) - 1),
179467 ) else {
179468 if (sign_alias != src_alias) try cg.asmRegisterRegister(
179469 .{ ._dqa, .mov },
179470 sign_alias,
179471 src_alias,
179472 );
179473 try cg.asmRegisterImmediate(
179474 .{ switch (src_reg_mask.info.scalar) {
179475 else => unreachable,
179476 .byte, .word => .p_w,
179477 .dword => .p_d,
179478 .qword => .p_q,
179479 }, .sll },
179480 sign_alias,
179481 .u(src_reg_mask.info.scalar.bitSize(cg.target) - 1),
179482 );
179483 }
179484 break :sign_reg sign_reg;
179485 },
179486 .zero_extend => sign_reg: {
179487 const sign_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.sse);
179488 const src_alias = registerAlias(src_reg_mask.reg, mask_size);
179489 const sign_alias = registerAlias(sign_reg, mask_size);
179490 if (has_avx) {
179491 try cg.asmRegisterRegisterRegister(
179492 .{ .vp_, .xor },
179493 sign_alias,
179494 sign_alias,
179495 sign_alias,
179496 );
179497 try cg.asmRegisterRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179498 else => unreachable,
179499 .byte => .vp_b,
179500 .word => .vp_w,
179501 .dword => .vp_d,
179502 .qword => .vp_q,
179503 }, .sub }, sign_alias, sign_alias, src_alias);
179504 } else {
179505 try cg.asmRegisterRegister(.{ .p_, .xor }, sign_alias, sign_alias);
179506 try cg.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179507 else => unreachable,
179508 .byte => .p_b,
179509 .word => .p_w,
179510 .dword => .p_d,
179511 .qword => .p_q,
179512 }, .sub }, sign_alias, src_alias);
179513 }
179514 break :sign_reg sign_reg;
179515 },
179516 .msb, .sign_extend => src_reg_mask.reg,
179288 };179517 };
179289 const pack_lock = cg.register_manager.lockReg(pack_reg);179518 const sign_lock = cg.register_manager.lockReg(sign_reg);
179290 defer if (pack_lock) |lock| cg.register_manager.unlockReg(lock);179519 defer if (sign_lock) |lock| cg.register_manager.unlockReg(lock);
179291179520
179292 var mask_size: u32 = @intCast(ty.vectorLen(zcu) * @divExact(src_reg_mask.info.scalar.bitSize(cg.target), 8));179521 const pack_reg = switch (src_reg_mask.info.scalar) {
179293 switch (src_reg_mask.info.scalar) {179522 else => sign_reg,
179294 else => {},179523 .word => pack_reg: {
179295 .word => {179524 const pack_reg = if (sign_reg == src_reg_mask.reg)
179296 const src_alias = registerAlias(src_reg_mask.reg, mask_size);179525 try cg.register_manager.allocReg(null, abi.RegisterClass.sse)
179526 else
179527 sign_reg;
179528 const sign_alias = registerAlias(sign_reg, mask_size);
179297 const pack_alias = registerAlias(pack_reg, mask_size);179529 const pack_alias = registerAlias(pack_reg, mask_size);
179298 if (has_avx) {179530 if (has_avx) {
179299 try cg.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, src_alias, src_alias);179531 try cg.asmRegisterRegisterRegister(.{ .vp_b, .ackssw }, pack_alias, sign_alias, sign_alias);
179300 } else {179532 } else {
179301 try cg.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, src_alias);179533 if (pack_alias != sign_alias) try cg.asmRegisterRegister(.{ ._dqa, .mov }, pack_alias, sign_alias);
179302 try cg.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);179534 try cg.asmRegisterRegister(.{ .p_b, .ackssw }, pack_alias, pack_alias);
179303 }179535 }
179304 mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;179536 mask_size = std.math.divCeil(u32, mask_size, 2) catch unreachable;
179537 break :pack_reg pack_reg;
179305 },179538 },
179306 }179539 };
179540 const pack_lock = cg.register_manager.lockReg(pack_reg);
179541 defer if (pack_lock) |lock| cg.register_manager.unlockReg(lock);
179542
179307 try cg.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {179543 try cg.asmRegisterRegister(.{ switch (src_reg_mask.info.scalar) {
179544 else => unreachable,
179308 .byte, .word => if (has_avx) .vp_b else .p_b,179545 .byte, .word => if (has_avx) .vp_b else .p_b,
179309 .dword => if (has_avx) .v_ps else ._ps,179546 .dword => if (has_avx) .v_ps else ._ps,
179310 .qword => if (has_avx) .v_pd else ._pd,179547 .qword => if (has_avx) .v_pd else ._pd,
179311 else => unreachable,
179312 }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size));179548 }, .movmsk }, bits_reg.to32(), registerAlias(pack_reg, mask_size));
179313 if (src_reg_mask.info.inverted) try cg.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size));179549 if (src_reg_mask.info.inverted) try cg.asmRegister(.{ ._, .not }, registerAlias(bits_reg, abi_size));
179314 try cg.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{});179550 try cg.genSetReg(dst_reg, ty, .{ .register = bits_reg }, .{});
...@@ -179432,6 +179668,7 @@ fn genSetMem(...@@ -179432,6 +179668,7 @@ fn genSetMem(
179432 .unreach,179668 .unreach,
179433 .dead,179669 .dead,
179434 .indirect_load_frame,179670 .indirect_load_frame,
179671 .indirect_mask,
179435 .register_tee,179672 .register_tee,
179436 .elementwise_gpr,179673 .elementwise_gpr,
179437 .elementwise_sse,179674 .elementwise_sse,
...@@ -180697,32 +180934,32 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {...@@ -180697,32 +180934,32 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void {
180697 const dst_lock = self.register_manager.lockReg(dst_reg);180934 const dst_lock = self.register_manager.lockReg(dst_reg);
180698 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);180935 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
180699180936
180700 const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .all and180937 const mir_tag = @as(?Mir.Inst.FixedTag, if ((pred_reg_mask.info.kind == .sign_extend and
180701 elem_ty.toIntern() != .f32_type and elem_ty.toIntern() != .f64_type) or pred_reg_mask.info.scalar == .byte)180938 elem_ty.toIntern() != .f32_type and elem_ty.toIntern() != .f64_type) or pred_reg_mask.info.scalar == .byte)
180702 if (has_avx)180939 if (has_avx)
180703 .{ .vp_b, .blendv }180940 .{ .vp_b, .blendv }
180704 else if (has_blend)180941 else if (has_blend)
180705 .{ .p_b, .blendv }180942 .{ .p_b, .blendv }
180706 else if (pred_reg_mask.info.kind == .all)180943 else if (pred_reg_mask.info.kind == .sign_extend)
180707 .{ .p_, undefined }180944 .{ .p_, undefined }
180708 else180945 else
180709 null180946 null
180710 else if ((pred_reg_mask.info.kind == .all and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or180947 else if ((pred_reg_mask.info.kind == .sign_extend and (elem_ty.toIntern() != .f64_type or !self.hasFeature(.sse2))) or
180711 pred_reg_mask.info.scalar == .dword)180948 pred_reg_mask.info.scalar == .dword)
180712 if (has_avx)180949 if (has_avx)
180713 .{ .v_ps, .blendv }180950 .{ .v_ps, .blendv }
180714 else if (has_blend)180951 else if (has_blend)
180715 .{ ._ps, .blendv }180952 .{ ._ps, .blendv }
180716 else if (pred_reg_mask.info.kind == .all)180953 else if (pred_reg_mask.info.kind == .sign_extend)
180717 .{ ._ps, undefined }180954 .{ ._ps, undefined }
180718 else180955 else
180719 null180956 null
180720 else if (pred_reg_mask.info.kind == .all or pred_reg_mask.info.scalar == .qword)180957 else if (pred_reg_mask.info.kind == .sign_extend or pred_reg_mask.info.scalar == .qword)
180721 if (has_avx)180958 if (has_avx)
180722 .{ .v_pd, .blendv }180959 .{ .v_pd, .blendv }
180723 else if (has_blend)180960 else if (has_blend)
180724 .{ ._pd, .blendv }180961 .{ ._pd, .blendv }
180725 else if (pred_reg_mask.info.kind == .all)180962 else if (pred_reg_mask.info.kind == .sign_extend)
180726 .{ ._pd, undefined }180963 .{ ._pd, undefined }
180727 else180964 else
180728 null180965 null
...@@ -181635,6 +181872,17 @@ fn resolveCallingConventionValues(...@@ -181635,6 +181872,17 @@ fn resolveCallingConventionValues(
181635 ret_gpr = ret_gpr[1..];181872 ret_gpr = ret_gpr[1..];
181636 param_gpr_index += 1;181873 param_gpr_index += 1;
181637 },181874 },
181875 .bool_vector_mask => {
181876 const len = ret_ty.vectorLen(zcu);
181877 const elem_size =
181878 @divExact(16, std.math.ceilPowerOfTwoAssert(u32, @min(len, 16)));
181879 ret_tracking[ret_tracking_len] = .init(.{ .register_mask = .{
181880 .reg = registerAlias(ret_sse[0], elem_size * len),
181881 .info = .{ .kind = .lsb, .scalar = .fromSize(elem_size) },
181882 } });
181883 ret_tracking_len += 1;
181884 ret_sse = ret_sse[1..];
181885 },
181638 .integer_per_element => {181886 .integer_per_element => {
181639 const len: u32 = @intCast(ret_ty.vectorLen(zcu));181887 const len: u32 = @intCast(ret_ty.vectorLen(zcu));
181640 const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8));181888 const alias_size: u32 = @intCast(@min(ret_ty.childType(zcu).abiSize(zcu), 8));
...@@ -181809,6 +182057,17 @@ fn resolveCallingConventionValues(...@@ -181809,6 +182057,17 @@ fn resolveCallingConventionValues(
181809 else => unreachable,182057 else => unreachable,
181810 },182058 },
181811 .none => {},182059 .none => {},
182060 .bool_vector_mask => {
182061 arg_mcv[arg_mcv_len] = .{ .indirect_mask = .{
182062 .reg = param_gpr[param_gpr_index].to64(),
182063 .info = .{ .kind = .lsb, .scalar = .fromSize(@divExact(
182064 16,
182065 std.math.ceilPowerOfTwoAssert(u32, @min(ty.vectorLen(zcu), 16)),
182066 )) },
182067 } };
182068 arg_mcv_len += 1;
182069 param_gpr_index += 1;
182070 },
181812 .integer_per_element,182071 .integer_per_element,
181813 .sse_per_element,182072 .sse_per_element,
181814 .sse_per_xword,182073 .sse_per_xword,
...@@ -182441,6 +182700,7 @@ const Temp = struct {...@@ -182441,6 +182700,7 @@ const Temp = struct {
182441 .memory,182700 .memory,
182442 .indirect,182701 .indirect,
182443 .indirect_load_frame,182702 .indirect_load_frame,
182703 .indirect_mask,
182444 .lea_frame,182704 .lea_frame,
182445 .load_nav,182705 .load_nav,
182446 .lea_nav,182706 .lea_nav,
...@@ -182918,6 +183178,7 @@ const Temp = struct {...@@ -182918,6 +183178,7 @@ const Temp = struct {
182918 .register_overflow,183178 .register_overflow,
182919 .register_mask,183179 .register_mask,
182920 .indirect_load_frame,183180 .indirect_load_frame,
183181 .indirect_mask,
182921 .register_tee,183182 .register_tee,
182922 .elementwise_gpr,183183 .elementwise_gpr,
182923 .elementwise_sse,183184 .elementwise_sse,
...@@ -183423,6 +183684,7 @@ const Temp = struct {...@@ -183423,6 +183684,7 @@ const Temp = struct {
183423 .x87up,183684 .x87up,
183424 .none,183685 .none,
183425 .win_i128,183686 .win_i128,
183687 .bool_vector_mask,
183426 .integer_per_element,183688 .integer_per_element,
183427 .sse_per_element,183689 .sse_per_element,
183428 .sse_sse_x87_per_qword,183690 .sse_sse_x87_per_qword,
...@@ -183440,6 +183702,7 @@ const Temp = struct {...@@ -183440,6 +183702,7 @@ const Temp = struct {
183440 .x87up,183702 .x87up,
183441 .none,183703 .none,
183442 .win_i128,183704 .win_i128,
183705 .bool_vector_mask,
183443 .integer_per_element,183706 .integer_per_element,
183444 .sse_per_element,183707 .sse_per_element,
183445 .sse_sse_x87_per_qword,183708 .sse_sse_x87_per_qword,
...@@ -183464,6 +183727,7 @@ const Temp = struct {...@@ -183464,6 +183727,7 @@ const Temp = struct {
183464 .memory,183727 .memory,
183465 .none,183728 .none,
183466 .win_i128,183729 .win_i128,
183730 .bool_vector_mask,
183467 .integer_per_element,183731 .integer_per_element,
183468 .sse_per_element,183732 .sse_per_element,
183469 .sse_sse_x87_per_qword,183733 .sse_sse_x87_per_qword,
...@@ -183502,6 +183766,1331 @@ const Temp = struct {...@@ -183502,6 +183766,1331 @@ const Temp = struct {
183502 try cg.asmOpOnly(.{ .@"rep _sb", .sto });183766 try cg.asmOpOnly(.{ .@"rep _sb", .sto });
183503 }183767 }
183504183768
183769 fn copyToMask(dst: *Temp, src: *Temp, cg: *CodeGen) Select.Error!void {
183770 var ops: [2]Temp = .{ dst.*, src.* };
183771 try cg.select(&.{}, &.{}, &ops, comptime &.{ .{
183772 .required_features = .{ .avx2, null, null, null },
183773 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183774 .patterns = &.{
183775 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
183776 },
183777 .extra_temps = .{
183778 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183779 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183780 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183781 .unused,
183782 .unused,
183783 .unused,
183784 .unused,
183785 .unused,
183786 .unused,
183787 .unused,
183788 .unused,
183789 },
183790 .each = .{ .once = &.{
183791 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
183792 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183793 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183794 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183795 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183796 } },
183797 }, .{
183798 .required_features = .{ .avx2, null, null, null },
183799 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183800 .patterns = &.{
183801 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183802 },
183803 .extra_temps = .{
183804 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183805 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183806 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183807 .unused,
183808 .unused,
183809 .unused,
183810 .unused,
183811 .unused,
183812 .unused,
183813 .unused,
183814 .unused,
183815 },
183816 .each = .{ .once = &.{
183817 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
183818 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183819 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183820 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183821 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183822 } },
183823 }, .{
183824 .required_features = .{ .avx2, null, null, null },
183825 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183826 .patterns = &.{
183827 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
183828 },
183829 .extra_temps = .{
183830 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183831 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183832 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183833 .unused,
183834 .unused,
183835 .unused,
183836 .unused,
183837 .unused,
183838 .unused,
183839 .unused,
183840 .unused,
183841 },
183842 .each = .{ .once = &.{
183843 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
183844 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183845 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183846 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183847 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183848 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183849 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183850 } },
183851 }, .{
183852 .required_features = .{ .avx2, null, null, null },
183853 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183854 .patterns = &.{
183855 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
183856 },
183857 .extra_temps = .{
183858 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183859 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183860 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183861 .unused,
183862 .unused,
183863 .unused,
183864 .unused,
183865 .unused,
183866 .unused,
183867 .unused,
183868 .unused,
183869 },
183870 .each = .{ .once = &.{
183871 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
183872 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183873 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183874 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183875 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183876 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183877 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183878 } },
183879 }, .{
183880 .required_features = .{ .avx, null, null, null },
183881 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183882 .patterns = &.{
183883 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183884 },
183885 .extra_temps = .{
183886 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183887 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183888 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183889 .unused,
183890 .unused,
183891 .unused,
183892 .unused,
183893 .unused,
183894 .unused,
183895 .unused,
183896 .unused,
183897 },
183898 .each = .{ .once = &.{
183899 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183900 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
183901 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183902 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183903 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183904 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183905 } },
183906 }, .{
183907 .required_features = .{ .avx, null, null, null },
183908 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183909 .patterns = &.{
183910 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
183911 },
183912 .extra_temps = .{
183913 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183914 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183915 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183916 .unused,
183917 .unused,
183918 .unused,
183919 .unused,
183920 .unused,
183921 .unused,
183922 .unused,
183923 .unused,
183924 },
183925 .each = .{ .once = &.{
183926 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183927 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
183928 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183929 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183930 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
183931 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
183932 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
183933 .{ ._, .vp_q, .sub, .src0x, .tmp2x, .src0x, ._ },
183934 } },
183935 }, .{
183936 .required_features = .{ .ssse3, null, null, null },
183937 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183938 .patterns = &.{
183939 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183940 },
183941 .extra_temps = .{
183942 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183943 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183944 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
183945 .unused,
183946 .unused,
183947 .unused,
183948 .unused,
183949 .unused,
183950 .unused,
183951 .unused,
183952 .unused,
183953 },
183954 .each = .{ .once = &.{
183955 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
183956 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
183957 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
183958 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183959 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
183960 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
183961 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
183962 } },
183963 }, .{
183964 .required_features = .{ .ssse3, null, null, null },
183965 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183966 .patterns = &.{
183967 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
183968 },
183969 .extra_temps = .{
183970 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
183971 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
183972 .unused,
183973 .unused,
183974 .unused,
183975 .unused,
183976 .unused,
183977 .unused,
183978 .unused,
183979 .unused,
183980 .unused,
183981 },
183982 .each = .{ .once = &.{
183983 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
183984 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
183985 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
183986 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
183987 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
183988 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
183989 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
183990 .{ ._, .p_q, .sub, .src0x, .src1x, ._, ._ },
183991 } },
183992 }, .{
183993 .required_features = .{ .sse2, null, null, null },
183994 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
183995 .patterns = &.{
183996 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
183997 },
183998 .extra_temps = .{
183999 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
184000 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184001 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184002 .unused,
184003 .unused,
184004 .unused,
184005 .unused,
184006 .unused,
184007 .unused,
184008 .unused,
184009 .unused,
184010 },
184011 .each = .{ .once = &.{
184012 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184013 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184014 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184015 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184016 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184017 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184018 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184019 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184020 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184021 } },
184022 }, .{
184023 .required_features = .{ .sse2, null, null, null },
184024 .src_constraints = .{ .{ .exact_bool_vec = 2 }, .{ .exact_bool_vec = 2 }, .any },
184025 .patterns = &.{
184026 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184027 },
184028 .extra_temps = .{
184029 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 8 } } },
184030 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184031 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184032 .unused,
184033 .unused,
184034 .unused,
184035 .unused,
184036 .unused,
184037 .unused,
184038 .unused,
184039 .unused,
184040 },
184041 .each = .{ .once = &.{
184042 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184043 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184044 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184045 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184046 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184047 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184048 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184049 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184050 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184051 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184052 .{ ._, .p_q, .sub, .src0x, .tmp2x, ._, ._ },
184053 } },
184054 }, .{
184055 .required_features = .{ .avx2, null, null, null },
184056 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184057 .patterns = &.{
184058 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184059 },
184060 .extra_temps = .{
184061 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184062 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184063 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184064 .unused,
184065 .unused,
184066 .unused,
184067 .unused,
184068 .unused,
184069 .unused,
184070 .unused,
184071 .unused,
184072 },
184073 .each = .{ .once = &.{
184074 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184075 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184076 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184077 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184078 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184079 } },
184080 }, .{
184081 .required_features = .{ .avx2, null, null, null },
184082 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184083 .patterns = &.{
184084 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184085 },
184086 .extra_temps = .{
184087 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184088 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184089 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184090 .unused,
184091 .unused,
184092 .unused,
184093 .unused,
184094 .unused,
184095 .unused,
184096 .unused,
184097 .unused,
184098 },
184099 .each = .{ .once = &.{
184100 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184101 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184102 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184103 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184104 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184105 } },
184106 }, .{
184107 .required_features = .{ .avx2, null, null, null },
184108 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184109 .patterns = &.{
184110 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184111 },
184112 .extra_temps = .{
184113 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184114 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184115 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184116 .unused,
184117 .unused,
184118 .unused,
184119 .unused,
184120 .unused,
184121 .unused,
184122 .unused,
184123 .unused,
184124 },
184125 .each = .{ .once = &.{
184126 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184127 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184128 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184129 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184130 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184131 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184132 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184133 } },
184134 }, .{
184135 .required_features = .{ .avx2, null, null, null },
184136 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184137 .patterns = &.{
184138 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184139 },
184140 .extra_temps = .{
184141 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184142 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184143 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184144 .unused,
184145 .unused,
184146 .unused,
184147 .unused,
184148 .unused,
184149 .unused,
184150 .unused,
184151 .unused,
184152 },
184153 .each = .{ .once = &.{
184154 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184155 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184156 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184157 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184158 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184159 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184160 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184161 } },
184162 }, .{
184163 .required_features = .{ .avx, null, null, null },
184164 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184165 .patterns = &.{
184166 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184167 },
184168 .extra_temps = .{
184169 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184170 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184171 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184172 .unused,
184173 .unused,
184174 .unused,
184175 .unused,
184176 .unused,
184177 .unused,
184178 .unused,
184179 .unused,
184180 },
184181 .each = .{ .once = &.{
184182 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184183 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184184 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184185 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184186 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184187 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184188 } },
184189 }, .{
184190 .required_features = .{ .avx, null, null, null },
184191 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184192 .patterns = &.{
184193 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184194 },
184195 .extra_temps = .{
184196 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184197 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184198 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184199 .unused,
184200 .unused,
184201 .unused,
184202 .unused,
184203 .unused,
184204 .unused,
184205 .unused,
184206 .unused,
184207 },
184208 .each = .{ .once = &.{
184209 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184210 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184211 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184212 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184213 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184214 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184215 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184216 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184217 } },
184218 }, .{
184219 .required_features = .{ .ssse3, null, null, null },
184220 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184221 .patterns = &.{
184222 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184223 },
184224 .extra_temps = .{
184225 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184226 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184227 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184228 .unused,
184229 .unused,
184230 .unused,
184231 .unused,
184232 .unused,
184233 .unused,
184234 .unused,
184235 .unused,
184236 },
184237 .each = .{ .once = &.{
184238 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
184239 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184240 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
184241 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184242 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184243 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184244 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184245 } },
184246 }, .{
184247 .required_features = .{ .ssse3, null, null, null },
184248 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184249 .patterns = &.{
184250 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184251 },
184252 .extra_temps = .{
184253 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184254 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184255 .unused,
184256 .unused,
184257 .unused,
184258 .unused,
184259 .unused,
184260 .unused,
184261 .unused,
184262 .unused,
184263 .unused,
184264 },
184265 .each = .{ .once = &.{
184266 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184267 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
184268 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184269 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184270 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184271 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184272 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184273 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184274 } },
184275 }, .{
184276 .required_features = .{ .sse2, null, null, null },
184277 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184278 .patterns = &.{
184279 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184280 },
184281 .extra_temps = .{
184282 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184283 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184284 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184285 .unused,
184286 .unused,
184287 .unused,
184288 .unused,
184289 .unused,
184290 .unused,
184291 .unused,
184292 .unused,
184293 },
184294 .each = .{ .once = &.{
184295 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184296 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184297 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184298 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184299 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184300 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184301 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184302 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184303 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184304 } },
184305 }, .{
184306 .required_features = .{ .sse2, null, null, null },
184307 .src_constraints = .{ .{ .exact_bool_vec = 4 }, .{ .exact_bool_vec = 4 }, .any },
184308 .patterns = &.{
184309 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184310 },
184311 .extra_temps = .{
184312 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 4 } } },
184313 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184314 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184315 .unused,
184316 .unused,
184317 .unused,
184318 .unused,
184319 .unused,
184320 .unused,
184321 .unused,
184322 .unused,
184323 },
184324 .each = .{ .once = &.{
184325 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184326 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184327 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184328 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184329 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184330 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184331 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184332 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184333 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184334 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184335 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184336 } },
184337 }, .{
184338 .required_features = .{ .avx2, null, null, null },
184339 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184340 .patterns = &.{
184341 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184342 },
184343 .extra_temps = .{
184344 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184345 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184346 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184347 .unused,
184348 .unused,
184349 .unused,
184350 .unused,
184351 .unused,
184352 .unused,
184353 .unused,
184354 .unused,
184355 },
184356 .each = .{ .once = &.{
184357 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184358 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184359 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184360 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184361 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184362 } },
184363 }, .{
184364 .required_features = .{ .avx2, null, null, null },
184365 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184366 .patterns = &.{
184367 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184368 },
184369 .extra_temps = .{
184370 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184371 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184372 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184373 .unused,
184374 .unused,
184375 .unused,
184376 .unused,
184377 .unused,
184378 .unused,
184379 .unused,
184380 .unused,
184381 },
184382 .each = .{ .once = &.{
184383 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184384 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184385 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184386 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184387 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184388 } },
184389 }, .{
184390 .required_features = .{ .avx2, null, null, null },
184391 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184392 .patterns = &.{
184393 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184394 },
184395 .extra_temps = .{
184396 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184397 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184398 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184399 .unused,
184400 .unused,
184401 .unused,
184402 .unused,
184403 .unused,
184404 .unused,
184405 .unused,
184406 .unused,
184407 },
184408 .each = .{ .once = &.{
184409 .{ ._, .vp_b, .broadcast, .src0x, .src1b, ._, ._ },
184410 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184411 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184412 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184413 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184414 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184415 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184416 } },
184417 }, .{
184418 .required_features = .{ .avx2, null, null, null },
184419 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184420 .patterns = &.{
184421 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184422 },
184423 .extra_temps = .{
184424 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184425 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184426 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184427 .unused,
184428 .unused,
184429 .unused,
184430 .unused,
184431 .unused,
184432 .unused,
184433 .unused,
184434 .unused,
184435 },
184436 .each = .{ .once = &.{
184437 .{ ._, .vp_b, .broadcast, .src0x, .src1x, ._, ._ },
184438 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184439 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184440 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184441 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184442 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184443 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184444 } },
184445 }, .{
184446 .required_features = .{ .avx, null, null, null },
184447 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184448 .patterns = &.{
184449 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184450 },
184451 .extra_temps = .{
184452 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184453 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184454 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184455 .unused,
184456 .unused,
184457 .unused,
184458 .unused,
184459 .unused,
184460 .unused,
184461 .unused,
184462 .unused,
184463 },
184464 .each = .{ .once = &.{
184465 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184466 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184467 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184468 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184469 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184470 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184471 } },
184472 }, .{
184473 .required_features = .{ .avx, null, null, null },
184474 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184475 .patterns = &.{
184476 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184477 },
184478 .extra_temps = .{
184479 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184480 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184481 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184482 .unused,
184483 .unused,
184484 .unused,
184485 .unused,
184486 .unused,
184487 .unused,
184488 .unused,
184489 .unused,
184490 },
184491 .each = .{ .once = &.{
184492 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184493 .{ ._, .vp_b, .shuf, .src0x, .src1x, .tmp2x, ._ },
184494 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184495 .{ ._, .v_dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184496 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp2x, ._ },
184497 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp2x, ._ },
184498 .{ ._, .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
184499 .{ ._, .vp_b, .sub, .src0x, .tmp2x, .src0x, ._ },
184500 } },
184501 }, .{
184502 .required_features = .{ .ssse3, null, null, null },
184503 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184504 .patterns = &.{
184505 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184506 },
184507 .extra_temps = .{
184508 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184509 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184510 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184511 .unused,
184512 .unused,
184513 .unused,
184514 .unused,
184515 .unused,
184516 .unused,
184517 .unused,
184518 .unused,
184519 },
184520 .each = .{ .once = &.{
184521 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
184522 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184523 .{ ._, .p_b, .shuf, .src0x, .tmp2x, ._, ._ },
184524 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184525 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184526 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184527 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184528 } },
184529 }, .{
184530 .required_features = .{ .ssse3, null, null, null },
184531 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184532 .patterns = &.{
184533 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184534 },
184535 .extra_temps = .{
184536 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184537 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184538 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184539 .unused,
184540 .unused,
184541 .unused,
184542 .unused,
184543 .unused,
184544 .unused,
184545 .unused,
184546 .unused,
184547 },
184548 .each = .{ .once = &.{
184549 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184550 .{ ._, .p_b, .shuf, .src1x, .src0x, ._, ._ },
184551 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184552 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184553 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184554 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184555 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184556 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184557 } },
184558 }, .{
184559 .required_features = .{ .sse2, null, null, null },
184560 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184561 .patterns = &.{
184562 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184563 },
184564 .extra_temps = .{
184565 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184566 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184567 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184568 .unused,
184569 .unused,
184570 .unused,
184571 .unused,
184572 .unused,
184573 .unused,
184574 .unused,
184575 .unused,
184576 },
184577 .each = .{ .once = &.{
184578 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184579 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184580 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184581 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184582 .{ ._, .p_, .unpcklqdq, .src0x, .src0x, ._, ._ },
184583 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184584 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184585 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184586 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184587 } },
184588 }, .{
184589 .required_features = .{ .sse2, null, null, null },
184590 .src_constraints = .{ .{ .exact_bool_vec = 8 }, .{ .exact_bool_vec = 8 }, .any },
184591 .patterns = &.{
184592 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184593 },
184594 .extra_temps = .{
184595 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward, .smear = 2 } } },
184596 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184597 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184598 .unused,
184599 .unused,
184600 .unused,
184601 .unused,
184602 .unused,
184603 .unused,
184604 .unused,
184605 .unused,
184606 },
184607 .each = .{ .once = &.{
184608 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184609 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184610 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184611 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184612 .{ ._, .p_, .unpcklqdq, .tmp2x, .tmp2x, ._, ._ },
184613 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184614 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184615 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184616 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184617 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184618 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184619 } },
184620 }, .{
184621 .required_features = .{ .avx2, null, null, null },
184622 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184623 .patterns = &.{
184624 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184625 },
184626 .extra_temps = .{
184627 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184628 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184629 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184630 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184631 .unused,
184632 .unused,
184633 .unused,
184634 .unused,
184635 .unused,
184636 .unused,
184637 .unused,
184638 },
184639 .each = .{ .once = &.{
184640 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184641 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184642 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184643 .{ ._, .vp_q, .broadcast, .tmp3x, .lea(.tmp2q), ._, ._ },
184644 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184645 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184646 } },
184647 }, .{
184648 .required_features = .{ .avx2, null, null, null },
184649 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184650 .patterns = &.{
184651 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184652 },
184653 .extra_temps = .{
184654 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184655 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184656 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184657 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184658 .unused,
184659 .unused,
184660 .unused,
184661 .unused,
184662 .unused,
184663 .unused,
184664 .unused,
184665 },
184666 .each = .{ .once = &.{
184667 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184668 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184669 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184670 .{ ._, .vp_q, .broadcast, .tmp3x, .lea(.tmp2q), ._, ._ },
184671 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184672 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184673 .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ },
184674 .{ ._, .vp_b, .sub, .src0x, .tmp3x, .src0x, ._ },
184675 } },
184676 }, .{
184677 .required_features = .{ .avx, null, null, null },
184678 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184679 .patterns = &.{
184680 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184681 },
184682 .extra_temps = .{
184683 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184684 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184685 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184686 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184687 .unused,
184688 .unused,
184689 .unused,
184690 .unused,
184691 .unused,
184692 .unused,
184693 .unused,
184694 },
184695 .each = .{ .once = &.{
184696 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184697 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184698 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184699 .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184700 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184701 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184702 } },
184703 }, .{
184704 .required_features = .{ .avx, null, null, null },
184705 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184706 .patterns = &.{
184707 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184708 },
184709 .extra_temps = .{
184710 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184711 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184712 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184713 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
184714 .unused,
184715 .unused,
184716 .unused,
184717 .unused,
184718 .unused,
184719 .unused,
184720 .unused,
184721 },
184722 .each = .{ .once = &.{
184723 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184724 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lea(.tmp2x), ._ },
184725 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184726 .{ ._, .v_, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184727 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp3x, ._ },
184728 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp3x, ._ },
184729 .{ ._, .vp_, .xor, .tmp3x, .tmp3x, .tmp3x, ._ },
184730 .{ ._, .vp_b, .sub, .src0x, .tmp3x, .src0x, ._ },
184731 } },
184732 }, .{
184733 .required_features = .{ .ssse3, null, null, null },
184734 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184735 .patterns = &.{
184736 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184737 },
184738 .extra_temps = .{
184739 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184740 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184741 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184742 .{ .type = .usize, .kind = .{ .rc = .sse } },
184743 .unused,
184744 .unused,
184745 .unused,
184746 .unused,
184747 .unused,
184748 .unused,
184749 .unused,
184750 },
184751 .each = .{ .once = &.{
184752 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184753 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184754 .{ ._, .p_b, .shuf, .src0x, .lea(.tmp2x), ._, ._ },
184755 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184756 .{ ._, ._, .movddup, .tmp3x, .lea(.tmp2q), ._, ._ },
184757 .{ ._, .p_, .@"and", .src0x, .tmp3x, ._, ._ },
184758 .{ ._, .p_b, .cmpeq, .src0x, .tmp3x, ._, ._ },
184759 } },
184760 }, .{
184761 .required_features = .{ .ssse3, null, null, null },
184762 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184763 .patterns = &.{
184764 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_mut_sse, .none } },
184765 },
184766 .extra_temps = .{
184767 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .word, .smear = 8 } } },
184768 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184769 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184770 .unused,
184771 .unused,
184772 .unused,
184773 .unused,
184774 .unused,
184775 .unused,
184776 .unused,
184777 .unused,
184778 },
184779 .each = .{ .once = &.{
184780 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184781 .{ ._, .p_b, .shuf, .src1x, .lea(.tmp2x), ._, ._ },
184782 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184783 .{ ._, ._, .movddup, .src0x, .lea(.tmp2q), ._, ._ },
184784 .{ ._, .p_, .@"and", .src1x, .src0x, ._, ._ },
184785 .{ ._, .p_b, .cmpeq, .src1x, .src0x, ._, ._ },
184786 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184787 .{ ._, .p_b, .sub, .src0x, .src1x, ._, ._ },
184788 } },
184789 }, .{
184790 .required_features = .{ .sse3, null, null, null },
184791 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184792 .patterns = &.{
184793 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184794 },
184795 .extra_temps = .{
184796 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184797 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184798 .{ .type = .usize, .kind = .{ .rc = .sse } },
184799 .unused,
184800 .unused,
184801 .unused,
184802 .unused,
184803 .unused,
184804 .unused,
184805 .unused,
184806 .unused,
184807 },
184808 .each = .{ .once = &.{
184809 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184810 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184811 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184812 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184813 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184814 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp1q), ._, ._ },
184815 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184816 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184817 } },
184818 }, .{
184819 .required_features = .{ .sse3, null, null, null },
184820 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184821 .patterns = &.{
184822 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184823 },
184824 .extra_temps = .{
184825 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184826 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184827 .{ .type = .usize, .kind = .{ .rc = .sse } },
184828 .unused,
184829 .unused,
184830 .unused,
184831 .unused,
184832 .unused,
184833 .unused,
184834 .unused,
184835 .unused,
184836 },
184837 .each = .{ .once = &.{
184838 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184839 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184840 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184841 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184842 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184843 .{ ._, ._, .movddup, .src0x, .lea(.tmp1q), ._, ._ },
184844 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184845 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184846 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184847 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184848 } },
184849 }, .{
184850 .required_features = .{ .sse2, null, null, null },
184851 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184852 .patterns = &.{
184853 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184854 },
184855 .extra_temps = .{
184856 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184857 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184858 .{ .type = .usize, .kind = .{ .rc = .sse } },
184859 .unused,
184860 .unused,
184861 .unused,
184862 .unused,
184863 .unused,
184864 .unused,
184865 .unused,
184866 .unused,
184867 },
184868 .each = .{ .once = &.{
184869 .{ ._, ._dqa, .mov, .src0x, .src1x, ._, ._ },
184870 .{ ._, .p_, .unpcklbw, .src0x, .src0x, ._, ._ },
184871 .{ ._, .p_, .unpcklwd, .src0x, .src0x, ._, ._ },
184872 .{ ._, .p_, .unpckldq, .src0x, .src0x, ._, ._ },
184873 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184874 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp1x), ._, ._ },
184875 .{ ._, .p_, .@"and", .src0x, .tmp2x, ._, ._ },
184876 .{ ._, .p_b, .cmpeq, .src0x, .tmp2x, ._, ._ },
184877 } },
184878 }, .{
184879 .required_features = .{ .sse2, null, null, null },
184880 .src_constraints = .{ .{ .exact_bool_vec = 16 }, .{ .exact_bool_vec = 16 }, .any },
184881 .patterns = &.{
184882 .{ .src = .{ .{ .reg_mask = .{ .size = .xword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
184883 },
184884 .extra_temps = .{
184885 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184886 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184887 .{ .type = .usize, .kind = .{ .rc = .sse } },
184888 .unused,
184889 .unused,
184890 .unused,
184891 .unused,
184892 .unused,
184893 .unused,
184894 .unused,
184895 .unused,
184896 },
184897 .each = .{ .once = &.{
184898 .{ ._, ._dqa, .mov, .tmp2x, .src1x, ._, ._ },
184899 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp2x, ._, ._ },
184900 .{ ._, .p_, .unpcklwd, .tmp2x, .tmp2x, ._, ._ },
184901 .{ ._, .p_, .unpckldq, .tmp2x, .tmp2x, ._, ._ },
184902 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
184903 .{ ._, ._dqa, .mov, .src0x, .lea(.tmp1x), ._, ._ },
184904 .{ ._, .p_, .@"and", .tmp2x, .src0x, ._, ._ },
184905 .{ ._, .p_b, .cmpeq, .tmp2x, .src0x, ._, ._ },
184906 .{ ._, .p_, .xor, .src0x, .src0x, ._, ._ },
184907 .{ ._, .p_b, .sub, .src0x, .tmp2x, ._, ._ },
184908 } },
184909 }, .{
184910 .required_features = .{ .avx2, null, null, null },
184911 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184912 .patterns = &.{
184913 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .mem, .none } },
184914 },
184915 .extra_temps = .{
184916 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184917 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184918 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184919 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184920 .unused,
184921 .unused,
184922 .unused,
184923 .unused,
184924 .unused,
184925 .unused,
184926 .unused,
184927 },
184928 .each = .{ .once = &.{
184929 .{ ._, .vp_d, .broadcast, .src0y, .src1d, ._, ._ },
184930 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184931 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184932 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184933 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184934 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184935 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184936 } },
184937 }, .{
184938 .required_features = .{ .avx2, null, null, null },
184939 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184940 .patterns = &.{
184941 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
184942 },
184943 .extra_temps = .{
184944 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184945 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184946 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184947 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184948 .unused,
184949 .unused,
184950 .unused,
184951 .unused,
184952 .unused,
184953 .unused,
184954 .unused,
184955 },
184956 .each = .{ .once = &.{
184957 .{ ._, .vp_d, .broadcast, .src0y, .src1x, ._, ._ },
184958 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184959 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184960 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184961 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184962 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184963 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184964 } },
184965 }, .{
184966 .required_features = .{ .avx2, null, null, null },
184967 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184968 .patterns = &.{
184969 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .mem, .none } },
184970 },
184971 .extra_temps = .{
184972 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
184973 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
184974 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
184975 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
184976 .unused,
184977 .unused,
184978 .unused,
184979 .unused,
184980 .unused,
184981 .unused,
184982 .unused,
184983 },
184984 .each = .{ .once = &.{
184985 .{ ._, .vp_d, .broadcast, .src0y, .src1d, ._, ._ },
184986 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
184987 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
184988 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
184989 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
184990 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
184991 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
184992 .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp3y, ._ },
184993 .{ ._, .vp_b, .sub, .src0y, .tmp3y, .src0y, ._ },
184994 } },
184995 }, .{
184996 .required_features = .{ .avx2, null, null, null },
184997 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
184998 .patterns = &.{
184999 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
185000 },
185001 .extra_temps = .{
185002 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185003 .{ .type = .vector_8_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185004 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185005 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185006 .unused,
185007 .unused,
185008 .unused,
185009 .unused,
185010 .unused,
185011 .unused,
185012 .unused,
185013 },
185014 .each = .{ .once = &.{
185015 .{ ._, .vp_d, .broadcast, .src0y, .src1x, ._, ._ },
185016 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185017 .{ ._, .vp_b, .shuf, .src0y, .src0y, .lea(.tmp2y), ._ },
185018 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185019 .{ ._, .vp_q, .broadcast, .tmp3y, .lea(.tmp2q), ._, ._ },
185020 .{ ._, .vp_, .@"and", .src0y, .src0y, .tmp3y, ._ },
185021 .{ ._, .vp_b, .cmpeq, .src0y, .src0y, .tmp3y, ._ },
185022 .{ ._, .vp_, .xor, .tmp3y, .tmp3y, .tmp3y, ._ },
185023 .{ ._, .vp_b, .sub, .src0y, .tmp3y, .src0y, ._ },
185024 } },
185025 }, .{
185026 .required_features = .{ .avx, null, null, null },
185027 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
185028 .patterns = &.{
185029 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_sign_extend, .is = .uninverted } }, .to_sse, .none } },
185030 },
185031 .extra_temps = .{
185032 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185033 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185034 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185035 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185036 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185037 .unused,
185038 .unused,
185039 .unused,
185040 .unused,
185041 .unused,
185042 .unused,
185043 },
185044 .each = .{ .once = &.{
185045 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185046 .{ ._, .vp_b, .shuf, .tmp3x, .src1x, .lea(.tmp2y), ._ },
185047 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lead(.tmp2y, 16), ._ },
185048 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185049 .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ },
185050 .{ ._, .vp_, .@"and", .tmp3x, .tmp3x, .tmp4x, ._ },
185051 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp4x, ._ },
185052 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .tmp4x, ._ },
185053 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp4x, ._ },
185054 .{ ._, .v_f128, .insert, .src0y, .tmp3y, .src0x, .ui(1) },
185055 } },
185056 }, .{
185057 .required_features = .{ .avx, null, null, null },
185058 .src_constraints = .{ .{ .exact_bool_vec = 32 }, .{ .exact_bool_vec = 32 }, .any },
185059 .patterns = &.{
185060 .{ .src = .{ .{ .reg_mask = .{ .size = .yword, .kind = .from_zero_extend, .is = .uninverted } }, .to_sse, .none } },
185061 },
185062 .extra_temps = .{
185063 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bytes_mem = .{ .direction = .forward, .size = .dword, .smear = 8 } } },
185064 .{ .type = .vector_16_u8, .kind = .{ .bits_mem = .{ .direction = .forward } } },
185065 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
185066 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185067 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
185068 .unused,
185069 .unused,
185070 .unused,
185071 .unused,
185072 .unused,
185073 .unused,
185074 },
185075 .each = .{ .once = &.{
185076 .{ ._, ._, .lea, .tmp2p, .mem(.tmp0), ._, ._ },
185077 .{ ._, .vp_b, .shuf, .tmp3x, .src1x, .lea(.tmp2y), ._ },
185078 .{ ._, .vp_b, .shuf, .src0x, .src1x, .lead(.tmp2y, 16), ._ },
185079 .{ ._, ._, .lea, .tmp2p, .mem(.tmp1), ._, ._ },
185080 .{ ._, .v_dqa, .mov, .tmp4x, .lea(.tmp2x), ._, ._ },
185081 .{ ._, .vp_, .@"and", .tmp3x, .tmp3x, .tmp4x, ._ },
185082 .{ ._, .vp_, .@"and", .src0x, .src0x, .tmp4x, ._ },
185083 .{ ._, .vp_b, .cmpeq, .tmp3x, .tmp3x, .tmp4x, ._ },
185084 .{ ._, .vp_b, .cmpeq, .src0x, .src0x, .tmp4x, ._ },
185085 .{ ._, .vp_, .xor, .tmp4x, .tmp4x, .tmp4x, ._ },
185086 .{ ._, .vp_b, .sub, .tmp3x, .tmp4x, .tmp3x, ._ },
185087 .{ ._, .vp_b, .sub, .src0x, .tmp4x, .src0x, ._ },
185088 .{ ._, .v_f128, .insert, .src0y, .tmp3y, .src0x, .ui(1) },
185089 } },
185090 } });
185091 dst.*, src.* = ops;
185092 }
185093
183505 fn wrapInt(temp: *Temp, cg: *CodeGen) Select.Error!void {185094 fn wrapInt(temp: *Temp, cg: *CodeGen) Select.Error!void {
183506 var ops: [1]Temp = .{temp.*};185095 var ops: [1]Temp = .{temp.*};
183507 var res: [1]Temp = undefined;185096 var res: [1]Temp = undefined;
...@@ -188030,6 +189619,7 @@ const Temp = struct {...@@ -188030,6 +189619,7 @@ const Temp = struct {
188030 .unreach,189619 .unreach,
188031 .dead,189620 .dead,
188032 .indirect_load_frame,189621 .indirect_load_frame,
189622 .indirect_mask,
188033 .register_tee,189623 .register_tee,
188034 .elementwise_gpr,189624 .elementwise_gpr,
188035 .elementwise_sse,189625 .elementwise_sse,
...@@ -188885,25 +190475,60 @@ const Select = struct {...@@ -188885,25 +190475,60 @@ const Select = struct {
188885 to_sse,190475 to_sse,
188886 mut_sse,190476 mut_sse,
188887 to_mut_sse,190477 to_mut_sse,
188888 reg_mask: RegMaskSpec,190478 reg_mask: struct {
188889 all_reg_mask: RegMaskSpec,
188890
188891 const RegMaskSpec = struct {
188892 size: Memory.Size,190479 size: Memory.Size,
190480 kind: enum {
190481 any,
190482 lsb,
190483 msb,
190484 zero_extend,
190485 sign_extend,
190486 from_lsb,
190487 from_msb,
190488 from_zero_extend,
190489 from_sign_extend,
190490 },
188893 is: enum {190491 is: enum {
188894 any,190492 any,
188895 uninverted,190493 uninverted,
188896 inverted,190494 inverted,
188897
188898 fn matches(is: @This(), inverted: bool) bool {
188899 return switch (is) {
188900 .any => true,
188901 .uninverted => !inverted,
188902 .inverted => inverted,
188903 };
188904 }
188905 } = .any,190495 } = .any,
188906 };190496
190497 fn matches(spec: @This(), info: MaskInfo) bool {
190498 return switch (spec.kind) {
190499 .any => true,
190500 .lsb => switch (info.kind) {
190501 .lsb, .msb, .zero_extend, .sign_extend => true,
190502 },
190503 .msb => switch (info.kind) {
190504 .lsb, .zero_extend => false,
190505 .msb, .sign_extend => true,
190506 },
190507 .zero_extend => info.kind == .zero_extend,
190508 .sign_extend => info.kind == .sign_extend,
190509 .from_lsb => switch (info.kind) {
190510 .lsb => true,
190511 .msb, .zero_extend, .sign_extend => false,
190512 },
190513 .from_msb => switch (info.kind) {
190514 .lsb, .msb => true,
190515 .zero_extend, .sign_extend => false,
190516 },
190517 .from_sign_extend => switch (info.kind) {
190518 .lsb, .msb, .sign_extend => true,
190519 .zero_extend => false,
190520 },
190521 .from_zero_extend => switch (info.kind) {
190522 .lsb, .zero_extend => true,
190523 .msb, .sign_extend => false,
190524 },
190525 } and switch (spec.is) {
190526 .any => true,
190527 .uninverted => !info.inverted,
190528 .inverted => info.inverted,
190529 };
190530 }
190531 },
188907190532
188908 fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {190533 fn matches(src: Src, temp: Temp, cg: *CodeGen) bool {
188909 return switch (src) {190534 return switch (src) {
...@@ -188988,13 +190613,7 @@ const Select = struct {...@@ -188988,13 +190613,7 @@ const Select = struct {
188988 .reg_mask => |mask_spec| switch (temp.tracking(cg).short) {190613 .reg_mask => |mask_spec| switch (temp.tracking(cg).short) {
188989 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >=190614 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) >=
188990 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and190615 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and
188991 mask_spec.is.matches(reg_mask.info.inverted),190616 mask_spec.matches(reg_mask.info),
188992 else => false,
188993 },
188994 .all_reg_mask => |mask_spec| switch (temp.tracking(cg).short) {
188995 .register_mask => |reg_mask| mask_spec.size.bitSize(cg.target) ==
188996 reg_mask.info.scalar.bitSize(cg.target) * temp.typeOf(cg).vectorLen(cg.pt.zcu) and
188997 reg_mask.info.kind == .all and mask_spec.is.matches(reg_mask.info.inverted),
188998 else => false,190617 else => false,
188999 },190618 },
189000 };190619 };
...@@ -189002,7 +190621,7 @@ const Select = struct {...@@ -189002,7 +190621,7 @@ const Select = struct {
189002190621
189003 fn convert(src: Src, temp: *Temp, cg: *CodeGen) InnerError!bool {190622 fn convert(src: Src, temp: *Temp, cg: *CodeGen) InnerError!bool {
189004 return switch (src) {190623 return switch (src) {
189005 .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask, .all_reg_mask => false,190624 .none, .any, .imm, .imm8, .imm16, .imm32, .simm32, .reg_mask => false,
189006 .mem, .to_mem => try temp.toBase(false, cg),190625 .mem, .to_mem => try temp.toBase(false, cg),
189007 .mut_mem, .to_mut_mem => try temp.toBase(true, cg),190626 .mut_mem, .to_mut_mem => try temp.toBase(true, cg),
189008 .to_reg => |reg| try temp.toReg(reg, cg),190627 .to_reg => |reg| try temp.toReg(reg, cg),
...@@ -189073,8 +190692,8 @@ const Select = struct {...@@ -189073,8 +190692,8 @@ const Select = struct {
189073 pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size },190692 pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size },
189074 pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false },190693 pand_mask_mem: struct { ref: Select.Operand.Ref, invert: bool = false },
189075 ptest_mask_mem: Select.Operand.Ref,190694 ptest_mask_mem: Select.Operand.Ref,
189076 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },190695 pshufb_bytes_mem: struct { direction: Direction, repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
189077 bits_mem: enum { forward, reverse },190696 bits_mem: struct { direction: Direction, smear: u4 = 1 },
189078 splat_int_mem: struct { ref: Select.Operand.Ref, inside: enum { umin, smin, smax } = .umin, outside: enum { smin, smax } },190697 splat_int_mem: struct { ref: Select.Operand.Ref, inside: enum { umin, smin, smax } = .umin, outside: enum { smin, smax } },
189079 splat_float_mem: struct { ref: Select.Operand.Ref, inside: enum { zero } = .zero, outside: f16 },190698 splat_float_mem: struct { ref: Select.Operand.Ref, inside: enum { zero } = .zero, outside: f16 },
189080 frame: FrameIndex,190699 frame: FrameIndex,
...@@ -189109,6 +190728,8 @@ const Select = struct {...@@ -189109,6 +190728,8 @@ const Select = struct {
189109 }190728 }
189110 };190729 };
189111190730
190731 const Direction = enum { forward, reverse };
190732
189112 fn lock(kind: Kind, cg: *CodeGen) ![2]?RegisterLock {190733 fn lock(kind: Kind, cg: *CodeGen) ![2]?RegisterLock {
189113 var reg_locks: [2]?RegisterLock = @splat(null);190734 var reg_locks: [2]?RegisterLock = @splat(null);
189114 const regs: [2]Register = switch (kind) {190735 const regs: [2]Register = switch (kind) {
...@@ -189430,11 +191051,15 @@ const Select = struct {...@@ -189430,11 +191051,15 @@ const Select = struct {
189430 var index: u7 = 0;191051 var index: u7 = 0;
189431 for (0..@intCast(ref_ty.vectorLen(zcu))) |_| {191052 for (0..@intCast(ref_ty.vectorLen(zcu))) |_| {
189432 switch (mask_info.kind) {191053 switch (mask_info.kind) {
189433 .sign => {191054 .lsb, .zero_extend => {
191055 elems[index] = 1;
191056 @memset(elems[index + 1 ..][0 .. elem_bytes - 1], std.math.minInt(u8));
191057 },
191058 .msb => {
189434 @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8));191059 @memset(elems[index..][0 .. elem_bytes - 1], std.math.minInt(u8));
189435 elems[index + elem_bytes - 1] = @bitCast(@as(i8, std.math.minInt(i8)));191060 elems[index + elem_bytes - 1] = @bitCast(@as(i8, std.math.minInt(i8)));
189436 },191061 },
189437 .all => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)),191062 .sign_extend => @memset(elems[index..][0..elem_bytes], std.math.maxInt(u8)),
189438 }191063 }
189439 index += elem_bytes;191064 index += elem_bytes;
189440 }191065 }
...@@ -189443,31 +191068,40 @@ const Select = struct {...@@ -189443,31 +191068,40 @@ const Select = struct {
189443 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },191068 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
189444 } }))), true };191069 } }))), true };
189445 },191070 },
189446 .pshufb_bswap_mem => |bswap_spec| {191071 .pshufb_bytes_mem => |bytes_spec| {
189447 const zcu = pt.zcu;191072 const zcu = pt.zcu;
189448 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);191073 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
189449 var elem_buf: [32]u8 = @splat(1 << 7);191074 var elem_buf: [32]u8 = @splat(1 << 7);
189450 const elems = elem_buf[0..spec.type.vectorLen(zcu)];191075 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
189451 const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8));191076 const len: usize = @intCast(@divExact(bytes_spec.size.bitSize(cg.target), 8));
189452 var to_index: u6 = 0;191077 var to_index: u6 = 0;
189453 for (0..bswap_spec.repeat) |_| for (0..len) |from_index| {191078 for (0..bytes_spec.repeat) |_| for (0..len) |from_index| {
189454 @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));191079 @memset(elems[to_index..][0..bytes_spec.smear], @intCast(switch (bytes_spec.direction) {
189455 to_index += bswap_spec.smear;191080 .forward => from_index,
191081 .reverse => len - 1 - from_index,
191082 }));
191083 to_index += bytes_spec.smear;
189456 };191084 };
189457 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{191085 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
189458 .ty = spec.type.toIntern(),191086 .ty = spec.type.toIntern(),
189459 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },191087 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
189460 } }))), true };191088 } }))), true };
189461 },191089 },
189462 .bits_mem => |direction| {191090 .bits_mem => |bits_spec| {
189463 const zcu = pt.zcu;191091 const zcu = pt.zcu;
189464 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);191092 assert(spec.type.isVector(zcu) and spec.type.childType(zcu).toIntern() == .u8_type);
189465 var bytes: [32]u8 = undefined;191093 var elem_buf: [32]u8 = @splat(1 << 7);
189466 const elems = bytes[0..spec.type.vectorLen(zcu)];191094 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
189467 for (elems, 0..) |*elem, index| elem.* = switch (direction) {191095 var from_index: u6 = 0;
189468 .forward => @as(u8, 1 << 0) << @truncate(index),191096 var to_index: u6 = 0;
189469 .reverse => @as(u8, 1 << 7) >> @truncate(index),191097 while (elems.len - to_index > 0) {
189470 };191098 @memset(elems[to_index..][0..bits_spec.smear], @intCast(switch (bits_spec.direction) {
191099 .forward => @as(u8, 1 << 0) << @truncate(from_index),
191100 .reverse => @as(u8, 1 << 7) >> @truncate(from_index),
191101 }));
191102 from_index += 1;
191103 to_index += bits_spec.smear;
191104 }
189471 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{191105 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
189472 .ty = spec.type.toIntern(),191106 .ty = spec.type.toIntern(),
189473 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },191107 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, io, pt.tid, elems, .maybe_embedded_nulls) },
src/codegen/x86_64/abi.zig+3-1
...@@ -24,6 +24,8 @@ pub const Class = enum {...@@ -24,6 +24,8 @@ pub const Class = enum {
24 float,24 float,
25 /// A `Class.sse` containing two `f32`s.25 /// A `Class.sse` containing two `f32`s.
26 float_combine,26 float_combine,
27 /// Clang uses different element sizes depending on the vector length.
28 bool_vector_mask,
27 /// Clang passes each vector element in a separate `Class.integer`.29 /// Clang passes each vector element in a separate `Class.integer`.
28 integer_per_element,30 integer_per_element,
29 /// Clang passes each vector element in a separate `Class.sse`.31 /// Clang passes each vector element in a separate `Class.sse`.
...@@ -150,7 +152,7 @@ pub fn classifyWindows(init_ty: Type, zcu: *Zcu, target: *const std.Target, ctx:...@@ -150,7 +152,7 @@ pub fn classifyWindows(init_ty: Type, zcu: *Zcu, target: *const std.Target, ctx:
150 .{ 16, .sse_per_xword };152 .{ 16, .sse_per_xword };
151 if (elem_ty.toIntern() == .bool_type) {153 if (elem_ty.toIntern() == .bool_type) {
152 if (len > reg_size) return if (ctx == .arg) .integer_per_element else .memory;154 if (len > reg_size) return if (ctx == .arg) .integer_per_element else .memory;
153 return if (ctx == .arg) .memory else .sse;155 return .bool_vector_mask;
154 }156 }
155 const elem_size = elem_ty.abiSize(zcu);157 const elem_size = elem_ty.abiSize(zcu);
156 const unaligned_size = elem_size * len;158 const unaligned_size = elem_size * len;
src/dev.zig+14
...@@ -52,6 +52,10 @@ pub const Env = enum {...@@ -52,6 +52,10 @@ pub const Env = enum {
52 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-linux --listen=-`52 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-linux --listen=-`
53 @"x86_64-linux",53 @"x86_64-linux",
5454
55 /// - sema
56 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-windows --listen=-`
57 @"x86_64-windows",
58
55 pub inline fn supports(comptime dev_env: Env, comptime feature: Feature) bool {59 pub inline fn supports(comptime dev_env: Env, comptime feature: Feature) bool {
56 return switch (dev_env) {60 return switch (dev_env) {
57 .full => true,61 .full => true,
...@@ -216,6 +220,16 @@ pub const Env = enum {...@@ -216,6 +220,16 @@ pub const Env = enum {
216 => true,220 => true,
217 else => Env.sema.supports(feature),221 else => Env.sema.supports(feature),
218 },222 },
223 .@"x86_64-windows" => switch (feature) {
224 .build_command,
225 .stdio_listen,
226 .incremental,
227 .legalize,
228 .x86_64_backend,
229 .coff2_linker,
230 => true,
231 else => Env.sema.supports(feature),
232 },
219 };233 };
220 }234 }
221235
test/c_abi/main.zig-5
...@@ -295,7 +295,6 @@ extern fn c_test_vector_2_bool() void;...@@ -295,7 +295,6 @@ extern fn c_test_vector_2_bool() void;
295295
296test "@Vector(2, bool)" {296test "@Vector(2, bool)" {
297 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;297 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
298 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
299298
300 const vec = c_ret_vector_2_bool();299 const vec = c_ret_vector_2_bool();
301 try expect(vec[0] == true);300 try expect(vec[0] == true);
...@@ -337,7 +336,6 @@ extern fn c_test_vector_4_bool() void;...@@ -337,7 +336,6 @@ extern fn c_test_vector_4_bool() void;
337336
338test "@Vector(4, bool)" {337test "@Vector(4, bool)" {
339 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;338 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
340 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
341339
342 const vec = c_ret_vector_4_bool();340 const vec = c_ret_vector_4_bool();
343 try expect(vec[0] == true);341 try expect(vec[0] == true);
...@@ -391,7 +389,6 @@ extern fn c_test_vector_8_bool() void;...@@ -391,7 +389,6 @@ extern fn c_test_vector_8_bool() void;
391389
392test "@Vector(8, bool)" {390test "@Vector(8, bool)" {
393 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;391 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
394 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
395392
396 const vec = c_ret_vector_8_bool();393 const vec = c_ret_vector_8_bool();
397 try expect(vec[0] == false);394 try expect(vec[0] == false);
...@@ -469,7 +466,6 @@ extern fn c_test_vector_16_bool() void;...@@ -469,7 +466,6 @@ extern fn c_test_vector_16_bool() void;
469466
470test "@Vector(16, bool)" {467test "@Vector(16, bool)" {
471 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;468 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
472 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows) return error.SkipZigTest;
473469
474 const vec = c_ret_vector_16_bool();470 const vec = c_ret_vector_16_bool();
475 try expect(vec[0] == true);471 try expect(vec[0] == true);
...@@ -595,7 +591,6 @@ extern fn c_test_vector_32_bool() void;...@@ -595,7 +591,6 @@ extern fn c_test_vector_32_bool() void;
595591
596test "@Vector(32, bool)" {592test "@Vector(32, bool)" {
597 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;593 if (builtin.zig_backend == .stage2_llvm and (builtin.cpu.arch != .powerpc and builtin.cpu.arch != .wasm32)) return error.SkipZigTest;
598 if (builtin.zig_backend == .stage2_x86_64 and builtin.os.tag == .windows and builtin.cpu.has(.x86, .avx)) return error.SkipZigTest;
599594
600 const vec = c_ret_vector_32_bool();595 const vec = c_ret_vector_32_bool();
601 try expect(vec[0] == true);596 try expect(vec[0] == true);