authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-11 18:24:37-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-21 21:51:08-04:00
logc5c1c8538dc047bc7df9acd685791c4ba3e20ffb
treeca21b8d0f19c2b3e2a5d73d1659b495e2e2a6b4c
parented284c1f98ae08a36a7ef0776e22a4cfb2527fee

x86_64: rewrite wrapping multiplication

14 files changed, 3197 insertions(+), 374 deletions(-)

lib/std/zig/Zir.zig+5-1
......@@ -2136,7 +2136,7 @@ pub const Inst = struct {
21362136 ref_start_index = static_len,
21372137 _,
21382138
2139 pub const static_len = 93;
2139 pub const static_len = 97;
21402140
21412141 pub fn toRef(i: Index) Inst.Ref {
21422142 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));
......@@ -2221,6 +2221,10 @@ pub const Inst = struct {
22212221 slice_const_u8_sentinel_0_type,
22222222 vector_16_i8_type,
22232223 vector_32_i8_type,
2224 vector_1_u8_type,
2225 vector_2_u8_type,
2226 vector_4_u8_type,
2227 vector_8_u8_type,
22242228 vector_16_u8_type,
22252229 vector_32_u8_type,
22262230 vector_8_i16_type,
src/Air.zig+4
......@@ -985,6 +985,10 @@ pub const Inst = struct {
985985 slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type),
986986 vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type),
987987 vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type),
988 vector_1_u8_type = @intFromEnum(InternPool.Index.vector_1_u8_type),
989 vector_2_u8_type = @intFromEnum(InternPool.Index.vector_2_u8_type),
990 vector_4_u8_type = @intFromEnum(InternPool.Index.vector_4_u8_type),
991 vector_8_u8_type = @intFromEnum(InternPool.Index.vector_8_u8_type),
988992 vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type),
989993 vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type),
990994 vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type),
src/InternPool.zig+20
......@@ -4573,6 +4573,10 @@ pub const Index = enum(u32) {
45734573
45744574 vector_16_i8_type,
45754575 vector_32_i8_type,
4576 vector_1_u8_type,
4577 vector_2_u8_type,
4578 vector_4_u8_type,
4579 vector_8_u8_type,
45764580 vector_16_u8_type,
45774581 vector_32_u8_type,
45784582 vector_8_i16_type,
......@@ -5089,6 +5093,14 @@ pub const static_keys = [_]Key{
50895093 .{ .vector_type = .{ .len = 16, .child = .i8_type } },
50905094 // @Vector(32, i8)
50915095 .{ .vector_type = .{ .len = 32, .child = .i8_type } },
5096 // @Vector(1, u8)
5097 .{ .vector_type = .{ .len = 1, .child = .u8_type } },
5098 // @Vector(2, u8)
5099 .{ .vector_type = .{ .len = 2, .child = .u8_type } },
5100 // @Vector(4, u8)
5101 .{ .vector_type = .{ .len = 4, .child = .u8_type } },
5102 // @Vector(8, u8)
5103 .{ .vector_type = .{ .len = 8, .child = .u8_type } },
50925104 // @Vector(16, u8)
50935105 .{ .vector_type = .{ .len = 16, .child = .u8_type } },
50945106 // @Vector(32, u8)
......@@ -11766,6 +11778,10 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
1176611778 .slice_const_u8_sentinel_0_type,
1176711779 .vector_16_i8_type,
1176811780 .vector_32_i8_type,
11781 .vector_1_u8_type,
11782 .vector_2_u8_type,
11783 .vector_4_u8_type,
11784 .vector_8_u8_type,
1176911785 .vector_16_u8_type,
1177011786 .vector_32_u8_type,
1177111787 .vector_8_i16_type,
......@@ -12106,6 +12122,10 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
1210612122
1210712123 .vector_16_i8_type,
1210812124 .vector_32_i8_type,
12125 .vector_1_u8_type,
12126 .vector_2_u8_type,
12127 .vector_4_u8_type,
12128 .vector_8_u8_type,
1210912129 .vector_16_u8_type,
1211012130 .vector_32_u8_type,
1211112131 .vector_8_i16_type,
src/Sema.zig+4
......@@ -36377,6 +36377,10 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
3637736377 .slice_const_u8_sentinel_0_type,
3637836378 .vector_16_i8_type,
3637936379 .vector_32_i8_type,
36380 .vector_1_u8_type,
36381 .vector_2_u8_type,
36382 .vector_4_u8_type,
36383 .vector_8_u8_type,
3638036384 .vector_16_u8_type,
3638136385 .vector_32_u8_type,
3638236386 .vector_8_i16_type,
src/Type.zig+4
......@@ -4201,6 +4201,10 @@ pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_senti
42014201
42024202pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };
42034203pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type };
4204pub const vector_1_u8: Type = .{ .ip_index = .vector_1_u8_type };
4205pub const vector_2_u8: Type = .{ .ip_index = .vector_2_u8_type };
4206pub const vector_4_u8: Type = .{ .ip_index = .vector_4_u8_type };
4207pub const vector_8_u8: Type = .{ .ip_index = .vector_8_u8_type };
42044208pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type };
42054209pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type };
42064210pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type };
src/arch/x86_64/CodeGen.zig+3082-364
......@@ -2069,7 +2069,7 @@ fn asmRegisterMemoryImmediate(
20692069 } else {
20702070 const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) {
20712071 .signed => |s| @bitCast(s),
2072 .unsigned => unreachable,
2072 .unsigned => |u| @as(u32, @intCast(u)),
20732073 .reloc => unreachable,
20742074 } });
20752075 assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m)));
......@@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24182418}
24192419
24202420fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2421 @setEvalBranchQuota(13_600);
2421 @setEvalBranchQuota(13_800);
24222422 const pt = cg.pt;
24232423 const zcu = pt.zcu;
24242424 const ip = &zcu.intern_pool;
......@@ -2457,9 +2457,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
24572457 .shr, .shr_exact => try cg.airShlShrBinOp(inst),
24582458 .shl, .shl_exact => try cg.airShlShrBinOp(inst),
24592459
2460 .mul_wrap,
2461 => |air_tag| try cg.airMulDivBinOp(inst, air_tag),
2462
24632460 .add_sat => try cg.airAddSat(inst),
24642461 .sub_sat => try cg.airSubSat(inst),
24652462 .mul_sat => try cg.airMulSat(inst),
......@@ -5215,13 +5212,2516 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52155212 .each = .{ .once = &.{
52165213 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
52175214 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
5218 .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
5215 .{ ._, ._, .sub, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
5216 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
5217 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5218 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5219 } },
5220 }, .{
5221 .required_features = .{ .@"64bit", null, null, null },
5222 .src_constraints = .{
5223 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5224 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
5225 .any,
5226 },
5227 .patterns = &.{
5228 .{ .src = .{ .to_mem, .to_mem, .none } },
5229 },
5230 .extra_temps = .{
5231 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5232 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5233 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5234 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5235 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5236 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5237 .unused,
5238 .unused,
5239 .unused,
5240 .unused,
5241 .unused,
5242 },
5243 .dst_temps = .{ .mem, .unused },
5244 .each = .{ .once = &.{
5245 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5246 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5247 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5248 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5249 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ },
5250 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5251 .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
5252 .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ },
5253 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ },
5254 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5255 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
5256 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5257 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5258 } },
5259 }, .{
5260 .src_constraints = .{
5261 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
5262 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
5263 .any,
5264 },
5265 .patterns = &.{
5266 .{ .src = .{ .to_mem, .to_mem, .none } },
5267 },
5268 .extra_temps = .{
5269 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5270 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5271 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5272 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
5273 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5274 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5275 .unused,
5276 .unused,
5277 .unused,
5278 .unused,
5279 .unused,
5280 },
5281 .dst_temps = .{ .mem, .unused },
5282 .each = .{ .once = &.{
5283 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5284 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5285 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5286 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5287 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ },
5288 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5289 .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ },
5290 .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ },
5291 .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ },
5292 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5293 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
5294 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
5295 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5296 } },
5297 }, .{
5298 .required_features = .{ .f16c, null, null, null },
5299 .src_constraints = .{
5300 .{ .scalar_float = .{ .of = .word, .is = .word } },
5301 .{ .scalar_float = .{ .of = .word, .is = .word } },
5302 .any,
5303 },
5304 .patterns = &.{
5305 .{ .src = .{ .to_sse, .to_sse, .none } },
5306 },
5307 .extra_temps = .{
5308 .{ .type = .f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5309 .unused,
5310 .unused,
5311 .unused,
5312 .unused,
5313 .unused,
5314 .unused,
5315 .unused,
5316 .unused,
5317 .unused,
5318 .unused,
5319 },
5320 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5321 .each = .{ .once = &.{
5322 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5323 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5324 .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ },
5325 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5326 } },
5327 }, .{
5328 .required_features = .{ .sse, null, null, null },
5329 .src_constraints = .{
5330 .{ .scalar_float = .{ .of = .word, .is = .word } },
5331 .{ .scalar_float = .{ .of = .word, .is = .word } },
5332 .any,
5333 },
5334 .patterns = &.{
5335 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 }, .none } },
5336 },
5337 .call_frame = .{ .alignment = .@"16" },
5338 .extra_temps = .{
5339 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5340 .unused,
5341 .unused,
5342 .unused,
5343 .unused,
5344 .unused,
5345 .unused,
5346 .unused,
5347 .unused,
5348 .unused,
5349 .unused,
5350 },
5351 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5352 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5353 .each = .{ .once = &.{
5354 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
5355 } },
5356 }, .{
5357 .required_features = .{ .f16c, null, null, null },
5358 .src_constraints = .{
5359 .{ .scalar_float = .{ .of = .qword, .is = .word } },
5360 .{ .scalar_float = .{ .of = .qword, .is = .word } },
5361 .any,
5362 },
5363 .patterns = &.{
5364 .{ .src = .{ .mem, .mem, .none } },
5365 .{ .src = .{ .to_sse, .mem, .none } },
5366 .{ .src = .{ .mem, .to_sse, .none } },
5367 .{ .src = .{ .to_sse, .to_sse, .none } },
5368 },
5369 .extra_temps = .{
5370 .{ .type = .vector_4_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5371 .unused,
5372 .unused,
5373 .unused,
5374 .unused,
5375 .unused,
5376 .unused,
5377 .unused,
5378 .unused,
5379 .unused,
5380 .unused,
5381 },
5382 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5383 .each = .{ .once = &.{
5384 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
5385 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5386 .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ },
5387 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
5388 } },
5389 }, .{
5390 .required_features = .{ .f16c, null, null, null },
5391 .src_constraints = .{
5392 .{ .scalar_float = .{ .of = .xword, .is = .word } },
5393 .{ .scalar_float = .{ .of = .xword, .is = .word } },
5394 .any,
5395 },
5396 .patterns = &.{
5397 .{ .src = .{ .mem, .mem, .none } },
5398 .{ .src = .{ .to_sse, .mem, .none } },
5399 .{ .src = .{ .mem, .to_sse, .none } },
5400 .{ .src = .{ .to_sse, .to_sse, .none } },
5401 },
5402 .extra_temps = .{
5403 .{ .type = .vector_8_f32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
5404 .unused,
5405 .unused,
5406 .unused,
5407 .unused,
5408 .unused,
5409 .unused,
5410 .unused,
5411 .unused,
5412 .unused,
5413 .unused,
5414 },
5415 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5416 .each = .{ .once = &.{
5417 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
5418 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },
5419 .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ },
5420 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },
5421 } },
5422 }, .{
5423 .required_features = .{ .f16c, null, null, null },
5424 .src_constraints = .{
5425 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
5426 .{ .multiple_scalar_float = .{ .of = .xword, .is = .word } },
5427 .any,
5428 },
5429 .patterns = &.{
5430 .{ .src = .{ .to_mem, .to_mem, .none } },
5431 },
5432 .extra_temps = .{
5433 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5434 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5435 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5436 .unused,
5437 .unused,
5438 .unused,
5439 .unused,
5440 .unused,
5441 .unused,
5442 .unused,
5443 .unused,
5444 },
5445 .dst_temps = .{ .mem, .unused },
5446 .clobbers = .{ .eflags = true },
5447 .each = .{ .once = &.{
5448 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5449 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5450 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5451 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ },
5452 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },
5453 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5454 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5455 } },
5456 }, .{
5457 .required_features = .{ .avx, null, null, null },
5458 .src_constraints = .{
5459 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5460 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5461 .any,
5462 },
5463 .patterns = &.{
5464 .{ .src = .{ .to_mem, .to_mem, .none } },
5465 },
5466 .call_frame = .{ .alignment = .@"16" },
5467 .extra_temps = .{
5468 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5469 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5470 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5471 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5472 .unused,
5473 .unused,
5474 .unused,
5475 .unused,
5476 .unused,
5477 .unused,
5478 .unused,
5479 },
5480 .dst_temps = .{ .mem, .unused },
5481 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5482 .each = .{ .once = &.{
5483 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5484 .{ .@"0:", .vp_, .xor, .tmp2x, .tmp2x, .tmp2x, ._ },
5485 .{ ._, .vp_w, .insr, .tmp1x, .tmp2x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0) },
5486 .{ ._, .vp_w, .insr, .tmp2x, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0) },
5487 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5488 .{ ._, .vp_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
5489 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5490 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5491 } },
5492 }, .{
5493 .required_features = .{ .sse4_1, null, null, null },
5494 .src_constraints = .{
5495 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5496 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5497 .any,
5498 },
5499 .patterns = &.{
5500 .{ .src = .{ .to_mem, .to_mem, .none } },
5501 },
5502 .call_frame = .{ .alignment = .@"16" },
5503 .extra_temps = .{
5504 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5505 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5506 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5507 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5508 .unused,
5509 .unused,
5510 .unused,
5511 .unused,
5512 .unused,
5513 .unused,
5514 .unused,
5515 },
5516 .dst_temps = .{ .mem, .unused },
5517 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5518 .each = .{ .once = &.{
5519 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5520 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5521 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
5522 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5523 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5524 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5525 .{ ._, .p_w, .extr, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1x, .ui(0), ._ },
5526 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5527 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5528 } },
5529 }, .{
5530 .required_features = .{ .sse2, null, null, null },
5531 .src_constraints = .{
5532 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5533 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5534 .any,
5535 },
5536 .patterns = &.{
5537 .{ .src = .{ .to_mem, .to_mem, .none } },
5538 },
5539 .call_frame = .{ .alignment = .@"16" },
5540 .extra_temps = .{
5541 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5542 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5543 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5544 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5545 .{ .type = .f16, .kind = .{ .reg = .ax } },
5546 .unused,
5547 .unused,
5548 .unused,
5549 .unused,
5550 .unused,
5551 .unused,
5552 },
5553 .dst_temps = .{ .mem, .unused },
5554 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5555 .each = .{ .once = &.{
5556 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5557 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
5558 .{ ._, .p_, .xor, .tmp2x, .tmp2x, ._, ._ },
5559 .{ ._, .p_w, .insr, .tmp1x, .memia(.src0w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5560 .{ ._, .p_w, .insr, .tmp2x, .memia(.src1w, .tmp0, .add_unaligned_size), .ui(0), ._ },
5561 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
5562 .{ ._, .p_w, .extr, .tmp4d, .tmp1x, .ui(0), ._ },
5563 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp4w, ._, ._ },
5564 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5565 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5566 } },
5567 }, .{
5568 .required_features = .{ .sse, null, null, null },
5569 .src_constraints = .{
5570 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5571 .{ .multiple_scalar_float = .{ .of = .word, .is = .word } },
5572 .any,
5573 },
5574 .patterns = &.{
5575 .{ .src = .{ .to_mem, .to_mem, .none } },
5576 },
5577 .call_frame = .{ .alignment = .@"16" },
5578 .extra_temps = .{
5579 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5580 .{ .type = .f16, .kind = .{ .reg = .ax } },
5581 .{ .type = .f32, .kind = .mem },
5582 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
5583 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5584 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
5585 .unused,
5586 .unused,
5587 .unused,
5588 .unused,
5589 .unused,
5590 },
5591 .dst_temps = .{ .mem, .unused },
5592 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
5593 .each = .{ .once = &.{
5594 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5595 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
5596 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
5597 .{ ._, ._ss, .mov, .tmp3x, .mem(.tmp2d), ._, ._ },
5598 .{ ._, ._, .movzx, .tmp1d, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
5599 .{ ._, ._, .mov, .mem(.tmp2d), .tmp1d, ._, ._ },
5600 .{ ._, ._ss, .mov, .tmp4x, .mem(.tmp2d), ._, ._ },
5601 .{ ._, ._, .call, .tmp5d, ._, ._, ._ },
5602 .{ ._, ._ss, .mov, .mem(.tmp2d), .tmp3x, ._, ._ },
5603 .{ ._, ._, .mov, .tmp1d, .mem(.tmp2d), ._, ._ },
5604 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
5605 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
5606 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5607 } },
5608 }, .{
5609 .required_features = .{ .avx, null, null, null },
5610 .src_constraints = .{
5611 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5612 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5613 .any,
5614 },
5615 .patterns = &.{
5616 .{ .src = .{ .to_sse, .mem, .none } },
5617 .{ .src = .{ .to_sse, .to_sse, .none } },
5618 },
5619 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5620 .each = .{ .once = &.{
5621 .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ },
5622 } },
5623 }, .{
5624 .required_features = .{ .sse, null, null, null },
5625 .src_constraints = .{
5626 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5627 .{ .scalar_float = .{ .of = .dword, .is = .dword } },
5628 .any,
5629 },
5630 .patterns = &.{
5631 .{ .src = .{ .to_mut_sse, .mem, .none } },
5632 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5633 },
5634 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5635 .each = .{ .once = &.{
5636 .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ },
5637 } },
5638 }, .{
5639 .required_features = .{ .avx, null, null, null },
5640 .src_constraints = .{
5641 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5642 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5643 .any,
5644 },
5645 .patterns = &.{
5646 .{ .src = .{ .to_sse, .mem, .none } },
5647 .{ .src = .{ .to_sse, .to_sse, .none } },
5648 },
5649 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5650 .each = .{ .once = &.{
5651 .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ },
5652 } },
5653 }, .{
5654 .required_features = .{ .sse, null, null, null },
5655 .src_constraints = .{
5656 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5657 .{ .scalar_float = .{ .of = .xword, .is = .dword } },
5658 .any,
5659 },
5660 .patterns = &.{
5661 .{ .src = .{ .to_mut_sse, .mem, .none } },
5662 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5663 },
5664 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5665 .each = .{ .once = &.{
5666 .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ },
5667 } },
5668 }, .{
5669 .required_features = .{ .avx, null, null, null },
5670 .src_constraints = .{
5671 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
5672 .{ .scalar_float = .{ .of = .yword, .is = .dword } },
5673 .any,
5674 },
5675 .patterns = &.{
5676 .{ .src = .{ .to_sse, .mem, .none } },
5677 .{ .src = .{ .to_sse, .to_sse, .none } },
5678 },
5679 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5680 .each = .{ .once = &.{
5681 .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ },
5682 } },
5683 }, .{
5684 .required_features = .{ .avx, null, null, null },
5685 .src_constraints = .{
5686 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
5687 .{ .multiple_scalar_float = .{ .of = .yword, .is = .dword } },
5688 .any,
5689 },
5690 .patterns = &.{
5691 .{ .src = .{ .to_mem, .to_mem, .none } },
5692 },
5693 .extra_temps = .{
5694 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5695 .{ .type = .vector_8_f32, .kind = .{ .rc = .sse } },
5696 .unused,
5697 .unused,
5698 .unused,
5699 .unused,
5700 .unused,
5701 .unused,
5702 .unused,
5703 .unused,
5704 .unused,
5705 },
5706 .dst_temps = .{ .mem, .unused },
5707 .clobbers = .{ .eflags = true },
5708 .each = .{ .once = &.{
5709 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5710 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5711 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5712 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5713 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5714 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5715 } },
5716 }, .{
5717 .required_features = .{ .sse, null, null, null },
5718 .src_constraints = .{
5719 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
5720 .{ .multiple_scalar_float = .{ .of = .xword, .is = .dword } },
5721 .any,
5722 },
5723 .patterns = &.{
5724 .{ .src = .{ .to_mem, .to_mem, .none } },
5725 },
5726 .extra_temps = .{
5727 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5728 .{ .type = .vector_4_f32, .kind = .{ .rc = .sse } },
5729 .unused,
5730 .unused,
5731 .unused,
5732 .unused,
5733 .unused,
5734 .unused,
5735 .unused,
5736 .unused,
5737 .unused,
5738 },
5739 .dst_temps = .{ .mem, .unused },
5740 .clobbers = .{ .eflags = true },
5741 .each = .{ .once = &.{
5742 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5743 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5744 .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5745 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5746 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5747 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5748 } },
5749 }, .{
5750 .required_features = .{ .avx, null, null, null },
5751 .src_constraints = .{
5752 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5753 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5754 .any,
5755 },
5756 .patterns = &.{
5757 .{ .src = .{ .to_sse, .mem, .none } },
5758 .{ .src = .{ .to_sse, .to_sse, .none } },
5759 },
5760 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5761 .each = .{ .once = &.{
5762 .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ },
5763 } },
5764 }, .{
5765 .required_features = .{ .sse2, null, null, null },
5766 .src_constraints = .{
5767 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5768 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5769 .any,
5770 },
5771 .patterns = &.{
5772 .{ .src = .{ .to_mut_sse, .mem, .none } },
5773 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5774 },
5775 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5776 .each = .{ .once = &.{
5777 .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ },
5778 } },
5779 }, .{
5780 .required_features = .{ .x87, null, null, null },
5781 .src_constraints = .{
5782 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5783 .{ .scalar_float = .{ .of = .qword, .is = .qword } },
5784 .any,
5785 },
5786 .patterns = &.{
5787 .{ .src = .{ .to_mem, .to_mem, .none } },
5788 },
5789 .extra_temps = .{
5790 .{ .type = .f64, .kind = .{ .reg = .st6 } },
5791 .{ .type = .f64, .kind = .{ .reg = .st7 } },
5792 .unused,
5793 .unused,
5794 .unused,
5795 .unused,
5796 .unused,
5797 .unused,
5798 .unused,
5799 .unused,
5800 .unused,
5801 },
5802 .dst_temps = .{ .mem, .unused },
5803 .each = .{ .once = &.{
5804 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
5805 .{ ._, .f_, .sub, .src1q, ._, ._, ._ },
5806 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
5807 } },
5808 }, .{
5809 .required_features = .{ .avx, null, null, null },
5810 .src_constraints = .{
5811 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5812 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5813 .any,
5814 },
5815 .patterns = &.{
5816 .{ .src = .{ .to_sse, .mem, .none } },
5817 .{ .src = .{ .to_sse, .to_sse, .none } },
5818 },
5819 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5820 .each = .{ .once = &.{
5821 .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ },
5822 } },
5823 }, .{
5824 .required_features = .{ .sse2, null, null, null },
5825 .src_constraints = .{
5826 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5827 .{ .scalar_float = .{ .of = .xword, .is = .qword } },
5828 .any,
5829 },
5830 .patterns = &.{
5831 .{ .src = .{ .to_mut_sse, .mem, .none } },
5832 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
5833 },
5834 .dst_temps = .{ .{ .ref = .src0 }, .unused },
5835 .each = .{ .once = &.{
5836 .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ },
5837 } },
5838 }, .{
5839 .required_features = .{ .avx, null, null, null },
5840 .src_constraints = .{
5841 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
5842 .{ .scalar_float = .{ .of = .yword, .is = .qword } },
5843 .any,
5844 },
5845 .patterns = &.{
5846 .{ .src = .{ .to_sse, .mem, .none } },
5847 .{ .src = .{ .to_sse, .to_sse, .none } },
5848 },
5849 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
5850 .each = .{ .once = &.{
5851 .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ },
5852 } },
5853 }, .{
5854 .required_features = .{ .avx, null, null, null },
5855 .src_constraints = .{
5856 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
5857 .{ .multiple_scalar_float = .{ .of = .yword, .is = .qword } },
5858 .any,
5859 },
5860 .patterns = &.{
5861 .{ .src = .{ .to_mem, .to_mem, .none } },
5862 },
5863 .extra_temps = .{
5864 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5865 .{ .type = .vector_4_f64, .kind = .{ .rc = .sse } },
5866 .unused,
5867 .unused,
5868 .unused,
5869 .unused,
5870 .unused,
5871 .unused,
5872 .unused,
5873 .unused,
5874 .unused,
5875 },
5876 .dst_temps = .{ .mem, .unused },
5877 .clobbers = .{ .eflags = true },
5878 .each = .{ .once = &.{
5879 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5880 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5881 .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
5882 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
5883 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
5884 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5885 } },
5886 }, .{
5887 .required_features = .{ .sse2, null, null, null },
5888 .src_constraints = .{
5889 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
5890 .{ .multiple_scalar_float = .{ .of = .xword, .is = .qword } },
5891 .any,
5892 },
5893 .patterns = &.{
5894 .{ .src = .{ .to_mem, .to_mem, .none } },
5895 },
5896 .extra_temps = .{
5897 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5898 .{ .type = .vector_2_f64, .kind = .{ .rc = .sse } },
5899 .unused,
5900 .unused,
5901 .unused,
5902 .unused,
5903 .unused,
5904 .unused,
5905 .unused,
5906 .unused,
5907 .unused,
5908 },
5909 .dst_temps = .{ .mem, .unused },
5910 .clobbers = .{ .eflags = true },
5911 .each = .{ .once = &.{
5912 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5913 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5914 .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5915 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
5916 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
5917 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5918 } },
5919 }, .{
5920 .required_features = .{ .x87, null, null, null },
5921 .src_constraints = .{
5922 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
5923 .{ .multiple_scalar_float = .{ .of = .qword, .is = .qword } },
5924 .any,
5925 },
5926 .patterns = &.{
5927 .{ .src = .{ .to_mem, .to_mem, .none } },
5928 },
5929 .extra_temps = .{
5930 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5931 .{ .type = .f64, .kind = .{ .reg = .st6 } },
5932 .{ .type = .f64, .kind = .{ .reg = .st7 } },
5933 .unused,
5934 .unused,
5935 .unused,
5936 .unused,
5937 .unused,
5938 .unused,
5939 .unused,
5940 .unused,
5941 },
5942 .dst_temps = .{ .mem, .unused },
5943 .each = .{ .once = &.{
5944 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5945 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5946 .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5947 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5948 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
5949 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
5950 } },
5951 }, .{
5952 .required_features = .{ .x87, null, null, null },
5953 .src_constraints = .{
5954 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5955 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5956 .any,
5957 },
5958 .patterns = &.{
5959 .{ .src = .{ .mem, .mem, .none } },
5960 },
5961 .extra_temps = .{
5962 .{ .type = .f80, .kind = .{ .reg = .st6 } },
5963 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5964 .unused,
5965 .unused,
5966 .unused,
5967 .unused,
5968 .unused,
5969 .unused,
5970 .unused,
5971 .unused,
5972 .unused,
5973 },
5974 .dst_temps = .{ .{ .rc = .x87 }, .unused },
5975 .each = .{ .once = &.{
5976 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
5977 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
5978 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
5979 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
5980 } },
5981 }, .{
5982 .required_features = .{ .x87, null, null, null },
5983 .src_constraints = .{
5984 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5985 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
5986 .any,
5987 },
5988 .patterns = &.{
5989 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },
5990 },
5991 .extra_temps = .{
5992 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5993 .unused,
5994 .unused,
5995 .unused,
5996 .unused,
5997 .unused,
5998 .unused,
5999 .unused,
6000 .unused,
6001 .unused,
6002 .unused,
6003 },
6004 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6005 .each = .{ .once = &.{
6006 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6007 .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ },
6008 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6009 } },
6010 }, .{
6011 .required_features = .{ .x87, null, null, null },
6012 .src_constraints = .{
6013 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6014 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6015 .any,
6016 },
6017 .patterns = &.{
6018 .{ .src = .{ .mem, .to_x87, .none } },
6019 .{ .src = .{ .to_x87, .to_x87, .none } },
6020 },
6021 .extra_temps = .{
6022 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6023 .unused,
6024 .unused,
6025 .unused,
6026 .unused,
6027 .unused,
6028 .unused,
6029 .unused,
6030 .unused,
6031 .unused,
6032 .unused,
6033 },
6034 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6035 .each = .{ .once = &.{
6036 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6037 .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ },
6038 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6039 } },
6040 }, .{
6041 .required_features = .{ .x87, null, null, null },
6042 .src_constraints = .{
6043 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6044 .{ .multiple_scalar_float = .{ .of = .xword, .is = .tbyte } },
6045 .any,
6046 },
6047 .patterns = &.{
6048 .{ .src = .{ .to_mem, .to_mem, .none } },
6049 },
6050 .extra_temps = .{
6051 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6052 .{ .type = .f80, .kind = .{ .reg = .st6 } },
6053 .{ .type = .f80, .kind = .{ .reg = .st7 } },
6054 .unused,
6055 .unused,
6056 .unused,
6057 .unused,
6058 .unused,
6059 .unused,
6060 .unused,
6061 .unused,
6062 },
6063 .dst_temps = .{ .mem, .unused },
6064 .each = .{ .once = &.{
6065 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6066 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6067 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6068 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
6069 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6070 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6071 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6072 } },
6073 }, .{
6074 .required_features = .{ .sse, null, null, null },
6075 .src_constraints = .{
6076 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
6077 .{ .scalar_float = .{ .of = .xword, .is = .xword } },
6078 .any,
6079 },
6080 .patterns = &.{
6081 .{ .src = .{ .{ .to_reg = .xmm0 }, .{ .to_reg = .xmm1 }, .none } },
6082 },
6083 .call_frame = .{ .alignment = .@"16" },
6084 .extra_temps = .{
6085 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6086 .unused,
6087 .unused,
6088 .unused,
6089 .unused,
6090 .unused,
6091 .unused,
6092 .unused,
6093 .unused,
6094 .unused,
6095 .unused,
6096 },
6097 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6098 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6099 .each = .{ .once = &.{
6100 .{ ._, ._, .call, .tmp0d, ._, ._, ._ },
6101 } },
6102 }, .{
6103 .required_features = .{ .avx, null, null, null },
6104 .src_constraints = .{
6105 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6106 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6107 .any,
6108 },
6109 .patterns = &.{
6110 .{ .src = .{ .to_mem, .to_mem, .none } },
6111 },
6112 .call_frame = .{ .alignment = .@"16" },
6113 .extra_temps = .{
6114 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6115 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6116 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6117 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6118 .unused,
6119 .unused,
6120 .unused,
6121 .unused,
6122 .unused,
6123 .unused,
6124 .unused,
6125 },
6126 .dst_temps = .{ .mem, .unused },
6127 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6128 .each = .{ .once = &.{
6129 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6130 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6131 .{ ._, .v_dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6132 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6133 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6134 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6135 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6136 } },
6137 }, .{
6138 .required_features = .{ .sse2, null, null, null },
6139 .src_constraints = .{
6140 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6141 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6142 .any,
6143 },
6144 .patterns = &.{
6145 .{ .src = .{ .to_mem, .to_mem, .none } },
6146 },
6147 .call_frame = .{ .alignment = .@"16" },
6148 .extra_temps = .{
6149 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6150 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6151 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6152 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6153 .unused,
6154 .unused,
6155 .unused,
6156 .unused,
6157 .unused,
6158 .unused,
6159 .unused,
6160 },
6161 .dst_temps = .{ .mem, .unused },
6162 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6163 .each = .{ .once = &.{
6164 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6165 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6166 .{ ._, ._dqa, .mov, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6167 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6168 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6169 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6170 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6171 } },
6172 }, .{
6173 .required_features = .{ .sse, null, null, null },
6174 .src_constraints = .{
6175 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6176 .{ .multiple_scalar_float = .{ .of = .xword, .is = .xword } },
6177 .any,
6178 },
6179 .patterns = &.{
6180 .{ .src = .{ .to_mem, .to_mem, .none } },
6181 },
6182 .call_frame = .{ .alignment = .@"16" },
6183 .extra_temps = .{
6184 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6185 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
6186 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6187 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
6188 .unused,
6189 .unused,
6190 .unused,
6191 .unused,
6192 .unused,
6193 .unused,
6194 .unused,
6195 },
6196 .dst_temps = .{ .mem, .unused },
6197 .clobbers = .{ .eflags = true, .caller_preserved = .ccc },
6198 .each = .{ .once = &.{
6199 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6200 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6201 .{ ._, ._ps, .mova, .tmp2x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6202 .{ ._, ._, .call, .tmp3d, ._, ._, ._ },
6203 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6204 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6205 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6206 } },
6207 } }) catch |err| switch (err) {
6208 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
6209 @tagName(air_tag),
6210 cg.typeOf(bin_op.lhs).fmt(pt),
6211 ops[0].tracking(cg),
6212 ops[1].tracking(cg),
6213 }),
6214 else => |e| return e,
6215 };
6216 switch (air_tag) {
6217 else => unreachable,
6218 .sub, .sub_optimized => {},
6219 .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) {
6220 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
6221 cg.typeOf(bin_op.lhs).fmt(pt),
6222 ops[0].tracking(cg),
6223 ops[1].tracking(cg),
6224 }),
6225 else => |e| return e,
6226 },
6227 }
6228 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
6229 },
6230 .sub_safe => unreachable,
6231 .mul, .mul_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, .mul) else {
6232 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
6233 const ty = cg.typeOf(bin_op.lhs);
6234 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
6235 var res: [1]Temp = undefined;
6236 cg.select(&res, &.{ty}, &ops, comptime &.{ .{
6237 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
6238 .patterns = &.{
6239 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
6240 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
6241 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
6242 },
6243 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6244 .clobbers = .{ .eflags = true },
6245 .each = .{ .once = &.{
6246 .{ ._, .i_, .mul, .src1b, ._, ._, ._ },
6247 } },
6248 }, .{
6249 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
6250 .patterns = &.{
6251 .{ .src = .{ .{ .to_reg = .al }, .mem, .none } },
6252 .{ .src = .{ .mem, .{ .to_reg = .al }, .none }, .commute = .{ 0, 1 } },
6253 .{ .src = .{ .{ .to_reg = .al }, .to_gpr, .none } },
6254 },
6255 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6256 .clobbers = .{ .eflags = true },
6257 .each = .{ .once = &.{
6258 .{ ._, ._, .mul, .src1b, ._, ._, ._ },
6259 } },
6260 }, .{
6261 .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any },
6262 .patterns = &.{
6263 .{ .src = .{ .mem, .imm16, .none } },
6264 .{ .src = .{ .imm16, .mem, .none }, .commute = .{ 0, 1 } },
6265 .{ .src = .{ .to_gpr, .imm16, .none } },
6266 .{ .src = .{ .imm16, .to_gpr, .none }, .commute = .{ 0, 1 } },
6267 },
6268 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6269 .clobbers = .{ .eflags = true },
6270 .each = .{ .once = &.{
6271 .{ ._, .i_, .mul, .dst0w, .src0w, .src1w, ._ },
6272 } },
6273 }, .{
6274 .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any },
6275 .patterns = &.{
6276 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6277 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6278 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6279 },
6280 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6281 .clobbers = .{ .eflags = true },
6282 .each = .{ .once = &.{
6283 .{ ._, .i_, .mul, .dst0w, .src1w, ._, ._ },
6284 } },
6285 }, .{
6286 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any },
6287 .patterns = &.{
6288 .{ .src = .{ .mem, .imm32, .none } },
6289 .{ .src = .{ .imm32, .mem, .none }, .commute = .{ 0, 1 } },
6290 .{ .src = .{ .to_gpr, .imm32, .none } },
6291 .{ .src = .{ .imm32, .to_gpr, .none }, .commute = .{ 0, 1 } },
6292 },
6293 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6294 .clobbers = .{ .eflags = true },
6295 .each = .{ .once = &.{
6296 .{ ._, .i_, .mul, .dst0d, .src0d, .src1d, ._ },
6297 } },
6298 }, .{
6299 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any },
6300 .patterns = &.{
6301 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6302 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6303 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6304 },
6305 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6306 .clobbers = .{ .eflags = true },
6307 .each = .{ .once = &.{
6308 .{ ._, .i_, .mul, .dst0d, .src1d, ._, ._ },
6309 } },
6310 }, .{
6311 .required_features = .{ .@"64bit", null, null, null },
6312 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },
6313 .patterns = &.{
6314 .{ .src = .{ .mem, .simm32, .none } },
6315 .{ .src = .{ .simm32, .mem, .none }, .commute = .{ 0, 1 } },
6316 .{ .src = .{ .to_gpr, .simm32, .none } },
6317 .{ .src = .{ .simm32, .to_gpr, .none }, .commute = .{ 0, 1 } },
6318 },
6319 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .general_purpose } }, .unused },
6320 .clobbers = .{ .eflags = true },
6321 .each = .{ .once = &.{
6322 .{ ._, .i_, .mul, .dst0q, .src0q, .src1q, ._ },
6323 } },
6324 }, .{
6325 .required_features = .{ .@"64bit", null, null, null },
6326 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },
6327 .patterns = &.{
6328 .{ .src = .{ .to_mut_gpr, .mem, .none } },
6329 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
6330 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
6331 },
6332 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6333 .clobbers = .{ .eflags = true },
6334 .each = .{ .once = &.{
6335 .{ ._, .i_, .mul, .dst0q, .src1q, ._, ._ },
6336 } },
6337 }, .{
6338 .required_features = .{ .@"64bit", null, null, null },
6339 .src_constraints = .{ .{ .int = .xword }, .{ .int = .xword }, .any },
6340 .patterns = &.{
6341 .{ .src = .{ .to_mem, .to_mem, .none } },
6342 },
6343 .extra_temps = .{
6344 .{ .type = .u64, .kind = .{ .reg = .rax } },
6345 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6346 .unused,
6347 .unused,
6348 .unused,
6349 .unused,
6350 .unused,
6351 .unused,
6352 .unused,
6353 .unused,
6354 .unused,
6355 },
6356 .dst_temps = .{ .mem, .unused },
6357 .clobbers = .{ .eflags = true },
6358 .each = .{ .once = &.{
6359 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
6360 .{ ._, ._, .mul, .src1q, ._, ._, ._ },
6361 .{ ._, ._, .mov, .dst0q, .tmp0q, ._, ._ },
6362 .{ ._, ._, .mov, .tmp0q, .src0q, ._, ._ },
6363 .{ ._, .i_, .mul, .tmp0q, .memd(.src1q, 8), ._, ._ },
6364 .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ },
6365 .{ ._, ._, .mov, .tmp0q, .src1q, ._, ._ },
6366 .{ ._, .i_, .mul, .tmp0q, .memd(.src0q, 8), ._, ._ },
6367 .{ ._, ._, .add, .tmp1q, .tmp0q, ._, ._ },
6368 .{ ._, ._, .mov, .memd(.dst0q, 8), .tmp1q, ._, ._ },
6369 } },
6370 }, .{
6371 .required_features = .{ .@"64bit", .bmi2, .adx, null },
6372 .src_constraints = .{
6373 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6374 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6375 .any,
6376 },
6377 .patterns = &.{
6378 .{ .src = .{ .to_mem, .to_mem, .none } },
6379 },
6380 .extra_temps = .{
6381 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6382 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6383 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6384 .{ .type = .isize, .kind = .{ .reg = .rcx } },
6385 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6386 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6387 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6388 .unused,
6389 .unused,
6390 .unused,
6391 .unused,
6392 },
6393 .dst_temps = .{ .mem, .unused },
6394 .clobbers = .{ .eflags = true },
6395 .each = .{ .once = &.{
6396 .{ ._, ._, .mov, .tmp0d, .sia(-8, .src0, .add_size), ._, ._ },
6397 .{ ._, ._, .lea, .tmp1p, .mem(.src1), ._, ._ },
6398 .{ .@"0:", ._, .xor, .tmp2d, .tmp2d, ._, ._ },
6399 .{ ._, ._, .@"or", .tmp2q, .memi(.src0q, .tmp0), ._, ._ },
6400 .{ ._, ._z, .j, .@"2f", ._, ._, ._ },
6401 .{ ._, ._, .lea, .tmp3p, .leaad(.tmp0, .sub_src0_size, 8), ._, ._ },
6402 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6403 .{ .@"1:", ._x, .mul, .tmp6q, .tmp5q, .leai(.tmp1q, .tmp3), ._ },
6404 .{ ._, ._x, .adc, .tmp5q, .tmp4q, ._, ._ },
6405 .{ ._, ._, .mov, .memiad(.dst0q, .tmp3, .add_size, -8), .tmp5q, ._, ._ },
6406 .{ ._, ._rcxz, .j, .@"1f", ._, ._, ._ },
6407 .{ ._, ._x, .ado, .tmp6q, .memia(.dst0q, .tmp3, .add_size), ._, ._ },
6408 .{ ._, ._, .mov, .tmp4q, .tmp6q, ._, ._ },
6409 .{ ._, ._, .lea, .tmp3p, .lead(.tmp3, 8), ._, ._ },
6410 .{ ._, ._mp, .j, .@"1b", ._, ._, ._ },
6411 .{ .@"2:", ._, .mov, .memi(.dst0q, .tmp0), .tmp2q, ._, ._ },
6412 .{ .@"1:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6413 .{ ._, ._, .sub, .tmp0d, .si(8), ._, ._ },
6414 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6415 } },
6416 }, .{
6417 .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null },
6418 .src_constraints = .{
6419 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6420 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6421 .any,
6422 },
6423 .patterns = &.{
6424 .{ .src = .{ .to_mem, .to_mem, .none } },
6425 },
6426 .extra_temps = .{
6427 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6428 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6429 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6430 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6431 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6432 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6433 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6434 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6435 .unused,
6436 .unused,
6437 .unused,
6438 },
6439 .dst_temps = .{ .mem, .unused },
6440 .clobbers = .{ .eflags = true },
6441 .each = .{ .once = &.{
6442 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6443 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6444 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6445 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6446 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6447 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6448 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6449 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6450 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6451 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6452 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6453 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6454 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6455 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6456 .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ },
6457 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6458 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6459 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6460 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6461 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6462 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
6463 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6464 } },
6465 }, .{
6466 .required_features = .{ .@"64bit", .bmi2, null, null },
6467 .src_constraints = .{
6468 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6469 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6470 .any,
6471 },
6472 .patterns = &.{
6473 .{ .src = .{ .to_mem, .to_mem, .none } },
6474 },
6475 .extra_temps = .{
6476 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6477 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6478 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6479 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6480 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6481 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6482 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6483 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6484 .unused,
6485 .unused,
6486 .unused,
6487 },
6488 .dst_temps = .{ .mem, .unused },
6489 .clobbers = .{ .eflags = true },
6490 .each = .{ .once = &.{
6491 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6492 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6493 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6494 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6495 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6496 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6497 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6498 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6499 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6500 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6501 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6502 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6503 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6504 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6505 .{ .@"2:", ._x, .mul, .tmp7q, .tmp6q, .leasi(.tmp1q, .@"8", .tmp2), ._ },
6506 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6507 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6508 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6509 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6510 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6511 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
6512 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
6513 } },
6514 }, .{
6515 .required_features = .{ .@"64bit", .slow_incdec, null, null },
6516 .src_constraints = .{
6517 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6518 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6519 .any,
6520 },
6521 .patterns = &.{
6522 .{ .src = .{ .to_mem, .to_mem, .none } },
6523 },
6524 .extra_temps = .{
6525 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6526 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6527 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6528 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6529 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6530 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6531 .{ .type = .u64, .kind = .{ .reg = .rax } },
6532 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6533 .unused,
6534 .unused,
6535 .unused,
6536 },
6537 .dst_temps = .{ .mem, .unused },
6538 .clobbers = .{ .eflags = true },
6539 .each = .{ .once = &.{
6540 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6541 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6542 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6543 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6544 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6545 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6546 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6547 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6548 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6549 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6550 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6551 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6552 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6553 .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ },
6554 .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ },
6555 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6556 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6557 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6558 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6559 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6560 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6561 .{ ._, ._, .sub, .tmp0d, .si(1), ._, ._ },
6562 .{ ._, ._ae, .j, .@"0b", ._, ._, ._ },
6563 } },
6564 }, .{
6565 .required_features = .{ .@"64bit", null, null, null },
6566 .src_constraints = .{
6567 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6568 .{ .remainder_int = .{ .of = .qword, .is = .qword } },
6569 .any,
6570 },
6571 .patterns = &.{
6572 .{ .src = .{ .to_mem, .to_mem, .none } },
6573 },
6574 .extra_temps = .{
6575 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6576 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
6577 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6578 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6579 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
6580 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
6581 .{ .type = .u64, .kind = .{ .reg = .rax } },
6582 .{ .type = .u64, .kind = .{ .reg = .rdx } },
6583 .unused,
6584 .unused,
6585 .unused,
6586 },
6587 .dst_temps = .{ .mem, .unused },
6588 .clobbers = .{ .eflags = true },
6589 .each = .{ .once = &.{
6590 .{ ._, ._, .mov, .tmp0d, .sia(-1, .src0, .add_size_div_8), ._, ._ },
6591 .{ ._, ._, .lea, .tmp1p, .memd(.src1, 8), ._, ._ },
6592 .{ .@"0:", ._, .lea, .tmp2p, .leaa(.tmp0, .sub_src0_size_div_8), ._, ._ },
6593 .{ ._, ._, .xor, .tmp3d, .tmp3d, ._, ._ },
6594 .{ ._, ._, .xor, .tmp4d, .tmp4d, ._, ._ },
6595 .{ ._, ._, .xor, .tmp5d, .tmp5d, ._, ._ },
6596 .{ ._, ._, .@"or", .tmp3q, .memsi(.src0q, .@"8", .tmp0), ._, ._ },
6597 .{ ._, ._nz, .j, .@"2f", ._, ._, ._ },
6598 .{ ._, ._, .mov, .memsi(.dst0q, .@"8", .tmp0), .tmp3q, ._, ._ },
6599 .{ ._, ._mp, .j, .@"3f", ._, ._, ._ },
6600 .{ .@"1:", ._, .adc, .tmp7q, .memsia(.dst0q, .@"8", .tmp2, .add_size), ._, ._ },
6601 .{ ._, ._, .adc, .tmp4b, .si(0), ._, ._ },
6602 .{ ._, ._, .mov, .tmp5q, .tmp7q, ._, ._ },
6603 .{ .@"2:", ._, .mov, .tmp6q, .tmp3q, ._, ._ },
6604 .{ ._, ._, .mul, .leasi(.tmp1q, .@"8", .tmp2), ._, ._, ._ },
6605 .{ ._, ._l, .sh, .tmp4b, .ui(4), ._, ._ },
6606 .{ ._, ._, .adc, .tmp6q, .tmp5q, ._, ._ },
6607 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp2, .add_size), .tmp6q, ._, ._ },
6608 .{ ._, ._c, .in, .tmp2p, ._, ._, ._ },
6609 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
6610 .{ .@"3:", ._, .lea, .tmp1p, .lead(.tmp1, 8), ._, ._ },
6611 .{ ._, ._c, .de, .tmp0d, ._, ._, ._ },
6612 .{ ._, ._ns, .j, .@"0b", ._, ._, ._ },
6613 } },
6614 }, .{
6615 .required_features = .{ .avx, null, null, null },
6616 .src_constraints = .{
6617 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6618 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6619 .any,
6620 },
6621 .patterns = &.{
6622 .{ .src = .{ .mem, .mem, .none } },
6623 .{ .src = .{ .to_sse, .mem, .none } },
6624 .{ .src = .{ .mem, .to_sse, .none } },
6625 .{ .src = .{ .to_sse, .to_sse, .none } },
6626 },
6627 .extra_temps = .{
6628 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6629 .unused,
6630 .unused,
6631 .unused,
6632 .unused,
6633 .unused,
6634 .unused,
6635 .unused,
6636 .unused,
6637 .unused,
6638 .unused,
6639 },
6640 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6641 .each = .{ .once = &.{
6642 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },
6643 .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6644 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6645 .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ },
6646 } },
6647 }, .{
6648 .required_features = .{ .sse4_1, null, null, null },
6649 .src_constraints = .{
6650 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6651 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6652 .any,
6653 },
6654 .patterns = &.{
6655 .{ .src = .{ .mem, .mem, .none } },
6656 .{ .src = .{ .to_sse, .mem, .none } },
6657 .{ .src = .{ .mem, .to_sse, .none } },
6658 .{ .src = .{ .to_sse, .to_sse, .none } },
6659 },
6660 .extra_temps = .{
6661 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6662 .unused,
6663 .unused,
6664 .unused,
6665 .unused,
6666 .unused,
6667 .unused,
6668 .unused,
6669 .unused,
6670 .unused,
6671 .unused,
6672 },
6673 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6674 .each = .{ .once = &.{
6675 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },
6676 .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6677 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6678 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
6679 } },
6680 }, .{
6681 .required_features = .{ .sse2, null, null, null },
6682 .src_constraints = .{
6683 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6684 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6685 .any,
6686 },
6687 .patterns = &.{
6688 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6689 },
6690 .extra_temps = .{
6691 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6692 .unused,
6693 .unused,
6694 .unused,
6695 .unused,
6696 .unused,
6697 .unused,
6698 .unused,
6699 .unused,
6700 .unused,
6701 .unused,
6702 },
6703 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6704 .each = .{ .once = &.{
6705 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6706 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
6707 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6708 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6709 .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ },
6710 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6711 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6712 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
6713 } },
6714 }, .{
6715 .required_features = .{ .avx2, null, null, null },
6716 .src_constraints = .{
6717 .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } },
6718 .{ .scalar_signed_int = .{ .of = .xword, .is = .byte } },
6719 .any,
6720 },
6721 .patterns = &.{
6722 .{ .src = .{ .mem, .mem, .none } },
6723 .{ .src = .{ .to_sse, .mem, .none } },
6724 .{ .src = .{ .mem, .to_sse, .none } },
6725 .{ .src = .{ .to_sse, .to_sse, .none } },
6726 },
6727 .extra_temps = .{
6728 .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6729 .unused,
6730 .unused,
6731 .unused,
6732 .unused,
6733 .unused,
6734 .unused,
6735 .unused,
6736 .unused,
6737 .unused,
6738 .unused,
6739 },
6740 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6741 .each = .{ .once = &.{
6742 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },
6743 .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ },
6744 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6745 .{ ._, .vp_b, .ackssw, .dst0y, .dst0y, .dst0y, ._ },
6746 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6747 } },
6748 }, .{
6749 .required_features = .{ .avx, null, null, null },
6750 .src_constraints = .{
6751 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6752 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6753 .any,
6754 },
6755 .patterns = &.{
6756 .{ .src = .{ .mem, .mem, .none } },
6757 .{ .src = .{ .to_sse, .mem, .none } },
6758 .{ .src = .{ .mem, .to_sse, .none } },
6759 .{ .src = .{ .to_sse, .to_sse, .none } },
6760 },
6761 .extra_temps = .{
6762 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6763 .unused,
6764 .unused,
6765 .unused,
6766 .unused,
6767 .unused,
6768 .unused,
6769 .unused,
6770 .unused,
6771 .unused,
6772 .unused,
6773 },
6774 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6775 .each = .{ .once = &.{
6776 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },
6777 .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6778 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6779 .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ },
6780 } },
6781 }, .{
6782 .required_features = .{ .sse4_1, null, null, null },
6783 .src_constraints = .{
6784 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6785 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6786 .any,
6787 },
6788 .patterns = &.{
6789 .{ .src = .{ .mem, .mem, .none } },
6790 .{ .src = .{ .to_sse, .mem, .none } },
6791 .{ .src = .{ .mem, .to_sse, .none } },
6792 .{ .src = .{ .to_sse, .to_sse, .none } },
6793 },
6794 .extra_temps = .{
6795 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6796 .unused,
6797 .unused,
6798 .unused,
6799 .unused,
6800 .unused,
6801 .unused,
6802 .unused,
6803 .unused,
6804 .unused,
6805 .unused,
6806 },
6807 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6808 .each = .{ .once = &.{
6809 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },
6810 .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6811 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6812 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6813 } },
6814 }, .{
6815 .required_features = .{ .sse2, null, null, null },
6816 .src_constraints = .{
6817 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6818 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6819 .any,
6820 },
6821 .patterns = &.{
6822 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6823 },
6824 .extra_temps = .{
6825 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
6826 .unused,
6827 .unused,
6828 .unused,
6829 .unused,
6830 .unused,
6831 .unused,
6832 .unused,
6833 .unused,
6834 .unused,
6835 .unused,
6836 },
6837 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6838 .each = .{ .once = &.{
6839 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6840 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6841 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6842 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6843 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6844 } },
6845 }, .{
6846 .required_features = .{ .avx2, null, null, null },
6847 .src_constraints = .{
6848 .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
6849 .{ .scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
6850 .any,
6851 },
6852 .patterns = &.{
6853 .{ .src = .{ .mem, .mem, .none } },
6854 .{ .src = .{ .to_sse, .mem, .none } },
6855 .{ .src = .{ .mem, .to_sse, .none } },
6856 .{ .src = .{ .to_sse, .to_sse, .none } },
6857 },
6858 .extra_temps = .{
6859 .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
6860 .unused,
6861 .unused,
6862 .unused,
6863 .unused,
6864 .unused,
6865 .unused,
6866 .unused,
6867 .unused,
6868 .unused,
6869 .unused,
6870 },
6871 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
6872 .each = .{ .once = &.{
6873 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },
6874 .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ },
6875 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6876 .{ ._, .vp_b, .ackusw, .dst0y, .dst0y, .dst0y, ._ },
6877 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
6878 } },
6879 }, .{
6880 .required_features = .{ .avx2, null, null, null },
6881 .src_constraints = .{
6882 .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } },
6883 .{ .multiple_scalar_signed_int = .{ .of = .xword, .is = .byte } },
6884 .any,
6885 },
6886 .patterns = &.{
6887 .{ .src = .{ .to_mem, .to_mem, .none } },
6888 },
6889 .extra_temps = .{
6890 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6891 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6892 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
6893 .unused,
6894 .unused,
6895 .unused,
6896 .unused,
6897 .unused,
6898 .unused,
6899 .unused,
6900 .unused,
6901 },
6902 .dst_temps = .{ .mem, .unused },
6903 .clobbers = .{ .eflags = true },
6904 .each = .{ .once = &.{
6905 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6906 .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6907 .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6908 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
6909 .{ ._, .vp_b, .ackssw, .tmp1y, .tmp1y, .tmp1y, ._ },
6910 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
6911 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6912 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
6913 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6914 } },
6915 }, .{
6916 .required_features = .{ .avx, null, null, null },
6917 .src_constraints = .{
6918 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6919 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6920 .any,
6921 },
6922 .patterns = &.{
6923 .{ .src = .{ .to_mem, .to_mem, .none } },
6924 },
6925 .extra_temps = .{
6926 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6927 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6928 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6929 .unused,
6930 .unused,
6931 .unused,
6932 .unused,
6933 .unused,
6934 .unused,
6935 .unused,
6936 .unused,
6937 },
6938 .dst_temps = .{ .mem, .unused },
6939 .clobbers = .{ .eflags = true },
6940 .each = .{ .once = &.{
6941 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6942 .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6943 .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6944 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
6945 .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ },
6946 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6947 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6948 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6949 } },
6950 }, .{
6951 .required_features = .{ .sse4_1, null, null, null },
6952 .src_constraints = .{
6953 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6954 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6955 .any,
6956 },
6957 .patterns = &.{
6958 .{ .src = .{ .to_mem, .to_mem, .none } },
6959 },
6960 .extra_temps = .{
6961 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6962 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6963 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6964 .unused,
6965 .unused,
6966 .unused,
6967 .unused,
6968 .unused,
6969 .unused,
6970 .unused,
6971 .unused,
6972 },
6973 .dst_temps = .{ .mem, .unused },
6974 .clobbers = .{ .eflags = true },
6975 .each = .{ .once = &.{
6976 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6977 .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6978 .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6979 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
6980 .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ },
6981 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
6982 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
6983 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
6984 } },
6985 }, .{
6986 .required_features = .{ .sse2, null, null, null },
6987 .src_constraints = .{
6988 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6989 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6990 .any,
6991 },
6992 .patterns = &.{
6993 .{ .src = .{ .to_mem, .to_mem, .none } },
6994 },
6995 .extra_temps = .{
6996 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
6997 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6998 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6999 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
7000 .unused,
7001 .unused,
7002 .unused,
7003 .unused,
7004 .unused,
7005 .unused,
7006 .unused,
7007 },
7008 .dst_temps = .{ .mem, .unused },
7009 .clobbers = .{ .eflags = true },
7010 .each = .{ .once = &.{
7011 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7012 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7013 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7014 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ },
7015 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7016 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7017 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7018 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ },
7019 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7020 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7021 .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ },
7022 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7023 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7024 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7025 } },
7026 }, .{
7027 .required_features = .{ .slow_incdec, null, null, null },
7028 .src_constraints = .{
7029 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7030 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7031 .any,
7032 },
7033 .patterns = &.{
7034 .{ .src = .{ .to_mem, .to_mem, .none } },
7035 },
7036 .extra_temps = .{
7037 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7038 .{ .type = .i8, .kind = .{ .reg = .al } },
7039 .unused,
7040 .unused,
7041 .unused,
7042 .unused,
7043 .unused,
7044 .unused,
7045 .unused,
7046 .unused,
7047 .unused,
7048 },
7049 .dst_temps = .{ .mem, .unused },
7050 .clobbers = .{ .eflags = true },
7051 .each = .{ .once = &.{
7052 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7053 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7054 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7055 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7056 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
7057 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7058 } },
7059 }, .{
7060 .src_constraints = .{
7061 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7062 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7063 .any,
7064 },
7065 .patterns = &.{
7066 .{ .src = .{ .to_mem, .to_mem, .none } },
7067 },
7068 .extra_temps = .{
7069 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7070 .{ .type = .i8, .kind = .{ .reg = .al } },
7071 .unused,
7072 .unused,
7073 .unused,
7074 .unused,
7075 .unused,
7076 .unused,
7077 .unused,
7078 .unused,
7079 .unused,
7080 },
7081 .dst_temps = .{ .mem, .unused },
7082 .clobbers = .{ .eflags = true },
7083 .each = .{ .once = &.{
7084 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7085 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7086 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7087 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7088 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7089 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7090 } },
7091 }, .{
7092 .required_features = .{ .avx2, null, null, null },
7093 .src_constraints = .{
7094 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7095 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7096 .any,
7097 },
7098 .patterns = &.{
7099 .{ .src = .{ .to_mem, .to_mem, .none } },
7100 },
7101 .extra_temps = .{
7102 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7103 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7104 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7105 .unused,
7106 .unused,
7107 .unused,
7108 .unused,
7109 .unused,
7110 .unused,
7111 .unused,
7112 .unused,
7113 },
7114 .dst_temps = .{ .mem, .unused },
7115 .clobbers = .{ .eflags = true },
7116 .each = .{ .once = &.{
7117 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7118 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7119 .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7120 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
7121 .{ ._, .vp_b, .ackusw, .tmp1y, .tmp1y, .tmp1y, ._ },
7122 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
7123 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7124 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7125 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7126 } },
7127 }, .{
7128 .required_features = .{ .avx, null, null, null },
7129 .src_constraints = .{
7130 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7131 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7132 .any,
7133 },
7134 .patterns = &.{
7135 .{ .src = .{ .to_mem, .to_mem, .none } },
7136 },
7137 .extra_temps = .{
7138 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7139 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7140 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7141 .unused,
7142 .unused,
7143 .unused,
7144 .unused,
7145 .unused,
7146 .unused,
7147 .unused,
7148 .unused,
7149 },
7150 .dst_temps = .{ .mem, .unused },
7151 .clobbers = .{ .eflags = true },
7152 .each = .{ .once = &.{
7153 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7154 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7155 .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7156 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
7157 .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ },
7158 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7159 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7160 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7161 } },
7162 }, .{
7163 .required_features = .{ .sse4_1, null, null, null },
7164 .src_constraints = .{
7165 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7166 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7167 .any,
7168 },
7169 .patterns = &.{
7170 .{ .src = .{ .to_mem, .to_mem, .none } },
7171 },
7172 .extra_temps = .{
7173 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7174 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7175 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7176 .unused,
7177 .unused,
7178 .unused,
7179 .unused,
7180 .unused,
7181 .unused,
7182 .unused,
7183 .unused,
7184 },
7185 .dst_temps = .{ .mem, .unused },
7186 .clobbers = .{ .eflags = true },
7187 .each = .{ .once = &.{
7188 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7189 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7190 .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7191 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
7192 .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ },
7193 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7194 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7195 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7196 } },
7197 }, .{
7198 .required_features = .{ .sse2, null, null, null },
7199 .src_constraints = .{
7200 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7201 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7202 .any,
7203 },
7204 .patterns = &.{
7205 .{ .src = .{ .to_mem, .to_mem, .none } },
7206 },
7207 .extra_temps = .{
7208 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7209 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7210 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7211 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7212 .unused,
7213 .unused,
7214 .unused,
7215 .unused,
7216 .unused,
7217 .unused,
7218 .unused,
7219 },
7220 .dst_temps = .{ .mem, .unused },
7221 .clobbers = .{ .eflags = true },
7222 .each = .{ .once = &.{
7223 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7224 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7225 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7226 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7227 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7228 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7229 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7230 .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ },
7231 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
7232 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
7233 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7234 } },
7235 }, .{
7236 .required_features = .{ .slow_incdec, null, null, null },
7237 .src_constraints = .{
7238 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7239 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7240 .any,
7241 },
7242 .patterns = &.{
7243 .{ .src = .{ .to_mem, .to_mem, .none } },
7244 },
7245 .extra_temps = .{
7246 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7247 .{ .type = .u8, .kind = .{ .reg = .al } },
7248 .unused,
7249 .unused,
7250 .unused,
7251 .unused,
7252 .unused,
7253 .unused,
7254 .unused,
7255 .unused,
7256 .unused,
7257 },
7258 .dst_temps = .{ .mem, .unused },
7259 .clobbers = .{ .eflags = true },
7260 .each = .{ .once = &.{
7261 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7262 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7263 .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7264 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7265 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
7266 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7267 } },
7268 }, .{
7269 .src_constraints = .{
7270 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7271 .{ .multiple_scalar_unsigned_int = .{ .of = .byte, .is = .byte } },
7272 .any,
7273 },
7274 .patterns = &.{
7275 .{ .src = .{ .to_mem, .to_mem, .none } },
7276 },
7277 .extra_temps = .{
7278 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7279 .{ .type = .u8, .kind = .{ .reg = .al } },
7280 .unused,
7281 .unused,
7282 .unused,
7283 .unused,
7284 .unused,
7285 .unused,
7286 .unused,
7287 .unused,
7288 .unused,
7289 },
7290 .dst_temps = .{ .mem, .unused },
7291 .clobbers = .{ .eflags = true },
7292 .each = .{ .once = &.{
7293 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7294 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7295 .{ ._, ._, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7296 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7297 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7298 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
7299 } },
7300 }, .{
7301 .required_features = .{ .avx, null, null, null },
7302 .src_constraints = .{
7303 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7304 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7305 .any,
7306 },
7307 .patterns = &.{
7308 .{ .src = .{ .to_sse, .mem, .none } },
7309 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7310 .{ .src = .{ .to_sse, .to_sse, .none } },
7311 },
7312 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7313 .each = .{ .once = &.{
7314 .{ ._, .vp_w, .mull, .dst0x, .src0x, .src1x, ._ },
7315 } },
7316 }, .{
7317 .required_features = .{ .sse2, null, null, null },
7318 .src_constraints = .{
7319 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7320 .{ .scalar_int = .{ .of = .xword, .is = .word } },
7321 .any,
7322 },
7323 .patterns = &.{
7324 .{ .src = .{ .to_mut_sse, .mem, .none } },
7325 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
7326 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
7327 },
7328 .dst_temps = .{ .{ .ref = .src0 }, .unused },
7329 .each = .{ .once = &.{
7330 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
7331 } },
7332 }, .{
7333 .required_features = .{ .avx2, null, null, null },
7334 .src_constraints = .{
7335 .{ .scalar_int = .{ .of = .yword, .is = .word } },
7336 .{ .scalar_int = .{ .of = .yword, .is = .word } },
7337 .any,
7338 },
7339 .patterns = &.{
7340 .{ .src = .{ .to_sse, .mem, .none } },
7341 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7342 .{ .src = .{ .to_sse, .to_sse, .none } },
7343 },
7344 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7345 .each = .{ .once = &.{
7346 .{ ._, .vp_w, .mull, .dst0y, .src0y, .src1y, ._ },
7347 } },
7348 }, .{
7349 .required_features = .{ .avx2, null, null, null },
7350 .src_constraints = .{
7351 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
7352 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
7353 .any,
7354 },
7355 .patterns = &.{
7356 .{ .src = .{ .to_mem, .to_mem, .none } },
7357 },
7358 .extra_temps = .{
7359 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7360 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7361 .unused,
7362 .unused,
7363 .unused,
7364 .unused,
7365 .unused,
7366 .unused,
7367 .unused,
7368 .unused,
7369 .unused,
7370 },
7371 .dst_temps = .{ .mem, .unused },
7372 .clobbers = .{ .eflags = true },
7373 .each = .{ .once = &.{
7374 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7375 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
7376 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
7377 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
7378 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7379 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7380 } },
7381 }, .{
7382 .required_features = .{ .avx, null, null, null },
7383 .src_constraints = .{
7384 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7385 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7386 .any,
7387 },
7388 .patterns = &.{
7389 .{ .src = .{ .to_mem, .to_mem, .none } },
7390 },
7391 .extra_temps = .{
7392 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7393 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7394 .unused,
7395 .unused,
7396 .unused,
7397 .unused,
7398 .unused,
7399 .unused,
7400 .unused,
7401 .unused,
7402 .unused,
7403 },
7404 .dst_temps = .{ .mem, .unused },
7405 .clobbers = .{ .eflags = true },
7406 .each = .{ .once = &.{
7407 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7408 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7409 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },
7410 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7411 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7412 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7413 } },
7414 }, .{
7415 .required_features = .{ .sse2, null, null, null },
7416 .src_constraints = .{
7417 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7418 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
7419 .any,
7420 },
7421 .patterns = &.{
7422 .{ .src = .{ .to_mem, .to_mem, .none } },
7423 },
7424 .extra_temps = .{
7425 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7426 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7427 .unused,
7428 .unused,
7429 .unused,
7430 .unused,
7431 .unused,
7432 .unused,
7433 .unused,
7434 .unused,
7435 .unused,
7436 },
7437 .dst_temps = .{ .mem, .unused },
7438 .clobbers = .{ .eflags = true },
7439 .each = .{ .once = &.{
7440 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7441 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7442 .{ ._, .p_w, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7443 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7444 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7445 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7446 } },
7447 }, .{
7448 .src_constraints = .{
7449 .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } },
7450 .{ .multiple_scalar_signed_int = .{ .of = .word, .is = .word } },
7451 .any,
7452 },
7453 .patterns = &.{
7454 .{ .src = .{ .to_mem, .to_mem, .none } },
7455 },
7456 .extra_temps = .{
7457 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7458 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
7459 .unused,
7460 .unused,
7461 .unused,
7462 .unused,
7463 .unused,
7464 .unused,
7465 .unused,
7466 .unused,
7467 .unused,
7468 },
7469 .dst_temps = .{ .mem, .unused },
7470 .clobbers = .{ .eflags = true },
7471 .each = .{ .once = &.{
7472 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7473 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
7474 .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
7475 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
7476 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
7477 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7478 } },
7479 }, .{
7480 .src_constraints = .{
7481 .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } },
7482 .{ .multiple_scalar_unsigned_int = .{ .of = .word, .is = .word } },
7483 .any,
7484 },
7485 .patterns = &.{
7486 .{ .src = .{ .to_mem, .to_mem, .none } },
7487 },
7488 .extra_temps = .{
7489 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7490 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },
7491 .unused,
7492 .unused,
7493 .unused,
7494 .unused,
7495 .unused,
7496 .unused,
7497 .unused,
7498 .unused,
7499 .unused,
7500 },
7501 .dst_temps = .{ .mem, .unused },
7502 .clobbers = .{ .eflags = true },
7503 .each = .{ .once = &.{
7504 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7505 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
7506 .{ ._, .i_, .mul, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
7507 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
7508 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
7509 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7510 } },
7511 }, .{
7512 .required_features = .{ .avx, null, null, null },
7513 .src_constraints = .{
7514 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7515 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7516 .any,
7517 },
7518 .patterns = &.{
7519 .{ .src = .{ .to_sse, .mem, .none } },
7520 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7521 .{ .src = .{ .to_sse, .to_sse, .none } },
7522 },
7523 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7524 .each = .{ .once = &.{
7525 .{ ._, .vp_d, .mull, .dst0x, .src0x, .src1x, ._ },
7526 } },
7527 }, .{
7528 .required_features = .{ .sse4_1, null, null, null },
7529 .src_constraints = .{
7530 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7531 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
7532 .any,
7533 },
7534 .patterns = &.{
7535 .{ .src = .{ .to_mut_sse, .mem, .none } },
7536 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
7537 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
7538 },
7539 .dst_temps = .{ .{ .ref = .src0 }, .unused },
7540 .each = .{ .once = &.{
7541 .{ ._, .p_d, .mull, .dst0x, .src1x, ._, ._ },
7542 } },
7543 }, .{
7544 .required_features = .{ .avx2, null, null, null },
7545 .src_constraints = .{
7546 .{ .scalar_int = .{ .of = .yword, .is = .dword } },
7547 .{ .scalar_int = .{ .of = .yword, .is = .dword } },
7548 .any,
7549 },
7550 .patterns = &.{
7551 .{ .src = .{ .to_sse, .mem, .none } },
7552 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
7553 .{ .src = .{ .to_sse, .to_sse, .none } },
7554 },
7555 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
7556 .each = .{ .once = &.{
7557 .{ ._, .vp_d, .mull, .dst0y, .src0y, .src1y, ._ },
7558 } },
7559 }, .{
7560 .required_features = .{ .avx2, null, null, null },
7561 .src_constraints = .{
7562 .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } },
7563 .{ .multiple_scalar_int = .{ .of = .yword, .is = .dword } },
7564 .any,
7565 },
7566 .patterns = &.{
7567 .{ .src = .{ .to_mem, .to_mem, .none } },
7568 },
7569 .extra_temps = .{
7570 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7571 .{ .type = .vector_8_u32, .kind = .{ .rc = .sse } },
7572 .unused,
7573 .unused,
7574 .unused,
7575 .unused,
7576 .unused,
7577 .unused,
7578 .unused,
7579 .unused,
7580 .unused,
7581 },
7582 .dst_temps = .{ .mem, .unused },
7583 .clobbers = .{ .eflags = true },
7584 .each = .{ .once = &.{
7585 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7586 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
7587 .{ ._, .vp_d, .mull, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
7588 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
7589 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
7590 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7591 } },
7592 }, .{
7593 .required_features = .{ .avx, null, null, null },
7594 .src_constraints = .{
7595 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7596 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7597 .any,
7598 },
7599 .patterns = &.{
7600 .{ .src = .{ .to_mem, .to_mem, .none } },
7601 },
7602 .extra_temps = .{
7603 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7604 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
7605 .unused,
7606 .unused,
7607 .unused,
7608 .unused,
7609 .unused,
7610 .unused,
7611 .unused,
7612 .unused,
7613 .unused,
7614 },
7615 .dst_temps = .{ .mem, .unused },
7616 .clobbers = .{ .eflags = true },
7617 .each = .{ .once = &.{
7618 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7619 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7620 .{ ._, .vp_d, .mull, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },
7621 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7622 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7623 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7624 } },
7625 }, .{
7626 .required_features = .{ .sse4_1, null, null, null },
7627 .src_constraints = .{
7628 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7629 .{ .multiple_scalar_int = .{ .of = .xword, .is = .dword } },
7630 .any,
7631 },
7632 .patterns = &.{
7633 .{ .src = .{ .to_mem, .to_mem, .none } },
7634 },
7635 .extra_temps = .{
7636 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7637 .{ .type = .vector_4_u32, .kind = .{ .rc = .sse } },
7638 .unused,
7639 .unused,
7640 .unused,
7641 .unused,
7642 .unused,
7643 .unused,
7644 .unused,
7645 .unused,
7646 .unused,
7647 },
7648 .dst_temps = .{ .mem, .unused },
7649 .clobbers = .{ .eflags = true },
7650 .each = .{ .once = &.{
7651 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7652 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7653 .{ ._, .p_d, .mull, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7654 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7655 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7656 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7657 } },
7658 }, .{
7659 .src_constraints = .{
7660 .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } },
7661 .{ .multiple_scalar_int = .{ .of = .dword, .is = .dword } },
7662 .any,
7663 },
7664 .patterns = &.{
7665 .{ .src = .{ .to_mem, .to_mem, .none } },
7666 },
7667 .extra_temps = .{
7668 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7669 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
7670 .unused,
7671 .unused,
7672 .unused,
7673 .unused,
7674 .unused,
7675 .unused,
7676 .unused,
7677 .unused,
7678 .unused,
7679 },
7680 .dst_temps = .{ .mem, .unused },
7681 .clobbers = .{ .eflags = true },
7682 .each = .{ .once = &.{
7683 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7684 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
7685 .{ ._, .i_, .mul, .tmp1d, .memia(.src1d, .tmp0, .add_unaligned_size), ._, ._ },
7686 .{ ._, ._, .mov, .memia(.dst0d, .tmp0, .add_unaligned_size), .tmp1d, ._, ._ },
7687 .{ ._, ._, .add, .tmp0p, .si(4), ._, ._ },
7688 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7689 } },
7690 }, .{
7691 .required_features = .{ .@"64bit", null, null, null },
7692 .src_constraints = .{
7693 .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } },
7694 .{ .multiple_scalar_int = .{ .of = .qword, .is = .qword } },
7695 .any,
7696 },
7697 .patterns = &.{
7698 .{ .src = .{ .to_mem, .to_mem, .none } },
7699 },
7700 .extra_temps = .{
7701 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7702 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7703 .unused,
7704 .unused,
7705 .unused,
7706 .unused,
7707 .unused,
7708 .unused,
7709 .unused,
7710 .unused,
7711 .unused,
7712 },
7713 .dst_temps = .{ .mem, .unused },
7714 .clobbers = .{ .eflags = true },
7715 .each = .{ .once = &.{
7716 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7717 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7718 .{ ._, .i_, .mul, .tmp1q, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
52197719 .{ ._, ._, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1q, ._, ._ },
52207720 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
52217721 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
52227722 } },
52237723 }, .{
5224 .required_features = .{ .@"64bit", null, null, null },
7724 .required_features = .{ .@"64bit", .bmi2, .adx, null },
52257725 .src_constraints = .{
52267726 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
52277727 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
......@@ -5236,33 +7736,46 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52367736 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
52377737 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
52387738 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7739 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7740 .{ .type = .isize, .kind = .{ .reg = .rcx } },
7741 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7742 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
52397743 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
5240 .unused,
5241 .unused,
5242 .unused,
5243 .unused,
52447744 .unused,
52457745 },
52467746 .dst_temps = .{ .mem, .unused },
7747 .clobbers = .{ .eflags = true },
52477748 .each = .{ .once = &.{
5248 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5249 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5250 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5251 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5252 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_8), ._, ._ },
5253 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5254 .{ .@"1:", ._, .mov, .tmp5q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
5255 .{ ._, ._, .sbb, .tmp5q, .leasi(.tmp2q, .@"8", .tmp4), ._, ._ },
5256 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp5q, ._, ._ },
5257 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5258 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
7749 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7750 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7751 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_size), ._, ._ },
7752 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7753 .{ ._, ._, .mov, .tmp4d, .sia(-8, .src0, .add_elem_size), ._, ._ },
7754 .{ .@"1:", ._, .xor, .tmp5d, .tmp5d, ._, ._ },
7755 .{ ._, ._, .@"or", .tmp5q, .leai(.tmp1q, .tmp4), ._, ._ },
7756 .{ ._, ._z, .j, .@"3f", ._, ._, ._ },
7757 .{ ._, ._, .lea, .tmp6p, .leaad(.tmp4, .sub_src0_elem_size, 8), ._, ._ },
7758 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7759 .{ .@"2:", ._x, .mul, .tmp9q, .tmp8q, .leai(.tmp2q, .tmp6), ._ },
7760 .{ ._, ._x, .adc, .tmp8q, .tmp7q, ._, ._ },
7761 .{ ._, ._, .mov, .leaiad(.tmp3q, .tmp6, .add_dst0_elem_size, -8), .tmp8q, ._, ._ },
7762 .{ ._, ._rcxz, .j, .@"2f", ._, ._, ._ },
7763 .{ ._, ._x, .ado, .tmp9q, .leaia(.tmp3q, .tmp6, .add_dst0_elem_size), ._, ._ },
7764 .{ ._, ._, .mov, .tmp7q, .tmp9q, ._, ._ },
7765 .{ ._, ._, .lea, .tmp6p, .lead(.tmp6, 8), ._, ._ },
7766 .{ ._, ._mp, .j, .@"2b", ._, ._, ._ },
7767 .{ .@"3:", ._, .mov, .leai(.tmp3q, .tmp4), .tmp5q, ._, ._ },
7768 .{ .@"2:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7769 .{ ._, ._, .sub, .tmp4d, .si(8), ._, ._ },
7770 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
52597771 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
52607772 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
52617773 } },
52627774 }, .{
7775 .required_features = .{ .@"64bit", .bmi2, .slow_incdec, null },
52637776 .src_constraints = .{
5264 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
5265 .{ .scalar_remainder_int = .{ .of = .dword, .is = .dword } },
7777 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7778 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
52667779 .any,
52677780 },
52687781 .patterns = &.{
......@@ -5274,26 +7787,205 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
52747787 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
52757788 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
52767789 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
5277 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },
5278 .unused,
5279 .unused,
5280 .unused,
5281 .unused,
5282 .unused,
7790 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7791 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7792 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7793 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7794 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7795 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
52837796 },
52847797 .dst_temps = .{ .mem, .unused },
7798 .clobbers = .{ .eflags = true },
52857799 .each = .{ .once = &.{
5286 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
5287 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5288 .{ ._, ._, .lea, .tmp2p, .memia(.src1, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5289 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
5290 .{ ._, ._, .mov, .tmp4p, .sa(.src0, .sub_elem_size_div_4), ._, ._ },
5291 .{ ._, ._c, .cl, ._, ._, ._, ._ },
5292 .{ .@"1:", ._, .mov, .tmp5d, .leasi(.tmp1d, .@"4", .tmp4), ._, ._ },
5293 .{ ._, ._, .sbb, .tmp5d, .leasi(.tmp2d, .@"4", .tmp4), ._, ._ },
5294 .{ ._, ._, .mov, .leasi(.tmp3d, .@"4", .tmp4), .tmp5d, ._, ._ },
5295 .{ ._, ._c, .in, .tmp4p, ._, ._, ._ },
5296 .{ ._, ._nz, .j, .@"1b", ._, ._, ._ },
7800 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7801 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7802 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7803 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7804 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7805 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7806 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7807 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7808 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7809 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7810 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7811 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7812 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7813 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7814 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7815 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7816 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7817 .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ },
7818 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7819 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7820 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7821 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7822 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7823 .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ },
7824 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
7825 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7826 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7827 } },
7828 }, .{
7829 .required_features = .{ .@"64bit", .bmi2, null, null },
7830 .src_constraints = .{
7831 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7832 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7833 .any,
7834 },
7835 .patterns = &.{
7836 .{ .src = .{ .to_mem, .to_mem, .none } },
7837 },
7838 .extra_temps = .{
7839 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7840 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7841 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7842 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7843 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7844 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7845 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7846 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7847 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7848 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7849 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7850 },
7851 .dst_temps = .{ .mem, .unused },
7852 .clobbers = .{ .eflags = true },
7853 .each = .{ .once = &.{
7854 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7855 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7856 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7857 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7858 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7859 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7860 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7861 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7862 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7863 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7864 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7865 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7866 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7867 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7868 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7869 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7870 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7871 .{ .@"3:", ._x, .mul, .tmp10q, .tmp9q, .leasi(.tmp2q, .@"8", .tmp5), ._ },
7872 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7873 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7874 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7875 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7876 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7877 .{ ._, ._c, .de, .tmp4d, ._, ._, ._ },
7878 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
7879 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7880 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7881 } },
7882 }, .{
7883 .required_features = .{ .@"64bit", .slow_incdec, null, null },
7884 .src_constraints = .{
7885 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7886 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7887 .any,
7888 },
7889 .patterns = &.{
7890 .{ .src = .{ .to_mem, .to_mem, .none } },
7891 },
7892 .extra_temps = .{
7893 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7894 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7895 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7896 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7897 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7898 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7899 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7900 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7901 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7902 .{ .type = .u64, .kind = .{ .reg = .rax } },
7903 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7904 },
7905 .dst_temps = .{ .mem, .unused },
7906 .clobbers = .{ .eflags = true },
7907 .each = .{ .once = &.{
7908 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7909 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7910 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7911 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7912 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7913 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7914 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7915 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7916 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7917 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7918 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7919 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7920 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7921 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7922 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7923 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7924 .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ },
7925 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ },
7926 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7927 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7928 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7929 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7930 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7931 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7932 .{ ._, ._, .sub, .tmp4d, .si(1), ._, ._ },
7933 .{ ._, ._ae, .j, .@"1b", ._, ._, ._ },
7934 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
7935 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
7936 } },
7937 }, .{
7938 .required_features = .{ .@"64bit", null, null, null },
7939 .src_constraints = .{
7940 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7941 .{ .scalar_remainder_int = .{ .of = .qword, .is = .qword } },
7942 .any,
7943 },
7944 .patterns = &.{
7945 .{ .src = .{ .to_mem, .to_mem, .none } },
7946 },
7947 .extra_temps = .{
7948 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7949 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7950 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7951 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
7952 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7953 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7954 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7955 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },
7956 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
7957 .{ .type = .u64, .kind = .{ .reg = .rax } },
7958 .{ .type = .u64, .kind = .{ .reg = .rdx } },
7959 },
7960 .dst_temps = .{ .mem, .unused },
7961 .clobbers = .{ .eflags = true },
7962 .each = .{ .once = &.{
7963 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size), ._, ._ },
7964 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_size), ._, ._ },
7965 .{ ._, ._, .lea, .tmp2p, .memiad(.src1, .tmp0, .add_size, 8), ._, ._ },
7966 .{ ._, ._, .lea, .tmp3p, .memia(.dst0, .tmp0, .add_size), ._, ._ },
7967 .{ ._, ._, .mov, .tmp4d, .sia(-1, .src0, .add_elem_size_div_8), ._, ._ },
7968 .{ .@"1:", ._, .lea, .tmp5p, .leaa(.tmp4, .sub_src0_elem_size_div_8), ._, ._ },
7969 .{ ._, ._, .xor, .tmp6d, .tmp6d, ._, ._ },
7970 .{ ._, ._, .xor, .tmp7d, .tmp7d, ._, ._ },
7971 .{ ._, ._, .xor, .tmp8d, .tmp8d, ._, ._ },
7972 .{ ._, ._, .@"or", .tmp6q, .leasi(.tmp1q, .@"8", .tmp4), ._, ._ },
7973 .{ ._, ._nz, .j, .@"3f", ._, ._, ._ },
7974 .{ ._, ._, .mov, .leasi(.tmp3q, .@"8", .tmp4), .tmp6q, ._, ._ },
7975 .{ ._, ._mp, .j, .@"4f", ._, ._, ._ },
7976 .{ .@"2:", ._, .adc, .tmp10q, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), ._, ._ },
7977 .{ ._, ._, .adc, .tmp7b, .si(0), ._, ._ },
7978 .{ ._, ._, .mov, .tmp8q, .tmp10q, ._, ._ },
7979 .{ .@"3:", ._, .mov, .tmp9q, .tmp6q, ._, ._ },
7980 .{ ._, ._, .mul, .leasi(.tmp2q, .@"8", .tmp5), ._, ._, ._ },
7981 .{ ._, ._l, .sh, .tmp7b, .ui(4), ._, ._ },
7982 .{ ._, ._, .adc, .tmp9q, .tmp8q, ._, ._ },
7983 .{ ._, ._, .mov, .leasia(.tmp3q, .@"8", .tmp5, .add_src0_elem_size), .tmp9q, ._, ._ },
7984 .{ ._, ._c, .in, .tmp5p, ._, ._, ._ },
7985 .{ ._, ._nz, .j, .@"2b", ._, ._, ._ },
7986 .{ .@"4:", ._, .lea, .tmp2p, .lead(.tmp2, 8), ._, ._ },
7987 .{ ._, ._c, .de, .tmp4d, ._, ._, ._ },
7988 .{ ._, ._ns, .j, .@"1b", ._, ._, ._ },
52977989 .{ ._, ._, .add, .tmp0p, .sa(.src0, .add_elem_size), ._, ._ },
52987990 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
52997991 } },
......@@ -5324,7 +8016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53248016 .each = .{ .once = &.{
53258017 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
53268018 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5327 .{ ._, .v_ss, .sub, .dst0x, .dst0x, .tmp0d, ._ },
8019 .{ ._, .v_ss, .mul, .dst0x, .dst0x, .tmp0d, ._ },
53288020 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
53298021 } },
53308022 }, .{
......@@ -5339,7 +8031,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53398031 },
53408032 .call_frame = .{ .alignment = .@"16" },
53418033 .extra_temps = .{
5342 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
8034 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
53438035 .unused,
53448036 .unused,
53458037 .unused,
......@@ -5386,7 +8078,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
53868078 .each = .{ .once = &.{
53878079 .{ ._, .v_ps, .cvtph2, .dst0x, .src0q, ._, ._ },
53888080 .{ ._, .v_ps, .cvtph2, .tmp0x, .src1q, ._, ._ },
5389 .{ ._, .v_ps, .sub, .dst0x, .dst0x, .tmp0x, ._ },
8081 .{ ._, .v_ps, .mul, .dst0x, .dst0x, .tmp0x, ._ },
53908082 .{ ._, .v_, .cvtps2ph, .dst0q, .dst0x, .rm(.{}), ._ },
53918083 } },
53928084 }, .{
......@@ -5419,7 +8111,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54198111 .each = .{ .once = &.{
54208112 .{ ._, .v_ps, .cvtph2, .dst0y, .src0x, ._, ._ },
54218113 .{ ._, .v_ps, .cvtph2, .tmp0y, .src1x, ._, ._ },
5422 .{ ._, .v_ps, .sub, .dst0y, .dst0y, .tmp0y, ._ },
8114 .{ ._, .v_ps, .mul, .dst0y, .dst0y, .tmp0y, ._ },
54238115 .{ ._, .v_, .cvtps2ph, .dst0x, .dst0y, .rm(.{}), ._ },
54248116 } },
54258117 }, .{
......@@ -5451,7 +8143,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54518143 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
54528144 .{ .@"0:", .v_ps, .cvtph2, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
54538145 .{ ._, .v_ps, .cvtph2, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
5454 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .tmp2y, ._ },
8146 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .tmp2y, ._ },
54558147 .{ ._, .v_, .cvtps2ph, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1y, .rm(.{}), ._ },
54568148 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
54578149 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5471,7 +8163,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
54718163 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
54728164 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
54738165 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5474 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
8166 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
54758167 .unused,
54768168 .unused,
54778169 .unused,
......@@ -5507,7 +8199,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55078199 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
55088200 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
55098201 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5510 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
8202 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
55118203 .unused,
55128204 .unused,
55138205 .unused,
......@@ -5544,7 +8236,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55448236 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
55458237 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
55468238 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5547 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
8239 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
55488240 .{ .type = .f16, .kind = .{ .reg = .ax } },
55498241 .unused,
55508242 .unused,
......@@ -5584,7 +8276,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
55848276 .{ .type = .f32, .kind = .mem },
55858277 .{ .type = .f16, .kind = .{ .reg = .xmm0 } },
55868278 .{ .type = .f16, .kind = .{ .reg = .xmm1 } },
5587 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subhf3" } } },
8279 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__mulhf3" } } },
55888280 .unused,
55898281 .unused,
55908282 .unused,
......@@ -5617,11 +8309,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56178309 },
56188310 .patterns = &.{
56198311 .{ .src = .{ .to_sse, .mem, .none } },
8312 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
56208313 .{ .src = .{ .to_sse, .to_sse, .none } },
56218314 },
56228315 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
56238316 .each = .{ .once = &.{
5624 .{ ._, .v_ss, .sub, .dst0x, .src0x, .src1d, ._ },
8317 .{ ._, .v_ss, .mul, .dst0x, .src0x, .src1d, ._ },
56258318 } },
56268319 }, .{
56278320 .required_features = .{ .sse, null, null, null },
......@@ -5632,11 +8325,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56328325 },
56338326 .patterns = &.{
56348327 .{ .src = .{ .to_mut_sse, .mem, .none } },
8328 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
56358329 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
56368330 },
56378331 .dst_temps = .{ .{ .ref = .src0 }, .unused },
56388332 .each = .{ .once = &.{
5639 .{ ._, ._ss, .sub, .dst0x, .src1d, ._, ._ },
8333 .{ ._, ._ss, .mul, .dst0x, .src1d, ._, ._ },
56408334 } },
56418335 }, .{
56428336 .required_features = .{ .avx, null, null, null },
......@@ -5647,11 +8341,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56478341 },
56488342 .patterns = &.{
56498343 .{ .src = .{ .to_sse, .mem, .none } },
8344 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
56508345 .{ .src = .{ .to_sse, .to_sse, .none } },
56518346 },
56528347 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
56538348 .each = .{ .once = &.{
5654 .{ ._, .v_ps, .sub, .dst0x, .src0x, .src1x, ._ },
8349 .{ ._, .v_ps, .mul, .dst0x, .src0x, .src1x, ._ },
56558350 } },
56568351 }, .{
56578352 .required_features = .{ .sse, null, null, null },
......@@ -5662,11 +8357,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56628357 },
56638358 .patterns = &.{
56648359 .{ .src = .{ .to_mut_sse, .mem, .none } },
8360 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
56658361 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
56668362 },
56678363 .dst_temps = .{ .{ .ref = .src0 }, .unused },
56688364 .each = .{ .once = &.{
5669 .{ ._, ._ps, .sub, .dst0x, .src1x, ._, ._ },
8365 .{ ._, ._ps, .mul, .dst0x, .src1x, ._, ._ },
56708366 } },
56718367 }, .{
56728368 .required_features = .{ .avx, null, null, null },
......@@ -5677,11 +8373,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
56778373 },
56788374 .patterns = &.{
56798375 .{ .src = .{ .to_sse, .mem, .none } },
8376 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
56808377 .{ .src = .{ .to_sse, .to_sse, .none } },
56818378 },
56828379 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
56838380 .each = .{ .once = &.{
5684 .{ ._, .v_ps, .sub, .dst0y, .src0y, .src1y, ._ },
8381 .{ ._, .v_ps, .mul, .dst0y, .src0y, .src1y, ._ },
56858382 } },
56868383 }, .{
56878384 .required_features = .{ .avx, null, null, null },
......@@ -5711,7 +8408,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57118408 .each = .{ .once = &.{
57128409 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
57138410 .{ .@"0:", .v_ps, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5714 .{ ._, .v_ps, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
8411 .{ ._, .v_ps, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
57158412 .{ ._, .v_ps, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
57168413 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
57178414 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5744,7 +8441,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57448441 .each = .{ .once = &.{
57458442 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
57468443 .{ .@"0:", ._ps, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5747 .{ ._, ._ps, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
8444 .{ ._, ._ps, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
57488445 .{ ._, ._ps, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
57498446 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
57508447 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5758,11 +8455,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57588455 },
57598456 .patterns = &.{
57608457 .{ .src = .{ .to_sse, .mem, .none } },
8458 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
57618459 .{ .src = .{ .to_sse, .to_sse, .none } },
57628460 },
57638461 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
57648462 .each = .{ .once = &.{
5765 .{ ._, .v_sd, .sub, .dst0x, .src0x, .src1q, ._ },
8463 .{ ._, .v_sd, .mul, .dst0x, .src0x, .src1q, ._ },
57668464 } },
57678465 }, .{
57688466 .required_features = .{ .sse2, null, null, null },
......@@ -5773,11 +8471,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57738471 },
57748472 .patterns = &.{
57758473 .{ .src = .{ .to_mut_sse, .mem, .none } },
8474 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
57768475 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
57778476 },
57788477 .dst_temps = .{ .{ .ref = .src0 }, .unused },
57798478 .each = .{ .once = &.{
5780 .{ ._, ._sd, .sub, .dst0x, .src1q, ._, ._ },
8479 .{ ._, ._sd, .mul, .dst0x, .src1q, ._, ._ },
57818480 } },
57828481 }, .{
57838482 .required_features = .{ .x87, null, null, null },
......@@ -5787,7 +8486,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
57878486 .any,
57888487 },
57898488 .patterns = &.{
5790 .{ .src = .{ .to_mem, .to_mem, .none } },
8489 .{ .src = .{ .mem, .mem, .none } },
57918490 },
57928491 .extra_temps = .{
57938492 .{ .type = .f64, .kind = .{ .reg = .st6 } },
......@@ -5805,7 +8504,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
58058504 .dst_temps = .{ .mem, .unused },
58068505 .each = .{ .once = &.{
58078506 .{ ._, .f_, .ld, .src0q, ._, ._, ._ },
5808 .{ ._, .f_, .sub, .src1q, ._, ._, ._ },
8507 .{ ._, .f_, .mul, .src1q, ._, ._, ._ },
58098508 .{ ._, .f_p, .st, .dst0q, ._, ._, ._ },
58108509 } },
58118510 }, .{
......@@ -5817,11 +8516,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
58178516 },
58188517 .patterns = &.{
58198518 .{ .src = .{ .to_sse, .mem, .none } },
8519 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
58208520 .{ .src = .{ .to_sse, .to_sse, .none } },
58218521 },
58228522 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
58238523 .each = .{ .once = &.{
5824 .{ ._, .v_pd, .sub, .dst0x, .src0x, .src1x, ._ },
8524 .{ ._, .v_pd, .mul, .dst0x, .src0x, .src1x, ._ },
58258525 } },
58268526 }, .{
58278527 .required_features = .{ .sse2, null, null, null },
......@@ -5832,11 +8532,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
58328532 },
58338533 .patterns = &.{
58348534 .{ .src = .{ .to_mut_sse, .mem, .none } },
8535 .{ .src = .{ .mem, .to_mut_sse, .none }, .commute = .{ 0, 1 } },
58358536 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
58368537 },
58378538 .dst_temps = .{ .{ .ref = .src0 }, .unused },
58388539 .each = .{ .once = &.{
5839 .{ ._, ._pd, .sub, .dst0x, .src1x, ._, ._ },
8540 .{ ._, ._pd, .mul, .dst0x, .src1x, ._, ._ },
58408541 } },
58418542 }, .{
58428543 .required_features = .{ .avx, null, null, null },
......@@ -5847,11 +8548,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
58478548 },
58488549 .patterns = &.{
58498550 .{ .src = .{ .to_sse, .mem, .none } },
8551 .{ .src = .{ .mem, .to_sse, .none }, .commute = .{ 0, 1 } },
58508552 .{ .src = .{ .to_sse, .to_sse, .none } },
58518553 },
58528554 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
58538555 .each = .{ .once = &.{
5854 .{ ._, .v_pd, .sub, .dst0y, .src0y, .src1y, ._ },
8556 .{ ._, .v_pd, .mul, .dst0y, .src0y, .src1y, ._ },
58558557 } },
58568558 }, .{
58578559 .required_features = .{ .avx, null, null, null },
......@@ -5881,7 +8583,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
58818583 .each = .{ .once = &.{
58828584 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
58838585 .{ .@"0:", .v_pd, .mova, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
5884 .{ ._, .v_pd, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
8586 .{ ._, .v_pd, .mul, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },
58858587 .{ ._, .v_pd, .mova, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },
58868588 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },
58878589 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5914,7 +8616,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
59148616 .each = .{ .once = &.{
59158617 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
59168618 .{ .@"0:", ._pd, .mova, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
5917 .{ ._, ._pd, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
8619 .{ ._, ._pd, .mul, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
59188620 .{ ._, ._pd, .mova, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
59198621 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
59208622 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5946,7 +8648,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
59468648 .each = .{ .once = &.{
59478649 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
59488650 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
5949 .{ ._, .f_, .sub, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
8651 .{ ._, .f_, .mul, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._, ._ },
59508652 .{ ._, .f_p, .st, .memia(.dst0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
59518653 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
59528654 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -5978,7 +8680,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
59788680 .each = .{ .once = &.{
59798681 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
59808682 .{ ._, .f_, .ld, .src1t, ._, ._, ._ },
5981 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
8683 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
59828684 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
59838685 } },
59848686 }, .{
......@@ -5990,34 +8692,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
59908692 },
59918693 .patterns = &.{
59928694 .{ .src = .{ .to_x87, .mem, .none }, .commute = .{ 0, 1 } },
5993 },
5994 .extra_temps = .{
5995 .{ .type = .f80, .kind = .{ .reg = .st7 } },
5996 .unused,
5997 .unused,
5998 .unused,
5999 .unused,
6000 .unused,
6001 .unused,
6002 .unused,
6003 .unused,
6004 .unused,
6005 .unused,
6006 },
6007 .dst_temps = .{ .{ .rc = .x87 }, .unused },
6008 .each = .{ .once = &.{
6009 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6010 .{ ._, .f_, .subr, .tmp0t, .src1t, ._, ._ },
6011 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
6012 } },
6013 }, .{
6014 .required_features = .{ .x87, null, null, null },
6015 .src_constraints = .{
6016 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6017 .{ .scalar_float = .{ .of = .xword, .is = .tbyte } },
6018 .any,
6019 },
6020 .patterns = &.{
60218695 .{ .src = .{ .mem, .to_x87, .none } },
60228696 .{ .src = .{ .to_x87, .to_x87, .none } },
60238697 },
......@@ -6037,7 +8711,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
60378711 .dst_temps = .{ .{ .rc = .x87 }, .unused },
60388712 .each = .{ .once = &.{
60398713 .{ ._, .f_, .ld, .src0t, ._, ._, ._ },
6040 .{ ._, .f_, .sub, .tmp0t, .src1t, ._, ._ },
8714 .{ ._, .f_, .mul, .tmp0t, .src1t, ._, ._ },
60418715 .{ ._, .f_p, .st, .dst0t, ._, ._, ._ },
60428716 } },
60438717 }, .{
......@@ -6068,7 +8742,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
60688742 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
60698743 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
60708744 .{ ._, .f_, .ld, .memia(.src1t, .tmp0, .add_unaligned_size), ._, ._, ._ },
6071 .{ ._, .f_p, .sub, ._, ._, ._, ._ },
8745 .{ ._, .f_p, .mul, ._, ._, ._, ._ },
60728746 .{ ._, .f_p, .st, .memia(.dst0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
60738747 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
60748748 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
......@@ -6085,7 +8759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
60858759 },
60868760 .call_frame = .{ .alignment = .@"16" },
60878761 .extra_temps = .{
6088 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
8762 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
60898763 .unused,
60908764 .unused,
60918765 .unused,
......@@ -6117,7 +8791,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
61178791 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
61188792 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
61198793 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6120 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
8794 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
61218795 .unused,
61228796 .unused,
61238797 .unused,
......@@ -6152,7 +8826,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
61528826 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
61538827 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
61548828 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6155 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
8829 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
61568830 .unused,
61578831 .unused,
61588832 .unused,
......@@ -6187,7 +8861,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
61878861 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
61888862 .{ .type = .f128, .kind = .{ .reg = .xmm0 } },
61898863 .{ .type = .f128, .kind = .{ .reg = .xmm1 } },
6190 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__subtf3" } } },
8864 .{ .type = .usize, .kind = .{ .symbol = &.{ .name = "__multf3" } } },
61918865 .unused,
61928866 .unused,
61938867 .unused,
......@@ -6210,28 +8884,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
62108884 } }) catch |err| switch (err) {
62118885 error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{
62128886 @tagName(air_tag),
6213 cg.typeOf(bin_op.lhs).fmt(pt),
8887 ty.fmt(pt),
62148888 ops[0].tracking(cg),
62158889 ops[1].tracking(cg),
62168890 }),
62178891 else => |e| return e,
62188892 };
6219 switch (air_tag) {
6220 else => unreachable,
6221 .sub, .sub_optimized => {},
6222 .sub_wrap => res[0].wrapInt(cg) catch |err| switch (err) {
6223 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
6224 cg.typeOf(bin_op.lhs).fmt(pt),
6225 ops[0].tracking(cg),
6226 ops[1].tracking(cg),
6227 }),
6228 else => |e| return e,
6229 },
6230 }
62318893 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
62328894 },
6233 .sub_safe => unreachable,
6234 .mul, .mul_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, .mul) else {
8895 .mul_safe => unreachable,
8896 .mul_wrap => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, air_tag) else {
62358897 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
62368898 const ty = cg.typeOf(bin_op.lhs);
62378899 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
......@@ -6628,6 +9290,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
66289290 .{ .src = .{ .to_sse, .to_sse, .none } },
66299291 },
66309292 .extra_temps = .{
9293 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9294 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
66319295 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
66329296 .unused,
66339297 .unused,
......@@ -6637,15 +9301,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
66379301 .unused,
66389302 .unused,
66399303 .unused,
6640 .unused,
6641 .unused,
66429304 },
66439305 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
66449306 .each = .{ .once = &.{
9307 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
66459308 .{ ._, .vp_w, .movsxb, .dst0x, .src0q, ._, ._ },
6646 .{ ._, .vp_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6647 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6648 .{ ._, .vp_b, .ackssw, .dst0x, .dst0x, .dst0x, ._ },
9309 .{ ._, .vp_w, .movsxb, .tmp2x, .src1q, ._, ._ },
9310 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
9311 .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ },
66499312 } },
66509313 }, .{
66519314 .required_features = .{ .sse4_1, null, null, null },
......@@ -6661,6 +9324,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
66619324 .{ .src = .{ .to_sse, .to_sse, .none } },
66629325 },
66639326 .extra_temps = .{
9327 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9328 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
66649329 .{ .type = .vector_8_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
66659330 .unused,
66669331 .unused,
......@@ -6670,29 +9335,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
66709335 .unused,
66719336 .unused,
66729337 .unused,
6673 .unused,
6674 .unused,
66759338 },
66769339 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
66779340 .each = .{ .once = &.{
9341 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
66789342 .{ ._, .p_w, .movsxb, .dst0x, .src0q, ._, ._ },
6679 .{ ._, .p_w, .movsxb, .tmp0x, .src1q, ._, ._ },
6680 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6681 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
9343 .{ ._, .p_w, .movsxb, .tmp2x, .src1q, ._, ._ },
9344 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
9345 .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ },
66829346 } },
66839347 }, .{
66849348 .required_features = .{ .sse2, null, null, null },
66859349 .src_constraints = .{
6686 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
6687 .{ .scalar_signed_int = .{ .of = .qword, .is = .byte } },
9350 .{ .scalar_int = .{ .of = .qword, .is = .byte } },
9351 .{ .scalar_int = .{ .of = .qword, .is = .byte } },
66889352 .any,
66899353 },
66909354 .patterns = &.{
66919355 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
66929356 },
66939357 .extra_temps = .{
6694 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
6695 .unused,
9358 .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
9359 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
66969360 .unused,
66979361 .unused,
66989362 .unused,
......@@ -6705,14 +9369,12 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67059369 },
67069370 .dst_temps = .{ .{ .ref = .src0 }, .unused },
67079371 .each = .{ .once = &.{
6708 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6709 .{ ._, .p_b, .cmpgt, .tmp0x, .src0x, ._, ._ },
6710 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6711 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6712 .{ ._, .p_b, .cmpgt, .tmp0x, .src1x, ._, ._ },
6713 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
9372 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
9373 .{ ._, .p_, .unpcklbw, .dst0x, .dst0x, ._, ._ },
9374 .{ ._, .p_, .unpcklbw, .src1x, .src1x, ._, ._ },
67149375 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6715 .{ ._, .p_b, .ackssw, .dst0x, .dst0x, ._, ._ },
9376 .{ ._, .p_, .@"and", .dst0x, .lea(.tmp1x), ._, ._ },
9377 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
67169378 } },
67179379 }, .{
67189380 .required_features = .{ .avx2, null, null, null },
......@@ -6728,6 +9390,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67289390 .{ .src = .{ .to_sse, .to_sse, .none } },
67299391 },
67309392 .extra_temps = .{
9393 .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9394 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
67319395 .{ .type = .vector_16_i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
67329396 .unused,
67339397 .unused,
......@@ -6737,15 +9401,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67379401 .unused,
67389402 .unused,
67399403 .unused,
6740 .unused,
6741 .unused,
67429404 },
67439405 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
67449406 .each = .{ .once = &.{
9407 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
67459408 .{ ._, .vp_w, .movsxb, .dst0y, .src0x, ._, ._ },
6746 .{ ._, .vp_w, .movsxb, .tmp0y, .src1x, ._, ._ },
6747 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6748 .{ ._, .vp_b, .ackssw, .dst0y, .dst0y, .dst0y, ._ },
9409 .{ ._, .vp_w, .movsxb, .tmp2y, .src1x, ._, ._ },
9410 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
9411 .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ },
67499412 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
67509413 } },
67519414 }, .{
......@@ -6762,6 +9425,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67629425 .{ .src = .{ .to_sse, .to_sse, .none } },
67639426 },
67649427 .extra_temps = .{
9428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9429 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
67659430 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
67669431 .unused,
67679432 .unused,
......@@ -6771,15 +9436,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67719436 .unused,
67729437 .unused,
67739438 .unused,
6774 .unused,
6775 .unused,
67769439 },
67779440 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
67789441 .each = .{ .once = &.{
9442 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
67799443 .{ ._, .vp_w, .movzxb, .dst0x, .src0q, ._, ._ },
6780 .{ ._, .vp_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6781 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp0x, ._ },
6782 .{ ._, .vp_b, .ackusw, .dst0x, .dst0x, .dst0x, ._ },
9444 .{ ._, .vp_w, .movzxb, .tmp2x, .src1q, ._, ._ },
9445 .{ ._, .vp_w, .mull, .dst0x, .dst0x, .tmp2x, ._ },
9446 .{ ._, .vp_b, .shuf, .dst0x, .dst0x, .lea(.tmp1x), ._ },
67839447 } },
67849448 }, .{
67859449 .required_features = .{ .sse4_1, null, null, null },
......@@ -6795,6 +9459,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
67959459 .{ .src = .{ .to_sse, .to_sse, .none } },
67969460 },
67979461 .extra_temps = .{
9462 .{ .type = .vector_16_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9463 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
67989464 .{ .type = .vector_8_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
67999465 .unused,
68009466 .unused,
......@@ -6804,46 +9470,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
68049470 .unused,
68059471 .unused,
68069472 .unused,
6807 .unused,
6808 .unused,
68099473 },
68109474 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
68119475 .each = .{ .once = &.{
9476 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
68129477 .{ ._, .p_w, .movzxb, .dst0x, .src0q, ._, ._ },
6813 .{ ._, .p_w, .movzxb, .tmp0x, .src1q, ._, ._ },
6814 .{ ._, .p_w, .mull, .dst0x, .tmp0x, ._, ._ },
6815 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
6816 } },
6817 }, .{
6818 .required_features = .{ .sse2, null, null, null },
6819 .src_constraints = .{
6820 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6821 .{ .scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
6822 .any,
6823 },
6824 .patterns = &.{
6825 .{ .src = .{ .to_mut_sse, .to_mut_sse, .none } },
6826 },
6827 .extra_temps = .{
6828 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
6829 .unused,
6830 .unused,
6831 .unused,
6832 .unused,
6833 .unused,
6834 .unused,
6835 .unused,
6836 .unused,
6837 .unused,
6838 .unused,
6839 },
6840 .dst_temps = .{ .{ .ref = .src0 }, .unused },
6841 .each = .{ .once = &.{
6842 .{ ._, .p_, .xor, .tmp0x, .tmp0x, ._, ._ },
6843 .{ ._, .p_, .unpcklbw, .dst0x, .tmp0x, ._, ._ },
6844 .{ ._, .p_, .unpcklbw, .src1x, .tmp0x, ._, ._ },
6845 .{ ._, .p_w, .mull, .dst0x, .src1x, ._, ._ },
6846 .{ ._, .p_b, .ackusw, .dst0x, .dst0x, ._, ._ },
9478 .{ ._, .p_w, .movzxb, .tmp2x, .src1q, ._, ._ },
9479 .{ ._, .p_w, .mull, .dst0x, .tmp2x, ._, ._ },
9480 .{ ._, .p_b, .shuf, .dst0x, .lea(.tmp1x), ._, ._ },
68479481 } },
68489482 }, .{
68499483 .required_features = .{ .avx2, null, null, null },
......@@ -6859,6 +9493,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
68599493 .{ .src = .{ .to_sse, .to_sse, .none } },
68609494 },
68619495 .extra_temps = .{
9496 .{ .type = .vector_32_u8, .kind = .{ .pshufb_trunc_mem = .{ .from = .word, .to = .byte } } },
9497 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
68629498 .{ .type = .vector_16_u16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .sse } } },
68639499 .unused,
68649500 .unused,
......@@ -6868,15 +9504,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
68689504 .unused,
68699505 .unused,
68709506 .unused,
6871 .unused,
6872 .unused,
68739507 },
68749508 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
68759509 .each = .{ .once = &.{
9510 .{ ._, ._, .lea, .tmp1p, .mem(.tmp0), ._, ._ },
68769511 .{ ._, .vp_w, .movzxb, .dst0y, .src0x, ._, ._ },
6877 .{ ._, .vp_w, .movzxb, .tmp0y, .src1x, ._, ._ },
6878 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp0y, ._ },
6879 .{ ._, .vp_b, .ackusw, .dst0y, .dst0y, .dst0y, ._ },
9512 .{ ._, .vp_w, .movzxb, .tmp2y, .src1x, ._, ._ },
9513 .{ ._, .vp_w, .mull, .dst0y, .dst0y, .tmp2y, ._ },
9514 .{ ._, .vp_b, .shuf, .dst0y, .dst0y, .lea(.tmp1y), ._ },
68809515 .{ ._, .v_q, .perm, .dst0y, .dst0y, .ui(0b10_00_10_00), ._ },
68819516 } },
68829517 }, .{
......@@ -6891,6 +9526,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
68919526 },
68929527 .extra_temps = .{
68939528 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9529 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9530 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
68949531 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
68959532 .{ .type = .vector_16_i16, .kind = .{ .rc = .sse } },
68969533 .unused,
......@@ -6899,19 +9536,19 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
68999536 .unused,
69009537 .unused,
69019538 .unused,
6902 .unused,
6903 .unused,
69049539 },
69059540 .dst_temps = .{ .mem, .unused },
69069541 .clobbers = .{ .eflags = true },
69079542 .each = .{ .once = &.{
9543 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9544 .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ },
69089545 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6909 .{ .@"0:", .vp_w, .movsxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
6910 .{ ._, .vp_w, .movsxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
6911 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
6912 .{ ._, .vp_b, .ackssw, .tmp1y, .tmp1y, .tmp1y, ._ },
6913 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
6914 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
9546 .{ .@"0:", .vp_w, .movsxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
9547 .{ ._, .vp_w, .movsxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
9548 .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ },
9549 .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ },
9550 .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ },
9551 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
69159552 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
69169553 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
69179554 } },
......@@ -6927,6 +9564,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
69279564 },
69289565 .extra_temps = .{
69299566 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9567 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9568 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
69309569 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
69319570 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
69329571 .unused,
......@@ -6935,18 +9574,18 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
69359574 .unused,
69369575 .unused,
69379576 .unused,
6938 .unused,
6939 .unused,
69409577 },
69419578 .dst_temps = .{ .mem, .unused },
69429579 .clobbers = .{ .eflags = true },
69439580 .each = .{ .once = &.{
9581 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9582 .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
69449583 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6945 .{ .@"0:", .vp_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6946 .{ ._, .vp_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6947 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
6948 .{ ._, .vp_b, .ackssw, .tmp1x, .tmp1x, .tmp1x, ._ },
6949 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
9584 .{ .@"0:", .vp_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9585 .{ ._, .vp_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9586 .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ },
9587 .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ },
9588 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
69509589 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
69519590 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
69529591 } },
......@@ -6962,6 +9601,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
69629601 },
69639602 .extra_temps = .{
69649603 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9604 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9605 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
69659606 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
69669607 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
69679608 .unused,
......@@ -6970,26 +9611,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
69709611 .unused,
69719612 .unused,
69729613 .unused,
6973 .unused,
6974 .unused,
69759614 },
69769615 .dst_temps = .{ .mem, .unused },
69779616 .clobbers = .{ .eflags = true },
69789617 .each = .{ .once = &.{
9618 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9619 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
69799620 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
6980 .{ .@"0:", .p_w, .movsxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
6981 .{ ._, .p_w, .movsxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
6982 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
6983 .{ ._, .p_b, .ackssw, .tmp1x, .tmp1x, ._, ._ },
6984 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
9621 .{ .@"0:", .p_w, .movsxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9622 .{ ._, .p_w, .movsxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9623 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9624 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
9625 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
69859626 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
69869627 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
69879628 } },
69889629 }, .{
6989 .required_features = .{ .sse2, null, null, null },
9630 .required_features = .{ .ssse3, null, null, null },
69909631 .src_constraints = .{
6991 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
6992 .{ .multiple_scalar_signed_int = .{ .of = .qword, .is = .byte } },
9632 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9633 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
69939634 .any,
69949635 },
69959636 .patterns = &.{
......@@ -6997,10 +9638,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
69979638 },
69989639 .extra_temps = .{
69999640 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7000 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
7001 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
7002 .{ .type = .vector_8_i16, .kind = .{ .rc = .sse } },
7003 .unused,
9641 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9642 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
9643 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9644 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
70049645 .unused,
70059646 .unused,
70069647 .unused,
......@@ -7011,26 +9652,24 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
70119652 .dst_temps = .{ .mem, .unused },
70129653 .clobbers = .{ .eflags = true },
70139654 .each = .{ .once = &.{
9655 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9656 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
70149657 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7015 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7016 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7017 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp2x, ._, ._ },
7018 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7019 .{ ._, .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7020 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7021 .{ ._, .p_b, .cmpgt, .tmp1x, .tmp3x, ._, ._ },
7022 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7023 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7024 .{ ._, .p_b, .ackssw, .tmp2x, .tmp2x, ._, ._ },
7025 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
9658 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9659 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
9660 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9661 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
9662 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9663 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
9664 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
70269665 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
70279666 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
70289667 } },
70299668 }, .{
7030 .required_features = .{ .slow_incdec, null, null, null },
9669 .required_features = .{ .sse3, null, null, null },
70319670 .src_constraints = .{
7032 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
7033 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
9671 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9672 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
70349673 .any,
70359674 },
70369675 .patterns = &.{
......@@ -7038,13 +9677,53 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
70389677 },
70399678 .extra_temps = .{
70409679 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7041 .{ .type = .i8, .kind = .{ .reg = .al } },
9680 .{ .type = .vector_8_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
9681 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
9682 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9683 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
70429684 .unused,
70439685 .unused,
70449686 .unused,
70459687 .unused,
70469688 .unused,
70479689 .unused,
9690 },
9691 .dst_temps = .{ .mem, .unused },
9692 .clobbers = .{ .eflags = true },
9693 .each = .{ .once = &.{
9694 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9695 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
9696 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
9697 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9698 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
9699 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9700 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
9701 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9702 .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ },
9703 .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ },
9704 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9705 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
9706 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9707 } },
9708 }, .{
9709 .required_features = .{ .sse2, null, null, null },
9710 .src_constraints = .{
9711 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9712 .{ .multiple_scalar_int = .{ .of = .qword, .is = .byte } },
9713 .any,
9714 },
9715 .patterns = &.{
9716 .{ .src = .{ .to_mem, .to_mem, .none } },
9717 },
9718 .extra_temps = .{
9719 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9720 .{ .type = .vector_16_u8, .kind = .{ .pand_trunc_mem = .{ .from = .word, .to = .byte } } },
9721 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
9722 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9723 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
9724 .unused,
9725 .unused,
9726 .unused,
70489727 .unused,
70499728 .unused,
70509729 .unused,
......@@ -7052,14 +9731,22 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
70529731 .dst_temps = .{ .mem, .unused },
70539732 .clobbers = .{ .eflags = true },
70549733 .each = .{ .once = &.{
9734 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9735 .{ ._, ._dqa, .mov, .tmp2x, .lea(.tmp0x), ._, ._ },
70559736 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7056 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
7057 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
7058 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7059 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
9737 .{ .@"0:", ._q, .mov, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9738 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp3x, ._, ._ },
9739 .{ ._, ._q, .mov, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9740 .{ ._, .p_, .unpcklbw, .tmp4x, .tmp4x, ._, ._ },
9741 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9742 .{ ._, .p_, .@"and", .tmp3x, .tmp2x, ._, ._ },
9743 .{ ._, .p_b, .ackusw, .tmp3x, .tmp3x, ._, ._ },
9744 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9745 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
70609746 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
70619747 } },
70629748 }, .{
9749 .required_features = .{ .slow_incdec, null, null, null },
70639750 .src_constraints = .{
70649751 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
70659752 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
......@@ -7088,14 +9775,13 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
70889775 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
70899776 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
70909777 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
7091 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
7092 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
9778 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },
9779 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
70939780 } },
70949781 }, .{
7095 .required_features = .{ .avx2, null, null, null },
70969782 .src_constraints = .{
7097 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
7098 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
9783 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
9784 .{ .multiple_scalar_signed_int = .{ .of = .byte, .is = .byte } },
70999785 .any,
71009786 },
71019787 .patterns = &.{
......@@ -7103,8 +9789,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71039789 },
71049790 .extra_temps = .{
71059791 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7106 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
7107 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
9792 .{ .type = .i8, .kind = .{ .reg = .al } },
9793 .unused,
71089794 .unused,
71099795 .unused,
71109796 .unused,
......@@ -7118,20 +9804,17 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71189804 .clobbers = .{ .eflags = true },
71199805 .each = .{ .once = &.{
71209806 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7121 .{ .@"0:", .vp_w, .movzxb, .tmp1y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
7122 .{ ._, .vp_w, .movzxb, .tmp2y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
7123 .{ ._, .vp_w, .mull, .tmp1y, .tmp1y, .tmp2y, ._ },
7124 .{ ._, .vp_b, .ackusw, .tmp1y, .tmp1y, .tmp1y, ._ },
7125 .{ ._, .v_q, .perm, .tmp1y, .tmp1y, .ui(0b10_00_10_00), ._ },
7126 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7127 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
7128 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
9807 .{ .@"0:", ._, .movsx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
9808 .{ ._, .i_, .mul, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._, ._ },
9809 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },
9810 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
9811 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
71299812 } },
71309813 }, .{
7131 .required_features = .{ .avx, null, null, null },
9814 .required_features = .{ .avx2, null, null, null },
71329815 .src_constraints = .{
7133 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
7134 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
9816 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
9817 .{ .multiple_scalar_unsigned_int = .{ .of = .xword, .is = .byte } },
71359818 .any,
71369819 },
71379820 .patterns = &.{
......@@ -7139,10 +9822,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71399822 },
71409823 .extra_temps = .{
71419824 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
7142 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7143 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7144 .unused,
7145 .unused,
9825 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9826 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
9827 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
9828 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },
71469829 .unused,
71479830 .unused,
71489831 .unused,
......@@ -7153,17 +9836,20 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71539836 .dst_temps = .{ .mem, .unused },
71549837 .clobbers = .{ .eflags = true },
71559838 .each = .{ .once = &.{
9839 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9840 .{ ._, .vp_q, .broadcast, .tmp2y, .lea(.tmp0q), ._, ._ },
71569841 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7157 .{ .@"0:", .vp_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7158 .{ ._, .vp_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7159 .{ ._, .vp_w, .mull, .tmp1x, .tmp1x, .tmp2x, ._ },
7160 .{ ._, .vp_b, .ackusw, .tmp1x, .tmp1x, .tmp1x, ._ },
7161 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
7162 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
9842 .{ .@"0:", .vp_w, .movzxb, .tmp3y, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
9843 .{ ._, .vp_w, .movzxb, .tmp4y, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },
9844 .{ ._, .vp_w, .mull, .tmp3y, .tmp3y, .tmp4y, ._ },
9845 .{ ._, .vp_b, .shuf, .tmp3y, .tmp3y, .tmp2y, ._ },
9846 .{ ._, .v_q, .perm, .tmp3y, .tmp3y, .ui(0b10_00_10_00), ._ },
9847 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
9848 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },
71639849 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
71649850 } },
71659851 }, .{
7166 .required_features = .{ .sse4_1, null, null, null },
9852 .required_features = .{ .avx, null, null, null },
71679853 .src_constraints = .{
71689854 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
71699855 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
......@@ -7174,6 +9860,8 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71749860 },
71759861 .extra_temps = .{
71769862 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9863 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9864 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
71779865 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
71789866 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
71799867 .unused,
......@@ -7182,23 +9870,23 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
71829870 .unused,
71839871 .unused,
71849872 .unused,
7185 .unused,
7186 .unused,
71879873 },
71889874 .dst_temps = .{ .mem, .unused },
71899875 .clobbers = .{ .eflags = true },
71909876 .each = .{ .once = &.{
9877 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9878 .{ ._, .v_, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
71919879 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7192 .{ .@"0:", .p_w, .movzxb, .tmp1x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7193 .{ ._, .p_w, .movzxb, .tmp2x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7194 .{ ._, .p_w, .mull, .tmp1x, .tmp2x, ._, ._ },
7195 .{ ._, .p_b, .ackusw, .tmp1x, .tmp1x, ._, ._ },
7196 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },
9880 .{ .@"0:", .vp_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9881 .{ ._, .vp_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9882 .{ ._, .vp_w, .mull, .tmp3x, .tmp3x, .tmp4x, ._ },
9883 .{ ._, .vp_b, .shuf, .tmp3x, .tmp3x, .tmp2x, ._ },
9884 .{ ._, .v_q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
71979885 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
71989886 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
71999887 } },
72009888 }, .{
7201 .required_features = .{ .sse2, null, null, null },
9889 .required_features = .{ .sse4_1, null, null, null },
72029890 .src_constraints = .{
72039891 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
72049892 .{ .multiple_scalar_unsigned_int = .{ .of = .qword, .is = .byte } },
......@@ -7209,10 +9897,10 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72099897 },
72109898 .extra_temps = .{
72119899 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
9900 .{ .type = .vector_8_u8, .kind = .{ .pshufb_trunc_mem = .{ .of = .xword, .from = .word, .to = .byte } } },
9901 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
72129902 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
72139903 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7214 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },
7215 .unused,
72169904 .unused,
72179905 .unused,
72189906 .unused,
......@@ -7223,15 +9911,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
72239911 .dst_temps = .{ .mem, .unused },
72249912 .clobbers = .{ .eflags = true },
72259913 .each = .{ .once = &.{
9914 .{ ._, ._, .lea, .tmp0p, .mem(.tmp1), ._, ._ },
9915 .{ ._, ._, .movddup, .tmp2x, .lea(.tmp0q), ._, ._ },
72269916 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
7227 .{ .@"0:", .p_, .xor, .tmp1x, .tmp1x, ._, ._ },
7228 .{ ._, ._q, .mov, .tmp2x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
7229 .{ ._, .p_, .unpcklbw, .tmp2x, .tmp1x, ._, ._ },
7230 .{ ._, ._q, .mov, .tmp3x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
7231 .{ ._, .p_, .unpcklbw, .tmp3x, .tmp1x, ._, ._ },
7232 .{ ._, .p_w, .mull, .tmp2x, .tmp3x, ._, ._ },
7233 .{ ._, .p_b, .ackusw, .tmp2x, .tmp2x, ._, ._ },
7234 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp2x, ._, ._ },
9917 .{ .@"0:", .p_w, .movzxb, .tmp3x, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
9918 .{ ._, .p_w, .movzxb, .tmp4x, .memia(.src1q, .tmp0, .add_unaligned_size), ._, ._ },
9919 .{ ._, .p_w, .mull, .tmp3x, .tmp4x, ._, ._ },
9920 .{ ._, .p_b, .shuf, .tmp3x, .tmp2x, ._, ._ },
9921 .{ ._, ._q, .mov, .memia(.dst0q, .tmp0, .add_unaligned_size), .tmp3x, ._, ._ },
72359922 .{ ._, ._, .add, .tmp0p, .si(8), ._, ._ },
72369923 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
72379924 } },
......@@ -8893,9 +11580,16 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
889311580 }),
889411581 else => |e| return e,
889511582 };
11583 res[0].wrapInt(cg) catch |err| switch (err) {
11584 error.SelectFailed => return cg.fail("failed to select wrap {} {} {}", .{
11585 cg.typeOf(bin_op.lhs).fmt(pt),
11586 ops[0].tracking(cg),
11587 ops[1].tracking(cg),
11588 }),
11589 else => |e| return e,
11590 };
889611591 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
889711592 },
8898 .mul_safe => unreachable,
889911593 .div_float, .div_float_optimized, .div_exact, .div_exact_optimized => |air_tag| if (use_old) try cg.airMulDivBinOp(inst, switch (air_tag) {
890011594 else => unreachable,
890111595 .div_float, .div_float_optimized => .div_float,
......@@ -32065,7 +34759,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3206534759 .{ .src = .{ .to_sse, .none, .none } },
3206634760 },
3206734761 .extra_temps = .{
32068 .{ .type = .vector_16_u8, .kind = .forward_bits },
34762 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3206934763 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3207034764 .unused,
3207134765 .unused,
......@@ -32089,7 +34783,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3208934783 .{ .src = .{ .to_mut_sse, .none, .none } },
3209034784 },
3209134785 .extra_temps = .{
32092 .{ .type = .vector_16_u8, .kind = .forward_bits },
34786 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3209334787 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3209434788 .unused,
3209534789 .unused,
......@@ -32113,7 +34807,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3211334807 .{ .src = .{ .to_sse, .none, .none } },
3211434808 },
3211534809 .extra_temps = .{
32116 .{ .type = .vector_16_u8, .kind = .reverse_bits },
34810 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3211734811 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3211834812 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3211934813 .unused,
......@@ -32141,7 +34835,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3214134835 .{ .src = .{ .to_sse, .none, .none } },
3214234836 },
3214334837 .extra_temps = .{
32144 .{ .type = .vector_16_u8, .kind = .reverse_bits },
34838 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3214534839 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3214634840 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3214734841 .unused,
......@@ -32170,7 +34864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3217034864 .{ .src = .{ .to_mut_sse, .none, .none } },
3217134865 },
3217234866 .extra_temps = .{
32173 .{ .type = .vector_16_u8, .kind = .reverse_bits },
34867 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3217434868 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3217534869 .unused,
3217634870 .unused,
......@@ -32199,7 +34893,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3219934893 .{ .src = .{ .to_mut_sse, .none, .none } },
3220034894 },
3220134895 .extra_temps = .{
32202 .{ .type = .vector_16_u8, .kind = .reverse_bits },
34896 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3220334897 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3220434898 .unused,
3220534899 .unused,
......@@ -32266,7 +34960,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3226634960 .{ .src = .{ .to_sse, .none, .none } },
3226734961 },
3226834962 .extra_temps = .{
32269 .{ .type = .vector_16_u8, .kind = .forward_bits },
34963 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3227034964 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3227134965 .unused,
3227234966 .unused,
......@@ -32292,7 +34986,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3229234986 .{ .src = .{ .to_mut_sse, .none, .none } },
3229334987 },
3229434988 .extra_temps = .{
32295 .{ .type = .vector_16_u8, .kind = .forward_bits },
34989 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3229634990 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3229734991 .unused,
3229834992 .unused,
......@@ -32318,7 +35012,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3231835012 .{ .src = .{ .to_sse, .none, .none } },
3231935013 },
3232035014 .extra_temps = .{
32321 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35015 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3232235016 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3232335017 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3232435018 .unused,
......@@ -32348,7 +35042,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3234835042 .{ .src = .{ .to_sse, .none, .none } },
3234935043 },
3235035044 .extra_temps = .{
32351 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35045 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3235235046 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3235335047 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3235435048 .unused,
......@@ -32379,7 +35073,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3237935073 .{ .src = .{ .to_mut_sse, .none, .none } },
3238035074 },
3238135075 .extra_temps = .{
32382 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35076 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3238335077 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3238435078 .unused,
3238535079 .unused,
......@@ -32410,7 +35104,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3241035104 .{ .src = .{ .to_mut_sse, .none, .none } },
3241135105 },
3241235106 .extra_temps = .{
32413 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35107 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3241435108 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3241535109 .unused,
3241635110 .unused,
......@@ -32480,7 +35174,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3248035174 .{ .src = .{ .to_sse, .none, .none } },
3248135175 },
3248235176 .extra_temps = .{
32483 .{ .type = .vector_16_u8, .kind = .forward_bits },
35177 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3248435178 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3248535179 .unused,
3248635180 .unused,
......@@ -32506,7 +35200,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3250635200 .{ .src = .{ .to_mut_sse, .none, .none } },
3250735201 },
3250835202 .extra_temps = .{
32509 .{ .type = .vector_16_u8, .kind = .forward_bits },
35203 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3251035204 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3251135205 .unused,
3251235206 .unused,
......@@ -32532,7 +35226,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3253235226 .{ .src = .{ .to_sse, .none, .none } },
3253335227 },
3253435228 .extra_temps = .{
32535 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35229 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3253635230 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3253735231 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3253835232 .unused,
......@@ -32562,7 +35256,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3256235256 .{ .src = .{ .to_sse, .none, .none } },
3256335257 },
3256435258 .extra_temps = .{
32565 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35259 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3256635260 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3256735261 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3256835262 .unused,
......@@ -32593,7 +35287,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3259335287 .{ .src = .{ .to_mut_sse, .none, .none } },
3259435288 },
3259535289 .extra_temps = .{
32596 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35290 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3259735291 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3259835292 .unused,
3259935293 .unused,
......@@ -32624,7 +35318,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3262435318 .{ .src = .{ .to_mut_sse, .none, .none } },
3262535319 },
3262635320 .extra_temps = .{
32627 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35321 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3262835322 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3262935323 .unused,
3263035324 .unused,
......@@ -32694,7 +35388,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3269435388 .{ .src = .{ .to_sse, .none, .none } },
3269535389 },
3269635390 .extra_temps = .{
32697 .{ .type = .vector_16_u8, .kind = .forward_bits },
35391 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3269835392 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3269935393 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3270035394 .unused,
......@@ -32720,7 +35414,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3272035414 .{ .src = .{ .to_mut_sse, .none, .none } },
3272135415 },
3272235416 .extra_temps = .{
32723 .{ .type = .vector_16_u8, .kind = .forward_bits },
35417 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3272435418 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3272535419 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3272635420 .unused,
......@@ -32746,7 +35440,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3274635440 .{ .src = .{ .to_mut_sse, .none, .none } },
3274735441 },
3274835442 .extra_temps = .{
32749 .{ .type = .vector_16_u8, .kind = .forward_bits },
35443 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3275035444 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3275135445 .unused,
3275235446 .unused,
......@@ -32773,7 +35467,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3277335467 },
3277435468 .extra_temps = .{
3277535469 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32776 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35470 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3277735471 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3277835472 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3277935473 .unused,
......@@ -32802,7 +35496,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3280235496 },
3280335497 .extra_temps = .{
3280435498 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32805 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35499 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3280635500 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3280735501 .unused,
3280835502 .unused,
......@@ -32871,7 +35565,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3287135565 .{ .src = .{ .to_sse, .none, .none } },
3287235566 },
3287335567 .extra_temps = .{
32874 .{ .type = .vector_16_u8, .kind = .forward_bits },
35568 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3287535569 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3287635570 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3287735571 .unused,
......@@ -32898,7 +35592,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3289835592 .{ .src = .{ .to_mut_sse, .none, .none } },
3289935593 },
3290035594 .extra_temps = .{
32901 .{ .type = .vector_16_u8, .kind = .forward_bits },
35595 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3290235596 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3290335597 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3290435598 .unused,
......@@ -32925,7 +35619,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3292535619 .{ .src = .{ .to_mut_sse, .none, .none } },
3292635620 },
3292735621 .extra_temps = .{
32928 .{ .type = .vector_16_u8, .kind = .forward_bits },
35622 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3292935623 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3293035624 .unused,
3293135625 .unused,
......@@ -32953,7 +35647,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3295335647 },
3295435648 .extra_temps = .{
3295535649 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32956 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35650 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3295735651 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3295835652 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3295935653 .unused,
......@@ -32984,7 +35678,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3298435678 },
3298535679 .extra_temps = .{
3298635680 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
32987 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35681 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3298835682 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3298935683 .unused,
3299035684 .unused,
......@@ -33056,7 +35750,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3305635750 .{ .src = .{ .to_sse, .none, .none } },
3305735751 },
3305835752 .extra_temps = .{
33059 .{ .type = .vector_16_u8, .kind = .forward_bits },
35753 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3306035754 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3306135755 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3306235756 .unused,
......@@ -33083,7 +35777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3308335777 .{ .src = .{ .to_mut_sse, .none, .none } },
3308435778 },
3308535779 .extra_temps = .{
33086 .{ .type = .vector_16_u8, .kind = .forward_bits },
35780 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3308735781 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word } } },
3308835782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3308935783 .unused,
......@@ -33110,7 +35804,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3311035804 .{ .src = .{ .to_mut_sse, .none, .none } },
3311135805 },
3311235806 .extra_temps = .{
33113 .{ .type = .vector_16_u8, .kind = .forward_bits },
35807 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3311435808 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3311535809 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3311635810 .unused,
......@@ -33139,7 +35833,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3313935833 },
3314035834 .extra_temps = .{
3314135835 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
33142 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35836 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3314335837 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3314435838 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3314535839 .unused,
......@@ -33170,7 +35864,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3317035864 },
3317135865 .extra_temps = .{
3317235866 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .word, .smear = 8 } } },
33173 .{ .type = .vector_16_u8, .kind = .reverse_bits },
35867 .{ .type = .vector_16_u8, .kind = .reverse_bits_mem },
3317435868 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3317535869 .unused,
3317635870 .unused,
......@@ -33242,7 +35936,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3324235936 .{ .src = .{ .to_sse, .none, .none } },
3324335937 },
3324435938 .extra_temps = .{
33245 .{ .type = .vector_16_u8, .kind = .forward_bits },
35939 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3324635940 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3324735941 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3324835942 .unused,
......@@ -33268,7 +35962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3326835962 .{ .src = .{ .to_mut_sse, .none, .none } },
3326935963 },
3327035964 .extra_temps = .{
33271 .{ .type = .vector_16_u8, .kind = .forward_bits },
35965 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3327235966 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3327335967 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3327435968 .unused,
......@@ -33294,7 +35988,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3329435988 .{ .src = .{ .to_mut_sse, .none, .none } },
3329535989 },
3329635990 .extra_temps = .{
33297 .{ .type = .vector_16_u8, .kind = .forward_bits },
35991 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3329835992 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3329935993 .unused,
3330035994 .unused,
......@@ -33322,7 +36016,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3332236016 },
3332336017 .extra_temps = .{
3332436018 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33325 .{ .type = .vector_32_u8, .kind = .reverse_bits },
36019 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
3332636020 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3332736021 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
3332836022 .unused,
......@@ -33392,7 +36086,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3339236086 .{ .src = .{ .to_sse, .none, .none } },
3339336087 },
3339436088 .extra_temps = .{
33395 .{ .type = .vector_16_u8, .kind = .forward_bits },
36089 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3339636090 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3339736091 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3339836092 .unused,
......@@ -33419,7 +36113,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3341936113 .{ .src = .{ .to_mut_sse, .none, .none } },
3342036114 },
3342136115 .extra_temps = .{
33422 .{ .type = .vector_16_u8, .kind = .forward_bits },
36116 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3342336117 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3342436118 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3342536119 .unused,
......@@ -33446,7 +36140,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3344636140 .{ .src = .{ .to_mut_sse, .none, .none } },
3344736141 },
3344836142 .extra_temps = .{
33449 .{ .type = .vector_16_u8, .kind = .forward_bits },
36143 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3345036144 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3345136145 .unused,
3345236146 .unused,
......@@ -33475,7 +36169,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3347536169 },
3347636170 .extra_temps = .{
3347736171 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33478 .{ .type = .vector_32_u8, .kind = .reverse_bits },
36172 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
3347936173 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3348036174 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
3348136175 .unused,
......@@ -33548,7 +36242,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3354836242 .{ .src = .{ .to_sse, .none, .none } },
3354936243 },
3355036244 .extra_temps = .{
33551 .{ .type = .vector_16_u8, .kind = .forward_bits },
36245 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3355236246 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3355336247 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3355436248 .unused,
......@@ -33575,7 +36269,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3357536269 .{ .src = .{ .to_mut_sse, .none, .none } },
3357636270 },
3357736271 .extra_temps = .{
33578 .{ .type = .vector_16_u8, .kind = .forward_bits },
36272 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3357936273 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword } } },
3358036274 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3358136275 .unused,
......@@ -33602,7 +36296,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3360236296 .{ .src = .{ .to_mut_sse, .none, .none } },
3360336297 },
3360436298 .extra_temps = .{
33605 .{ .type = .vector_16_u8, .kind = .forward_bits },
36299 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3360636300 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3360736301 .unused,
3360836302 .unused,
......@@ -33631,7 +36325,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3363136325 },
3363236326 .extra_temps = .{
3363336327 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .dword, .smear = 8 } } },
33634 .{ .type = .vector_32_u8, .kind = .reverse_bits },
36328 .{ .type = .vector_32_u8, .kind = .reverse_bits_mem },
3363536329 .{ .type = .vector_32_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3363636330 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
3363736331 .unused,
......@@ -33704,7 +36398,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3370436398 .{ .src = .{ .to_sse, .none, .none } },
3370536399 },
3370636400 .extra_temps = .{
33707 .{ .type = .vector_16_u8, .kind = .forward_bits },
36401 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3370836402 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
3370936403 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3371036404 .unused,
......@@ -33730,7 +36424,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3373036424 .{ .src = .{ .to_mut_sse, .none, .none } },
3373136425 },
3373236426 .extra_temps = .{
33733 .{ .type = .vector_16_u8, .kind = .forward_bits },
36427 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3373436428 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
3373536429 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3373636430 .unused,
......@@ -33756,7 +36450,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3375636450 .{ .src = .{ .to_mut_sse, .none, .none } },
3375736451 },
3375836452 .extra_temps = .{
33759 .{ .type = .vector_16_u8, .kind = .forward_bits },
36453 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3376036454 .unused,
3376136455 .unused,
3376236456 .unused,
......@@ -33827,7 +36521,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3382736521 .{ .src = .{ .to_sse, .none, .none } },
3382836522 },
3382936523 .extra_temps = .{
33830 .{ .type = .vector_16_u8, .kind = .forward_bits },
36524 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3383136525 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
3383236526 .unused,
3383336527 .unused,
......@@ -33855,7 +36549,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3385536549 .{ .src = .{ .to_mut_sse, .none, .none } },
3385636550 },
3385736551 .extra_temps = .{
33858 .{ .type = .vector_16_u8, .kind = .forward_bits },
36552 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3385936553 .unused,
3386036554 .unused,
3386136555 .unused,
......@@ -33929,7 +36623,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3392936623 .{ .src = .{ .to_sse, .none, .none } },
3393036624 },
3393136625 .extra_temps = .{
33932 .{ .type = .vector_16_u8, .kind = .forward_bits },
36626 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3393336627 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
3393436628 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3393536629 .unused,
......@@ -33956,7 +36650,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3395636650 .{ .src = .{ .to_mut_sse, .none, .none } },
3395736651 },
3395836652 .extra_temps = .{
33959 .{ .type = .vector_16_u8, .kind = .forward_bits },
36653 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3396036654 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .qword } } },
3396136655 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3396236656 .unused,
......@@ -33983,7 +36677,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3398336677 .{ .src = .{ .to_mut_sse, .none, .none } },
3398436678 },
3398536679 .extra_temps = .{
33986 .{ .type = .vector_16_u8, .kind = .forward_bits },
36680 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3398736681 .unused,
3398836682 .unused,
3398936683 .unused,
......@@ -34057,7 +36751,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3405736751 .{ .src = .{ .to_sse, .none, .none } },
3405836752 },
3405936753 .extra_temps = .{
34060 .{ .type = .vector_16_u8, .kind = .forward_bits },
36754 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3406136755 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3406236756 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3406336757 .unused,
......@@ -34083,7 +36777,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3408336777 .{ .src = .{ .to_mut_sse, .none, .none } },
3408436778 },
3408536779 .extra_temps = .{
34086 .{ .type = .vector_16_u8, .kind = .forward_bits },
36780 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3408736781 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3408836782 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3408936783 .unused,
......@@ -34109,7 +36803,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3410936803 .{ .src = .{ .to_sse, .none, .none } },
3411036804 },
3411136805 .extra_temps = .{
34112 .{ .type = .vector_16_u8, .kind = .forward_bits },
36806 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3411336807 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3411436808 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3411536809 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
......@@ -34142,7 +36836,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3414236836 .{ .src = .{ .to_mut_sse, .none, .none } },
3414336837 },
3414436838 .extra_temps = .{
34145 .{ .type = .vector_16_u8, .kind = .forward_bits },
36839 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3414636840 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3414736841 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3414836842 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -34175,7 +36869,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3417536869 .{ .src = .{ .to_sse, .none, .none } },
3417636870 },
3417736871 .extra_temps = .{
34178 .{ .type = .vector_16_u8, .kind = .forward_bits },
36872 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3417936873 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3418036874 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3418136875 .{ .type = .vector_16_u8, .kind = .{ .mut_rc = .{ .ref = .src0, .rc = .sse } } },
......@@ -34208,7 +36902,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3420836902 .{ .src = .{ .to_mut_sse, .none, .none } },
3420936903 },
3421036904 .extra_temps = .{
34211 .{ .type = .vector_16_u8, .kind = .forward_bits },
36905 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3421236906 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3421336907 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3421436908 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },
......@@ -34241,7 +36935,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3424136935 .{ .src = .{ .to_sse, .none, .none } },
3424236936 },
3424336937 .extra_temps = .{
34244 .{ .type = .vector_32_u8, .kind = .forward_bits },
36938 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
3424536939 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
3424636940 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3424736941 .unused,
......@@ -34268,7 +36962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3426836962 .{ .src = .{ .to_sse, .none, .none } },
3426936963 },
3427036964 .extra_temps = .{
34271 .{ .type = .vector_32_u8, .kind = .forward_bits },
36965 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
3427236966 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
3427336967 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3427436968 .unused,
......@@ -34297,7 +36991,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3429736991 .extra_temps = .{
3429836992 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3429936993 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34300 .{ .type = .vector_32_u8, .kind = .forward_bits },
36994 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
3430136995 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
3430236996 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
3430336997 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -34333,7 +37027,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3433337027 .extra_temps = .{
3433437028 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3433537029 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34336 .{ .type = .vector_32_u8, .kind = .forward_bits },
37030 .{ .type = .vector_32_u8, .kind = .forward_bits_mem },
3433737031 .{ .type = .vector_32_u8, .kind = .{ .pshufb_bswap_mem = .{ .repeat = 2, .size = .xword } } },
3433837032 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
3433937033 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },
......@@ -34369,7 +37063,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3436937063 .extra_temps = .{
3437037064 .{ .type = .usize, .kind = .{ .rc = .general_purpose } },
3437137065 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },
34372 .{ .type = .vector_16_u8, .kind = .forward_bits },
37066 .{ .type = .vector_16_u8, .kind = .forward_bits_mem },
3437337067 .{ .type = .vector_16_u8, .kind = .{ .pshufb_bswap_mem = .{ .size = .xword } } },
3437437068 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
3437537069 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },
......@@ -106842,10 +109536,11 @@ const Select = struct {
106842109536 f64_0x1p52_0x1p84_mem,
106843109537 u32_0x1p52_hi_0x1p84_hi_0_0_mem,
106844109538 f32_0_0x1p64_mem,
106845 pshufb_trunc_mem: struct { from: Memory.Size, to: Memory.Size },
109539 pshufb_trunc_mem: struct { of: Memory.Size = .none, from: Memory.Size, to: Memory.Size },
109540 pand_trunc_mem: struct { from: Memory.Size, to: Memory.Size },
106846109541 pshufb_bswap_mem: struct { repeat: u4 = 1, size: Memory.Size, smear: u4 = 1 },
106847 forward_bits,
106848 reverse_bits,
109542 forward_bits_mem,
109543 reverse_bits_mem,
106849109544 frame: FrameIndex,
106850109545 lazy_symbol: struct { kind: link.File.LazySymbol.Kind, ref: Select.Operand.Ref = .none },
106851109546 symbol: *const struct { lib: ?[]const u8 = null, name: []const u8 },
......@@ -107061,17 +109756,40 @@ const Select = struct {
107061109756 const zcu = pt.zcu;
107062109757 assert(spec.type.isVector(zcu));
107063109758 assert(spec.type.childType(zcu).toIntern() == .u8_type);
107064 var bytes: [16]u8 = @splat(1 << 7);
109759 var elem_buf: [32]u8 = @splat(1 << 7);
109760 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
109761 const of_bytes: u32 = @intCast(switch (trunc_spec.of) {
109762 .none => elems.len,
109763 else => |of| @divExact(of.bitSize(cg.target), 8),
109764 });
107065109765 const from_bytes: u32 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8));
107066109766 const to_bytes: u32 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8));
107067109767 var from_index: u32 = 0;
107068109768 var to_index: u32 = 0;
107069 while (from_index < bytes.len) {
107070 for (0..to_bytes) |byte_off| bytes[to_index + byte_off] = @intCast(from_index + byte_off);
109769 while (from_index < of_bytes) : ({
107071109770 from_index += from_bytes;
107072 to_index += to_bytes;
107073 }
107074 const elems = bytes[0..spec.type.vectorLen(zcu)];
109771 switch (from_index % 16) {
109772 0 => to_index = from_index,
109773 else => to_index += to_bytes,
109774 }
109775 }) for (0..to_bytes) |byte_off| {
109776 elems[to_index + byte_off] = @as(u4, @truncate(from_index + byte_off));
109777 };
109778 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
109779 .ty = spec.type.toIntern(),
109780 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
109781 } }))), true };
109782 },
109783 .pand_trunc_mem => |trunc_spec| {
109784 const zcu = pt.zcu;
109785 assert(spec.type.isVector(zcu));
109786 assert(spec.type.childType(zcu).toIntern() == .u8_type);
109787 var elem_buf: [32]u8 = @splat(0);
109788 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
109789 const from_bytes: u32 = @intCast(@divExact(trunc_spec.from.bitSize(cg.target), 8));
109790 const to_bytes: u32 = @intCast(@divExact(trunc_spec.to.bitSize(cg.target), 8));
109791 var index: u32 = 0;
109792 while (index < elems.len) : (index += from_bytes) @memset(elems[index..][0..to_bytes], std.math.maxInt(u8));
107075109793 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107076109794 .ty = spec.type.toIntern(),
107077109795 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
......@@ -107081,20 +109799,20 @@ const Select = struct {
107081109799 const zcu = pt.zcu;
107082109800 assert(spec.type.isVector(zcu));
107083109801 assert(spec.type.childType(zcu).toIntern() == .u8_type);
107084 var bytes: [32]u8 = @splat(1 << 7);
109802 var elem_buf: [32]u8 = @splat(1 << 7);
109803 const elems = elem_buf[0..spec.type.vectorLen(zcu)];
107085109804 const len: usize = @intCast(@divExact(bswap_spec.size.bitSize(cg.target), 8));
107086109805 var to_index: u32 = 0;
107087109806 for (0..bswap_spec.repeat) |_| for (0..len) |from_index| {
107088 @memset(bytes[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));
109807 @memset(elems[to_index..][0..bswap_spec.smear], @intCast(len - 1 - from_index));
107089109808 to_index += bswap_spec.smear;
107090109809 };
107091 const elems = bytes[0..spec.type.vectorLen(zcu)];
107092109810 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107093109811 .ty = spec.type.toIntern(),
107094109812 .storage = .{ .bytes = try zcu.intern_pool.getOrPutString(zcu.gpa, pt.tid, elems, .maybe_embedded_nulls) },
107095109813 } }))), true };
107096109814 },
107097 .forward_bits, .reverse_bits => {
109815 .forward_bits_mem, .reverse_bits_mem => {
107098109816 const zcu = pt.zcu;
107099109817 assert(spec.type.isVector(zcu));
107100109818 assert(spec.type.childType(zcu).toIntern() == .u8_type);
......@@ -107102,8 +109820,8 @@ const Select = struct {
107102109820 const elems = bytes[0..spec.type.vectorLen(zcu)];
107103109821 for (elems, 0..) |*elem, index| elem.* = switch (spec.kind) {
107104109822 else => unreachable,
107105 .forward_bits => @as(u8, 1 << 0) << @truncate(index),
107106 .reverse_bits => @as(u8, 1 << 7) >> @truncate(index),
109823 .forward_bits_mem => @as(u8, 1 << 0) << @truncate(index),
109824 .reverse_bits_mem => @as(u8, 1 << 7) >> @truncate(index),
107107109825 };
107108109826 return .{ try cg.tempMemFromValue(.fromInterned(try pt.intern(.{ .aggregate = .{
107109109827 .ty = spec.type.toIntern(),
src/arch/x86_64/Lower.zig+1-1
......@@ -579,7 +579,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
579579 .rrri => inst.data.rrri.fixes,
580580 .rri_s, .rri_u => inst.data.rri.fixes,
581581 .ri_s, .ri_u, .ri_64, .ir => inst.data.ri.fixes,
582 .rm, .rmi_s, .mr => inst.data.rx.fixes,
582 .rm, .rmi_s, .rmi_u, .mr => inst.data.rx.fixes,
583583 .mrr, .rrm, .rmr => inst.data.rrx.fixes,
584584 .rmi, .mri => inst.data.rix.fixes,
585585 .rrmr => inst.data.rrrx.fixes,
src/codegen/c/Type.zig+60
......@@ -1474,6 +1474,66 @@ pub const Pool = struct {
14741474 };
14751475 return pool.fromFields(allocator, .@"struct", &fields, kind);
14761476 },
1477 .vector_1_u8_type => {
1478 const vector_ctype = try pool.getVector(allocator, .{
1479 .elem_ctype = .u8,
1480 .len = 1,
1481 });
1482 if (!kind.isParameter()) return vector_ctype;
1483 var fields = [_]Info.Field{
1484 .{
1485 .name = .{ .index = .array },
1486 .ctype = vector_ctype,
1487 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1488 },
1489 };
1490 return pool.fromFields(allocator, .@"struct", &fields, kind);
1491 },
1492 .vector_2_u8_type => {
1493 const vector_ctype = try pool.getVector(allocator, .{
1494 .elem_ctype = .u8,
1495 .len = 2,
1496 });
1497 if (!kind.isParameter()) return vector_ctype;
1498 var fields = [_]Info.Field{
1499 .{
1500 .name = .{ .index = .array },
1501 .ctype = vector_ctype,
1502 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1503 },
1504 };
1505 return pool.fromFields(allocator, .@"struct", &fields, kind);
1506 },
1507 .vector_4_u8_type => {
1508 const vector_ctype = try pool.getVector(allocator, .{
1509 .elem_ctype = .u8,
1510 .len = 4,
1511 });
1512 if (!kind.isParameter()) return vector_ctype;
1513 var fields = [_]Info.Field{
1514 .{
1515 .name = .{ .index = .array },
1516 .ctype = vector_ctype,
1517 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1518 },
1519 };
1520 return pool.fromFields(allocator, .@"struct", &fields, kind);
1521 },
1522 .vector_8_u8_type => {
1523 const vector_ctype = try pool.getVector(allocator, .{
1524 .elem_ctype = .u8,
1525 .len = 8,
1526 });
1527 if (!kind.isParameter()) return vector_ctype;
1528 var fields = [_]Info.Field{
1529 .{
1530 .name = .{ .index = .array },
1531 .ctype = vector_ctype,
1532 .alignas = AlignAs.fromAbiAlignment(Type.u8.abiAlignment(zcu)),
1533 },
1534 };
1535 return pool.fromFields(allocator, .@"struct", &fields, kind);
1536 },
14771537 .vector_16_u8_type => {
14781538 const vector_ctype = try pool.getVector(allocator, .{
14791539 .elem_ctype = .u8,
test/behavior/x86_64/math.zig+9
......@@ -21684,6 +21684,15 @@ test mulUnsafe {
2168421684 try test_mul_unsafe.testIntVectors();
2168521685}
2168621686
21687inline fn mulWrap(comptime Type: type, lhs: Type, rhs: Type) Type {
21688 return lhs *% rhs;
21689}
21690test mulWrap {
21691 const test_mul_wrap = binary(mulWrap, .{});
21692 try test_mul_wrap.testInts();
21693 try test_mul_wrap.testIntVectors();
21694}
21695
2168721696inline fn multiply(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs * rhs) {
2168821697 return lhs * rhs;
2168921698}
test/cases/compile_errors/@import_zon_bad_type.zig+3-3
......@@ -117,9 +117,9 @@ export fn testMutablePointer() void {
117117// tmp.zig:37:38: note: imported here
118118// neg_inf.zon:1:1: error: expected type '?u8'
119119// tmp.zig:57:28: note: imported here
120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_487'
120// neg_inf.zon:1:1: error: expected type 'tmp.testNonExhaustiveEnum__enum_491'
121121// tmp.zig:62:39: note: imported here
122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_489'
122// neg_inf.zon:1:1: error: expected type 'tmp.testUntaggedUnion__union_493'
123123// tmp.zig:67:44: note: imported here
124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_492'
124// neg_inf.zon:1:1: error: expected type 'tmp.testTaggedUnionVoid__union_496'
125125// tmp.zig:72:50: note: imported here
test/cases/compile_errors/anytype_param_requires_comptime.zig+1-1
......@@ -15,6 +15,6 @@ pub export fn entry() void {
1515// error
1616//
1717// :7:25: error: unable to resolve comptime value
18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_461.C' must be comptime-known
18// :7:25: note: initializer of comptime-only struct 'tmp.S.foo__anon_465.C' must be comptime-known
1919// :4:16: note: struct requires comptime because of this field
2020// :4:16: note: types are not available at runtime
test/cases/compile_errors/bogus_method_call_on_slice.zig+1-1
......@@ -16,5 +16,5 @@ pub export fn entry2() void {
1616//
1717// :3:6: error: no field or member function named 'copy' in '[]const u8'
1818// :9:8: error: no field or member function named 'bar' in '@TypeOf(.{})'
19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_465'
19// :12:18: error: no field or member function named 'bar' in 'tmp.entry2__struct_469'
2020// :12:6: note: struct declared here
test/cases/compile_errors/coerce_anon_struct.zig+1-1
......@@ -6,6 +6,6 @@ export fn foo() void {
66
77// error
88//
9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_454'
9// :4:16: error: expected type 'tmp.T', found 'tmp.foo__struct_458'
1010// :3:16: note: struct declared here
1111// :1:11: note: struct declared here
test/cases/compile_errors/redundant_try.zig+2-2
......@@ -44,9 +44,9 @@ comptime {
4444//
4545// :5:23: error: expected error union type, found 'comptime_int'
4646// :10:23: error: expected error union type, found '@TypeOf(.{})'
47// :15:23: error: expected error union type, found 'tmp.test2__struct_491'
47// :15:23: error: expected error union type, found 'tmp.test2__struct_495'
4848// :15:23: note: struct declared here
49// :20:27: error: expected error union type, found 'tmp.test3__struct_493'
49// :20:27: error: expected error union type, found 'tmp.test3__struct_497'
5050// :20:27: note: struct declared here
5151// :25:23: error: expected error union type, found 'struct { comptime *const [5:0]u8 = "hello" }'
5252// :31:13: error: expected error union type, found 'u32'