authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-18 14:36:33-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-05-18 14:36:33-04:00
logb77e6013422a6e7066a14dbe82e8512636ba13d2
tree0ce64ba78449b0cfdee7d926fb1f3c35ad364e7f
parent74a3ae492797b1b2cf1936f0c91560585efdf6c6
parenta4eabd39794014c871670937155c94e11fce991b
signature Signed by PGP key B5690EEEBB952194

Merge pull request #23834 from jacobly0/x86_64-rewrite

x86_64: finish rewriting scalar overflow and saturate operations

35 files changed, 89520 insertions(+), 59476 deletions(-)

lib/std/debug/Dwarf.zig+3-1
...@@ -402,7 +402,7 @@ pub const ExceptionFrameHeader = struct {...@@ -402,7 +402,7 @@ pub const ExceptionFrameHeader = struct {
402 }402 }
403 }403 }
404404
405 if (len == 0) return bad();405 if (len == 0) return missing();
406 fbr.pos = left * entry_size;406 fbr.pos = left * entry_size;
407407
408 // Read past the pc_begin field of the entry408 // Read past the pc_begin field of the entry
...@@ -460,6 +460,8 @@ pub const ExceptionFrameHeader = struct {...@@ -460,6 +460,8 @@ pub const ExceptionFrameHeader = struct {
460 @sizeOf(usize),460 @sizeOf(usize),
461 native_endian,461 native_endian,
462 );462 );
463
464 if (pc < fde.pc_begin or pc >= fde.pc_begin + fde.pc_range) return missing();
463 }465 }
464};466};
465467
lib/std/debug/SelfInfo.zig+1-1
...@@ -1633,7 +1633,7 @@ pub fn unwindFrameDwarf(...@@ -1633,7 +1633,7 @@ pub fn unwindFrameDwarf(
1633 &cie,1633 &cie,
1634 &fde,1634 &fde,
1635 ) catch |err| switch (err) {1635 ) catch |err| switch (err) {
1636 error.InvalidDebugInfo => {1636 error.MissingDebugInfo => {
1637 // `.eh_frame_hdr` appears to be incomplete, so go ahead and populate `cie_map`1637 // `.eh_frame_hdr` appears to be incomplete, so go ahead and populate `cie_map`
1638 // and `fde_list`, and fall back to the binary search logic below.1638 // and `fde_list`, and fall back to the binary search logic below.
1639 try di.scanCieFdeInfo(allocator, base_address);1639 try di.scanCieFdeInfo(allocator, base_address);
lib/std/math.zig+21-11
...@@ -774,18 +774,15 @@ pub fn Log2IntCeil(comptime T: type) type {...@@ -774,18 +774,15 @@ pub fn Log2IntCeil(comptime T: type) type {
774/// Returns the smallest integer type that can hold both from and to.774/// Returns the smallest integer type that can hold both from and to.
775pub fn IntFittingRange(comptime from: comptime_int, comptime to: comptime_int) type {775pub fn IntFittingRange(comptime from: comptime_int, comptime to: comptime_int) type {
776 assert(from <= to);776 assert(from <= to);
777 if (from == 0 and to == 0) {
778 return u0;
779 }
780 const signedness: std.builtin.Signedness = if (from < 0) .signed else .unsigned;777 const signedness: std.builtin.Signedness = if (from < 0) .signed else .unsigned;
781 const largest_positive_integer = @max(if (from < 0) (-from) - 1 else from, to); // two's complement778 return @Type(.{ .int = .{
782 const base = log2(largest_positive_integer);779 .signedness = signedness,
783 const upper = (1 << base) - 1;780 .bits = @as(u16, @intFromBool(signedness == .signed)) +
784 var magnitude_bits = if (upper >= largest_positive_integer) base else base + 1;781 switch (if (from < 0) @max(@abs(from) - 1, to) else to) {
785 if (signedness == .signed) {782 0 => 0,
786 magnitude_bits += 1;783 else => |pos_max| 1 + log2(pos_max),
787 }784 },
788 return std.meta.Int(signedness, magnitude_bits);785 } });
789}786}
790787
791test IntFittingRange {788test IntFittingRange {
...@@ -1267,6 +1264,19 @@ pub fn log2_int(comptime T: type, x: T) Log2Int(T) {...@@ -1267,6 +1264,19 @@ pub fn log2_int(comptime T: type, x: T) Log2Int(T) {
1267 return @as(Log2Int(T), @intCast(@typeInfo(T).int.bits - 1 - @clz(x)));1264 return @as(Log2Int(T), @intCast(@typeInfo(T).int.bits - 1 - @clz(x)));
1268}1265}
12691266
1267test log2_int {
1268 try testing.expect(log2_int(u32, 1) == 0);
1269 try testing.expect(log2_int(u32, 2) == 1);
1270 try testing.expect(log2_int(u32, 3) == 1);
1271 try testing.expect(log2_int(u32, 4) == 2);
1272 try testing.expect(log2_int(u32, 5) == 2);
1273 try testing.expect(log2_int(u32, 6) == 2);
1274 try testing.expect(log2_int(u32, 7) == 2);
1275 try testing.expect(log2_int(u32, 8) == 3);
1276 try testing.expect(log2_int(u32, 9) == 3);
1277 try testing.expect(log2_int(u32, 10) == 3);
1278}
1279
1270/// Return the log base 2 of integer value x, rounding up to the1280/// Return the log base 2 of integer value x, rounding up to the
1271/// nearest integer.1281/// nearest integer.
1272pub fn log2_int_ceil(comptime T: type, x: T) Log2IntCeil(T) {1282pub fn log2_int_ceil(comptime T: type, x: T) Log2IntCeil(T) {
lib/std/math/big/int.zig+4-4
...@@ -415,12 +415,12 @@ pub const Mutable = struct {...@@ -415,12 +415,12 @@ pub const Mutable = struct {
415 // in the case that scalar happens to be small in magnitude within its type, but it415 // in the case that scalar happens to be small in magnitude within its type, but it
416 // is well worth being able to use the stack and not needing an allocator passed in.416 // is well worth being able to use the stack and not needing an allocator passed in.
417 // Note that Mutable.init still sets len to calcLimbLen(scalar) in any case.417 // Note that Mutable.init still sets len to calcLimbLen(scalar) in any case.
418 const limb_len = comptime switch (@typeInfo(@TypeOf(scalar))) {418 const limbs_len = comptime switch (@typeInfo(@TypeOf(scalar))) {
419 .comptime_int => calcLimbLen(scalar),419 .comptime_int => calcLimbLen(scalar),
420 .int => |info| calcTwosCompLimbCount(info.bits),420 .int => |info| calcTwosCompLimbCount(info.bits),
421 else => @compileError("expected scalar to be an int"),421 else => @compileError("expected scalar to be an int"),
422 };422 };
423 var limbs: [limb_len]Limb = undefined;423 var limbs: [limbs_len]Limb = undefined;
424 const operand = init(&limbs, scalar).toConst();424 const operand = init(&limbs, scalar).toConst();
425 return add(r, a, operand);425 return add(r, a, operand);
426 }426 }
...@@ -2454,12 +2454,12 @@ pub const Const = struct {...@@ -2454,12 +2454,12 @@ pub const Const = struct {
2454 // in the case that scalar happens to be small in magnitude within its type, but it2454 // in the case that scalar happens to be small in magnitude within its type, but it
2455 // is well worth being able to use the stack and not needing an allocator passed in.2455 // is well worth being able to use the stack and not needing an allocator passed in.
2456 // Note that Mutable.init still sets len to calcLimbLen(scalar) in any case.2456 // Note that Mutable.init still sets len to calcLimbLen(scalar) in any case.
2457 const limb_len = comptime switch (@typeInfo(@TypeOf(scalar))) {2457 const limbs_len = comptime switch (@typeInfo(@TypeOf(scalar))) {
2458 .comptime_int => calcLimbLen(scalar),2458 .comptime_int => calcLimbLen(scalar),
2459 .int => |info| calcTwosCompLimbCount(info.bits),2459 .int => |info| calcTwosCompLimbCount(info.bits),
2460 else => @compileError("expected scalar to be an int"),2460 else => @compileError("expected scalar to be an int"),
2461 };2461 };
2462 var limbs: [limb_len]Limb = undefined;2462 var limbs: [limbs_len]Limb = undefined;
2463 const rhs = Mutable.init(&limbs, scalar);2463 const rhs = Mutable.init(&limbs, scalar);
2464 return order(lhs, rhs.toConst());2464 return order(lhs, rhs.toConst());
2465 }2465 }
lib/std/math/big/int_test.zig-4
...@@ -2295,8 +2295,6 @@ test "sat shift-left signed simple positive" {...@@ -2295,8 +2295,6 @@ test "sat shift-left signed simple positive" {
2295}2295}
22962296
2297test "sat shift-left signed multi positive" {2297test "sat shift-left signed multi positive" {
2298 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2299
2300 var x: SignedDoubleLimb = 1;2298 var x: SignedDoubleLimb = 1;
2301 _ = &x;2299 _ = &x;
23022300
...@@ -2310,8 +2308,6 @@ test "sat shift-left signed multi positive" {...@@ -2310,8 +2308,6 @@ test "sat shift-left signed multi positive" {
2310}2308}
23112309
2312test "sat shift-left signed multi negative" {2310test "sat shift-left signed multi negative" {
2313 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
2314
2315 var x: SignedDoubleLimb = -1;2311 var x: SignedDoubleLimb = -1;
2316 _ = &x;2312 _ = &x;
23172313
lib/std/math/log2.zig+11-10
...@@ -12,12 +12,10 @@ const expect = std.testing.expect;...@@ -12,12 +12,10 @@ const expect = std.testing.expect;
12/// - log2(nan) = nan12/// - log2(nan) = nan
13pub fn log2(x: anytype) @TypeOf(x) {13pub fn log2(x: anytype) @TypeOf(x) {
14 const T = @TypeOf(x);14 const T = @TypeOf(x);
15 switch (@typeInfo(T)) {15 return switch (@typeInfo(T)) {
16 .comptime_float => {16 .comptime_float, .float => @log2(x),
17 return @as(comptime_float, @log2(x));
18 },
19 .float => return @log2(x),
20 .comptime_int => comptime {17 .comptime_int => comptime {
18 std.debug.assert(x > 0);
21 var x_shifted = x;19 var x_shifted = x;
22 // First, calculate floorPowerOfTwo(x)20 // First, calculate floorPowerOfTwo(x)
23 var shift_amt = 1;21 var shift_amt = 1;
...@@ -34,12 +32,15 @@ pub fn log2(x: anytype) @TypeOf(x) {...@@ -34,12 +32,15 @@ pub fn log2(x: anytype) @TypeOf(x) {
34 }32 }
35 return result;33 return result;
36 },34 },
37 .int => |IntType| switch (IntType.signedness) {35 .int => |int_info| math.log2_int(switch (int_info.signedness) {
38 .signed => @compileError("log2 not implemented for signed integers"),36 .signed => @Type(.{ .int = .{
39 .unsigned => return math.log2_int(T, x),37 .signedness = .unsigned,
40 },38 .bits = int_info.bits -| 1,
39 } }),
40 .unsigned => T,
41 }, @intCast(x)),
41 else => @compileError("log2 not implemented for " ++ @typeName(T)),42 else => @compileError("log2 not implemented for " ++ @typeName(T)),
42 }43 };
43}44}
4445
45test log2 {46test log2 {
lib/std/zig/Zir.zig+5-1
...@@ -2142,7 +2142,7 @@ pub const Inst = struct {...@@ -2142,7 +2142,7 @@ pub const Inst = struct {
2142 ref_start_index = static_len,2142 ref_start_index = static_len,
2143 _,2143 _,
21442144
2145 pub const static_len = 97;2145 pub const static_len = 101;
21462146
2147 pub fn toRef(i: Index) Inst.Ref {2147 pub fn toRef(i: Index) Inst.Ref {
2148 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));2148 return @enumFromInt(@intFromEnum(Index.ref_start_index) + @intFromEnum(i));
...@@ -2225,6 +2225,7 @@ pub const Inst = struct {...@@ -2225,6 +2225,7 @@ pub const Inst = struct {
2225 single_const_pointer_to_comptime_int_type,2225 single_const_pointer_to_comptime_int_type,
2226 slice_const_u8_type,2226 slice_const_u8_type,
2227 slice_const_u8_sentinel_0_type,2227 slice_const_u8_sentinel_0_type,
2228 vector_8_i8_type,
2228 vector_16_i8_type,2229 vector_16_i8_type,
2229 vector_32_i8_type,2230 vector_32_i8_type,
2230 vector_1_u8_type,2231 vector_1_u8_type,
...@@ -2233,8 +2234,10 @@ pub const Inst = struct {...@@ -2233,8 +2234,10 @@ pub const Inst = struct {
2233 vector_8_u8_type,2234 vector_8_u8_type,
2234 vector_16_u8_type,2235 vector_16_u8_type,
2235 vector_32_u8_type,2236 vector_32_u8_type,
2237 vector_4_i16_type,
2236 vector_8_i16_type,2238 vector_8_i16_type,
2237 vector_16_i16_type,2239 vector_16_i16_type,
2240 vector_4_u16_type,
2238 vector_8_u16_type,2241 vector_8_u16_type,
2239 vector_16_u16_type,2242 vector_16_u16_type,
2240 vector_4_i32_type,2243 vector_4_i32_type,
...@@ -2245,6 +2248,7 @@ pub const Inst = struct {...@@ -2245,6 +2248,7 @@ pub const Inst = struct {
2245 vector_4_i64_type,2248 vector_4_i64_type,
2246 vector_2_u64_type,2249 vector_2_u64_type,
2247 vector_4_u64_type,2250 vector_4_u64_type,
2251 vector_2_u128_type,
2248 vector_4_f16_type,2252 vector_4_f16_type,
2249 vector_8_f16_type,2253 vector_8_f16_type,
2250 vector_2_f32_type,2254 vector_2_f32_type,
lib/zig.h+20-11
...@@ -1115,14 +1115,15 @@ static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t...@@ -1115,14 +1115,15 @@ static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t
1115\1115\
1116 static inline uint##w##_t zig_shls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \1116 static inline uint##w##_t zig_shls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \
1117 uint##w##_t res; \1117 uint##w##_t res; \
1118 if (rhs >= bits) return lhs != UINT##w##_C(0) ? zig_maxInt_u(w, bits) : lhs; \1118 if (rhs < bits && !zig_shlo_u##w(&res, lhs, rhs, bits)) return res; \
1119 return zig_shlo_u##w(&res, lhs, (uint8_t)rhs, bits) ? zig_maxInt_u(w, bits) : res; \1119 return lhs == INT##w##_C(0) ? INT##w##_C(0) : zig_maxInt_u(w, bits); \
1120 } \1120 } \
1121\1121\
1122 static inline int##w##_t zig_shls_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \1122 static inline int##w##_t zig_shls_i##w(int##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \
1123 int##w##_t res; \1123 int##w##_t res; \
1124 if ((uint##w##_t)rhs < (uint##w##_t)bits && !zig_shlo_i##w(&res, lhs, (uint8_t)rhs, bits)) return res; \1124 if (rhs < bits && !zig_shlo_i##w(&res, lhs, rhs, bits)) return res; \
1125 return lhs < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \1125 return lhs == INT##w##_C(0) ? INT##w##_C(0) : \
1126 lhs < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \
1126 } \1127 } \
1127\1128\
1128 static inline uint##w##_t zig_adds_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \1129 static inline uint##w##_t zig_adds_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \
...@@ -1851,15 +1852,23 @@ static inline bool zig_shlo_i128(zig_i128 *res, zig_i128 lhs, uint8_t rhs, uint8...@@ -1851,15 +1852,23 @@ static inline bool zig_shlo_i128(zig_i128 *res, zig_i128 lhs, uint8_t rhs, uint8
18511852
1852static inline zig_u128 zig_shls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) {1853static inline zig_u128 zig_shls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) {
1853 zig_u128 res;1854 zig_u128 res;
1854 if (zig_cmp_u128(rhs, zig_make_u128(0, bits)) >= INT32_C(0))1855 if (zig_cmp_u128(rhs, zig_make_u128(0, bits)) < INT32_C(0) && !zig_shlo_u128(&res, lhs, (uint8_t)zig_lo_u128(rhs), bits)) return res;
1855 return zig_cmp_u128(lhs, zig_make_u128(0, 0)) != INT32_C(0) ? zig_maxInt_u(128, bits) : lhs;1856 switch (zig_cmp_u128(lhs, zig_make_u128(0, 0))) {
1856 return zig_shlo_u128(&res, lhs, (uint8_t)zig_lo_u128(rhs), bits) ? zig_maxInt_u(128, bits) : res;1857 case 0: return zig_make_u128(0, 0);
1858 case 1: return zig_maxInt_u(128, bits);
1859 default: zig_unreachable();
1860 }
1857}1861}
18581862
1859static inline zig_i128 zig_shls_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) {1863static inline zig_i128 zig_shls_i128(zig_i128 lhs, zig_u128 rhs, uint8_t bits) {
1860 zig_i128 res;1864 zig_i128 res;
1861 if (zig_cmp_u128(zig_bitCast_u128(rhs), zig_make_u128(0, bits)) < INT32_C(0) && !zig_shlo_i128(&res, lhs, (uint8_t)zig_lo_i128(rhs), bits)) return res;1865 if (zig_cmp_u128(rhs, zig_make_u128(0, bits)) < INT32_C(0) && !zig_shlo_i128(&res, lhs, (uint8_t)zig_lo_u128(rhs), bits)) return res;
1862 return zig_cmp_i128(lhs, zig_make_i128(0, 0)) < INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits);1866 switch (zig_cmp_i128(lhs, zig_make_i128(0, 0))) {
1867 case -1: return zig_minInt_i(128, bits);
1868 case 0: return zig_make_i128(0, 0);
1869 case 1: return zig_maxInt_i(128, bits);
1870 default: zig_unreachable();
1871 }
1863}1872}
18641873
1865static inline zig_u128 zig_adds_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) {1874static inline zig_u128 zig_adds_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) {
src/Air.zig+7-1
...@@ -257,7 +257,9 @@ pub const Inst = struct {...@@ -257,7 +257,9 @@ pub const Inst = struct {
257 /// it shifts out any bits that disagree with the resultant sign bit.257 /// it shifts out any bits that disagree with the resultant sign bit.
258 /// Uses the `bin_op` field.258 /// Uses the `bin_op` field.
259 shl_exact,259 shl_exact,
260 /// Saturating integer shift left. `<<|`260 /// Saturating integer shift left. `<<|`. The result is the same type as the `lhs`.
261 /// The `rhs` must have the same vector shape as the `lhs`, but with any unsigned
262 /// integer as the scalar type.
261 /// Uses the `bin_op` field.263 /// Uses the `bin_op` field.
262 shl_sat,264 shl_sat,
263 /// Bitwise XOR. `^`265 /// Bitwise XOR. `^`
...@@ -995,6 +997,7 @@ pub const Inst = struct {...@@ -995,6 +997,7 @@ pub const Inst = struct {
995 single_const_pointer_to_comptime_int_type = @intFromEnum(InternPool.Index.single_const_pointer_to_comptime_int_type),997 single_const_pointer_to_comptime_int_type = @intFromEnum(InternPool.Index.single_const_pointer_to_comptime_int_type),
996 slice_const_u8_type = @intFromEnum(InternPool.Index.slice_const_u8_type),998 slice_const_u8_type = @intFromEnum(InternPool.Index.slice_const_u8_type),
997 slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type),999 slice_const_u8_sentinel_0_type = @intFromEnum(InternPool.Index.slice_const_u8_sentinel_0_type),
1000 vector_8_i8_type = @intFromEnum(InternPool.Index.vector_8_i8_type),
998 vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type),1001 vector_16_i8_type = @intFromEnum(InternPool.Index.vector_16_i8_type),
999 vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type),1002 vector_32_i8_type = @intFromEnum(InternPool.Index.vector_32_i8_type),
1000 vector_1_u8_type = @intFromEnum(InternPool.Index.vector_1_u8_type),1003 vector_1_u8_type = @intFromEnum(InternPool.Index.vector_1_u8_type),
...@@ -1003,8 +1006,10 @@ pub const Inst = struct {...@@ -1003,8 +1006,10 @@ pub const Inst = struct {
1003 vector_8_u8_type = @intFromEnum(InternPool.Index.vector_8_u8_type),1006 vector_8_u8_type = @intFromEnum(InternPool.Index.vector_8_u8_type),
1004 vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type),1007 vector_16_u8_type = @intFromEnum(InternPool.Index.vector_16_u8_type),
1005 vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type),1008 vector_32_u8_type = @intFromEnum(InternPool.Index.vector_32_u8_type),
1009 vector_4_i16_type = @intFromEnum(InternPool.Index.vector_4_i16_type),
1006 vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type),1010 vector_8_i16_type = @intFromEnum(InternPool.Index.vector_8_i16_type),
1007 vector_16_i16_type = @intFromEnum(InternPool.Index.vector_16_i16_type),1011 vector_16_i16_type = @intFromEnum(InternPool.Index.vector_16_i16_type),
1012 vector_4_u16_type = @intFromEnum(InternPool.Index.vector_4_u16_type),
1008 vector_8_u16_type = @intFromEnum(InternPool.Index.vector_8_u16_type),1013 vector_8_u16_type = @intFromEnum(InternPool.Index.vector_8_u16_type),
1009 vector_16_u16_type = @intFromEnum(InternPool.Index.vector_16_u16_type),1014 vector_16_u16_type = @intFromEnum(InternPool.Index.vector_16_u16_type),
1010 vector_4_i32_type = @intFromEnum(InternPool.Index.vector_4_i32_type),1015 vector_4_i32_type = @intFromEnum(InternPool.Index.vector_4_i32_type),
...@@ -1015,6 +1020,7 @@ pub const Inst = struct {...@@ -1015,6 +1020,7 @@ pub const Inst = struct {
1015 vector_4_i64_type = @intFromEnum(InternPool.Index.vector_4_i64_type),1020 vector_4_i64_type = @intFromEnum(InternPool.Index.vector_4_i64_type),
1016 vector_2_u64_type = @intFromEnum(InternPool.Index.vector_2_u64_type),1021 vector_2_u64_type = @intFromEnum(InternPool.Index.vector_2_u64_type),
1017 vector_4_u64_type = @intFromEnum(InternPool.Index.vector_4_u64_type),1022 vector_4_u64_type = @intFromEnum(InternPool.Index.vector_4_u64_type),
1023 vector_2_u128_type = @intFromEnum(InternPool.Index.vector_2_u128_type),
1018 vector_4_f16_type = @intFromEnum(InternPool.Index.vector_4_f16_type),1024 vector_4_f16_type = @intFromEnum(InternPool.Index.vector_4_f16_type),
1019 vector_8_f16_type = @intFromEnum(InternPool.Index.vector_8_f16_type),1025 vector_8_f16_type = @intFromEnum(InternPool.Index.vector_8_f16_type),
1020 vector_2_f32_type = @intFromEnum(InternPool.Index.vector_2_f32_type),1026 vector_2_f32_type = @intFromEnum(InternPool.Index.vector_2_f32_type),
src/InternPool.zig+20
...@@ -4572,6 +4572,7 @@ pub const Index = enum(u32) {...@@ -4572,6 +4572,7 @@ pub const Index = enum(u32) {
4572 slice_const_u8_type,4572 slice_const_u8_type,
4573 slice_const_u8_sentinel_0_type,4573 slice_const_u8_sentinel_0_type,
45744574
4575 vector_8_i8_type,
4575 vector_16_i8_type,4576 vector_16_i8_type,
4576 vector_32_i8_type,4577 vector_32_i8_type,
4577 vector_1_u8_type,4578 vector_1_u8_type,
...@@ -4580,8 +4581,10 @@ pub const Index = enum(u32) {...@@ -4580,8 +4581,10 @@ pub const Index = enum(u32) {
4580 vector_8_u8_type,4581 vector_8_u8_type,
4581 vector_16_u8_type,4582 vector_16_u8_type,
4582 vector_32_u8_type,4583 vector_32_u8_type,
4584 vector_4_i16_type,
4583 vector_8_i16_type,4585 vector_8_i16_type,
4584 vector_16_i16_type,4586 vector_16_i16_type,
4587 vector_4_u16_type,
4585 vector_8_u16_type,4588 vector_8_u16_type,
4586 vector_16_u16_type,4589 vector_16_u16_type,
4587 vector_4_i32_type,4590 vector_4_i32_type,
...@@ -4592,6 +4595,7 @@ pub const Index = enum(u32) {...@@ -4592,6 +4595,7 @@ pub const Index = enum(u32) {
4592 vector_4_i64_type,4595 vector_4_i64_type,
4593 vector_2_u64_type,4596 vector_2_u64_type,
4594 vector_4_u64_type,4597 vector_4_u64_type,
4598 vector_2_u128_type,
4595 vector_4_f16_type,4599 vector_4_f16_type,
4596 vector_8_f16_type,4600 vector_8_f16_type,
4597 vector_2_f32_type,4601 vector_2_f32_type,
...@@ -5090,6 +5094,8 @@ pub const static_keys = [_]Key{...@@ -5090,6 +5094,8 @@ pub const static_keys = [_]Key{
5090 },5094 },
5091 } },5095 } },
50925096
5097 // @Vector(8, i8)
5098 .{ .vector_type = .{ .len = 8, .child = .i8_type } },
5093 // @Vector(16, i8)5099 // @Vector(16, i8)
5094 .{ .vector_type = .{ .len = 16, .child = .i8_type } },5100 .{ .vector_type = .{ .len = 16, .child = .i8_type } },
5095 // @Vector(32, i8)5101 // @Vector(32, i8)
...@@ -5106,10 +5112,14 @@ pub const static_keys = [_]Key{...@@ -5106,10 +5112,14 @@ pub const static_keys = [_]Key{
5106 .{ .vector_type = .{ .len = 16, .child = .u8_type } },5112 .{ .vector_type = .{ .len = 16, .child = .u8_type } },
5107 // @Vector(32, u8)5113 // @Vector(32, u8)
5108 .{ .vector_type = .{ .len = 32, .child = .u8_type } },5114 .{ .vector_type = .{ .len = 32, .child = .u8_type } },
5115 // @Vector(4, i16)
5116 .{ .vector_type = .{ .len = 4, .child = .i16_type } },
5109 // @Vector(8, i16)5117 // @Vector(8, i16)
5110 .{ .vector_type = .{ .len = 8, .child = .i16_type } },5118 .{ .vector_type = .{ .len = 8, .child = .i16_type } },
5111 // @Vector(16, i16)5119 // @Vector(16, i16)
5112 .{ .vector_type = .{ .len = 16, .child = .i16_type } },5120 .{ .vector_type = .{ .len = 16, .child = .i16_type } },
5121 // @Vector(4, u16)
5122 .{ .vector_type = .{ .len = 4, .child = .u16_type } },
5113 // @Vector(8, u16)5123 // @Vector(8, u16)
5114 .{ .vector_type = .{ .len = 8, .child = .u16_type } },5124 .{ .vector_type = .{ .len = 8, .child = .u16_type } },
5115 // @Vector(16, u16)5125 // @Vector(16, u16)
...@@ -5130,6 +5140,8 @@ pub const static_keys = [_]Key{...@@ -5130,6 +5140,8 @@ pub const static_keys = [_]Key{
5130 .{ .vector_type = .{ .len = 2, .child = .u64_type } },5140 .{ .vector_type = .{ .len = 2, .child = .u64_type } },
5131 // @Vector(8, u64)5141 // @Vector(8, u64)
5132 .{ .vector_type = .{ .len = 4, .child = .u64_type } },5142 .{ .vector_type = .{ .len = 4, .child = .u64_type } },
5143 // @Vector(2, u128)
5144 .{ .vector_type = .{ .len = 2, .child = .u128_type } },
5133 // @Vector(4, f16)5145 // @Vector(4, f16)
5134 .{ .vector_type = .{ .len = 4, .child = .f16_type } },5146 .{ .vector_type = .{ .len = 4, .child = .f16_type } },
5135 // @Vector(8, f16)5147 // @Vector(8, f16)
...@@ -11777,6 +11789,7 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {...@@ -11777,6 +11789,7 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
11777 .single_const_pointer_to_comptime_int_type,11789 .single_const_pointer_to_comptime_int_type,
11778 .slice_const_u8_type,11790 .slice_const_u8_type,
11779 .slice_const_u8_sentinel_0_type,11791 .slice_const_u8_sentinel_0_type,
11792 .vector_8_i8_type,
11780 .vector_16_i8_type,11793 .vector_16_i8_type,
11781 .vector_32_i8_type,11794 .vector_32_i8_type,
11782 .vector_1_u8_type,11795 .vector_1_u8_type,
...@@ -11785,8 +11798,10 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {...@@ -11785,8 +11798,10 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
11785 .vector_8_u8_type,11798 .vector_8_u8_type,
11786 .vector_16_u8_type,11799 .vector_16_u8_type,
11787 .vector_32_u8_type,11800 .vector_32_u8_type,
11801 .vector_4_i16_type,
11788 .vector_8_i16_type,11802 .vector_8_i16_type,
11789 .vector_16_i16_type,11803 .vector_16_i16_type,
11804 .vector_4_u16_type,
11790 .vector_8_u16_type,11805 .vector_8_u16_type,
11791 .vector_16_u16_type,11806 .vector_16_u16_type,
11792 .vector_4_i32_type,11807 .vector_4_i32_type,
...@@ -11797,6 +11812,7 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {...@@ -11797,6 +11812,7 @@ pub fn typeOf(ip: *const InternPool, index: Index) Index {
11797 .vector_4_i64_type,11812 .vector_4_i64_type,
11798 .vector_2_u64_type,11813 .vector_2_u64_type,
11799 .vector_4_u64_type,11814 .vector_4_u64_type,
11815 .vector_2_u128_type,
11800 .vector_4_f16_type,11816 .vector_4_f16_type,
11801 .vector_8_f16_type,11817 .vector_8_f16_type,
11802 .vector_2_f32_type,11818 .vector_2_f32_type,
...@@ -12121,6 +12137,7 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {...@@ -12121,6 +12137,7 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
12121 .slice_const_u8_sentinel_0_type,12137 .slice_const_u8_sentinel_0_type,
12122 => .pointer,12138 => .pointer,
1212312139
12140 .vector_8_i8_type,
12124 .vector_16_i8_type,12141 .vector_16_i8_type,
12125 .vector_32_i8_type,12142 .vector_32_i8_type,
12126 .vector_1_u8_type,12143 .vector_1_u8_type,
...@@ -12129,8 +12146,10 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {...@@ -12129,8 +12146,10 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
12129 .vector_8_u8_type,12146 .vector_8_u8_type,
12130 .vector_16_u8_type,12147 .vector_16_u8_type,
12131 .vector_32_u8_type,12148 .vector_32_u8_type,
12149 .vector_4_i16_type,
12132 .vector_8_i16_type,12150 .vector_8_i16_type,
12133 .vector_16_i16_type,12151 .vector_16_i16_type,
12152 .vector_4_u16_type,
12134 .vector_8_u16_type,12153 .vector_8_u16_type,
12135 .vector_16_u16_type,12154 .vector_16_u16_type,
12136 .vector_4_i32_type,12155 .vector_4_i32_type,
...@@ -12141,6 +12160,7 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {...@@ -12141,6 +12160,7 @@ pub fn zigTypeTag(ip: *const InternPool, index: Index) std.builtin.TypeId {
12141 .vector_4_i64_type,12160 .vector_4_i64_type,
12142 .vector_2_u64_type,12161 .vector_2_u64_type,
12143 .vector_4_u64_type,12162 .vector_4_u64_type,
12163 .vector_2_u128_type,
12144 .vector_4_f16_type,12164 .vector_4_f16_type,
12145 .vector_8_f16_type,12165 .vector_8_f16_type,
12146 .vector_2_f32_type,12166 .vector_2_f32_type,
src/Sema.zig+47-28
...@@ -14215,14 +14215,15 @@ fn zirShl(...@@ -14215,14 +14215,15 @@ fn zirShl(
14215 const rhs_ty = sema.typeOf(rhs);14215 const rhs_ty = sema.typeOf(rhs);
1421614216
14217 const src = block.nodeOffset(inst_data.src_node);14217 const src = block.nodeOffset(inst_data.src_node);
14218 const lhs_src = switch (air_tag) {14218 const lhs_src, const rhs_src = switch (air_tag) {
14219 .shl, .shl_sat => block.src(.{ .node_offset_bin_lhs = inst_data.src_node }),14219 .shl, .shl_sat => .{
14220 .shl_exact => block.builtinCallArgSrc(inst_data.src_node, 0),14220 block.src(.{ .node_offset_bin_lhs = inst_data.src_node }),
14221 else => unreachable,14221 block.src(.{ .node_offset_bin_rhs = inst_data.src_node }),
14222 };14222 },
14223 const rhs_src = switch (air_tag) {14223 .shl_exact => .{
14224 .shl, .shl_sat => block.src(.{ .node_offset_bin_rhs = inst_data.src_node }),14224 block.builtinCallArgSrc(inst_data.src_node, 0),
14225 .shl_exact => block.builtinCallArgSrc(inst_data.src_node, 1),14225 block.builtinCallArgSrc(inst_data.src_node, 1),
14226 },
14226 else => unreachable,14227 else => unreachable,
14227 };14228 };
1422814229
...@@ -14231,8 +14232,7 @@ fn zirShl(...@@ -14231,8 +14232,7 @@ fn zirShl(
14231 const scalar_ty = lhs_ty.scalarType(zcu);14232 const scalar_ty = lhs_ty.scalarType(zcu);
14232 const scalar_rhs_ty = rhs_ty.scalarType(zcu);14233 const scalar_rhs_ty = rhs_ty.scalarType(zcu);
1423314234
14234 // TODO coerce rhs if air_tag is not shl_sat14235 _ = try sema.checkIntType(block, rhs_src, scalar_rhs_ty);
14235 const rhs_is_comptime_int = try sema.checkIntType(block, rhs_src, scalar_rhs_ty);
1423614236
14237 const maybe_lhs_val = try sema.resolveValueResolveLazy(lhs);14237 const maybe_lhs_val = try sema.resolveValueResolveLazy(lhs);
14238 const maybe_rhs_val = try sema.resolveValueResolveLazy(rhs);14238 const maybe_rhs_val = try sema.resolveValueResolveLazy(rhs);
...@@ -14245,7 +14245,7 @@ fn zirShl(...@@ -14245,7 +14245,7 @@ fn zirShl(
14245 if (try rhs_val.compareAllWithZeroSema(.eq, pt)) {14245 if (try rhs_val.compareAllWithZeroSema(.eq, pt)) {
14246 return lhs;14246 return lhs;
14247 }14247 }
14248 if (scalar_ty.zigTypeTag(zcu) != .comptime_int and air_tag != .shl_sat) {14248 if (air_tag != .shl_sat and scalar_ty.zigTypeTag(zcu) != .comptime_int) {
14249 const bit_value = try pt.intValue(Type.comptime_int, scalar_ty.intInfo(zcu).bits);14249 const bit_value = try pt.intValue(Type.comptime_int, scalar_ty.intInfo(zcu).bits);
14250 if (rhs_ty.zigTypeTag(zcu) == .vector) {14250 if (rhs_ty.zigTypeTag(zcu) == .vector) {
14251 var i: usize = 0;14251 var i: usize = 0;
...@@ -14282,6 +14282,8 @@ fn zirShl(...@@ -14282,6 +14282,8 @@ fn zirShl(
14282 rhs_val.fmtValueSema(pt, sema),14282 rhs_val.fmtValueSema(pt, sema),
14283 });14283 });
14284 }14284 }
14285 } else if (scalar_rhs_ty.isSignedInt(zcu)) {
14286 return sema.fail(block, rhs_src, "shift by signed type '{}'", .{rhs_ty.fmt(pt)});
14285 }14287 }
1428614288
14287 const runtime_src = if (maybe_lhs_val) |lhs_val| rs: {14289 const runtime_src = if (maybe_lhs_val) |lhs_val| rs: {
...@@ -14309,18 +14311,34 @@ fn zirShl(...@@ -14309,18 +14311,34 @@ fn zirShl(
14309 return Air.internedToRef(val.toIntern());14311 return Air.internedToRef(val.toIntern());
14310 } else lhs_src;14312 } else lhs_src;
1431114313
14312 const new_rhs = if (air_tag == .shl_sat) rhs: {14314 const rt_rhs = switch (air_tag) {
14313 // Limit the RHS type for saturating shl to be an integer as small as the LHS.14315 else => unreachable,
14314 if (rhs_is_comptime_int or14316 .shl, .shl_exact => rhs,
14315 scalar_rhs_ty.intInfo(zcu).bits > scalar_ty.intInfo(zcu).bits)14317 // The backend can handle a large runtime rhs better than we can, but
14316 {14318 // we can limit a large comptime rhs better here. This also has the
14317 const max_int = Air.internedToRef((try lhs_ty.maxInt(pt, lhs_ty)).toIntern());14319 // necessary side effect of preventing rhs from being a `comptime_int`.
14318 const rhs_limited = try sema.analyzeMinMax(block, rhs_src, .min, &.{ rhs, max_int }, &.{ rhs_src, rhs_src });14320 .shl_sat => if (maybe_rhs_val) |rhs_val| Air.internedToRef(rt_rhs: {
14319 break :rhs try sema.intCast(block, src, lhs_ty, rhs_src, rhs_limited, rhs_src, false, false);14321 const bit_count = scalar_ty.intInfo(zcu).bits;
14320 } else {14322 const rt_rhs_scalar_ty = try pt.smallestUnsignedInt(bit_count);
14321 break :rhs rhs;14323 if (!rhs_ty.isVector(zcu)) break :rt_rhs (try pt.intValue(
14322 }14324 rt_rhs_scalar_ty,
14323 } else rhs;14325 @min(try rhs_val.getUnsignedIntSema(pt) orelse bit_count, bit_count),
14326 )).toIntern();
14327 const rhs_len = rhs_ty.vectorLen(zcu);
14328 const rhs_elems = try sema.arena.alloc(InternPool.Index, rhs_len);
14329 for (rhs_elems, 0..) |*rhs_elem, i| rhs_elem.* = (try pt.intValue(
14330 rt_rhs_scalar_ty,
14331 @min(try (try rhs_val.elemValue(pt, i)).getUnsignedIntSema(pt) orelse bit_count, bit_count),
14332 )).toIntern();
14333 break :rt_rhs try pt.intern(.{ .aggregate = .{
14334 .ty = (try pt.vectorType(.{
14335 .len = rhs_len,
14336 .child = rt_rhs_scalar_ty.toIntern(),
14337 })).toIntern(),
14338 .storage = .{ .elems = rhs_elems },
14339 } });
14340 }) else rhs,
14341 };
1432414342
14325 try sema.requireRuntimeBlock(block, src, runtime_src);14343 try sema.requireRuntimeBlock(block, src, runtime_src);
14326 if (block.wantSafety()) {14344 if (block.wantSafety()) {
...@@ -14374,7 +14392,7 @@ fn zirShl(...@@ -14374,7 +14392,7 @@ fn zirShl(
14374 return sema.tupleFieldValByIndex(block, op_ov, 0, op_ov_tuple_ty);14392 return sema.tupleFieldValByIndex(block, op_ov, 0, op_ov_tuple_ty);
14375 }14393 }
14376 }14394 }
14377 return block.addBinOp(air_tag, lhs, new_rhs);14395 return block.addBinOp(air_tag, lhs, rt_rhs);
14378}14396}
1437914397
14380fn zirShr(14398fn zirShr(
...@@ -36432,10 +36450,7 @@ fn generateUnionTagTypeSimple(...@@ -36432,10 +36450,7 @@ fn generateUnionTagTypeSimple(
36432 const enum_ty = try ip.getGeneratedTagEnumType(gpa, pt.tid, .{36450 const enum_ty = try ip.getGeneratedTagEnumType(gpa, pt.tid, .{
36433 .name = name,36451 .name = name,
36434 .owner_union_ty = union_type,36452 .owner_union_ty = union_type,
36435 .tag_ty = if (enum_field_names.len == 0)36453 .tag_ty = (try pt.smallestUnsignedInt(enum_field_names.len -| 1)).toIntern(),
36436 (try pt.intType(.unsigned, 0)).toIntern()
36437 else
36438 (try pt.smallestUnsignedInt(enum_field_names.len - 1)).toIntern(),
36439 .names = enum_field_names,36454 .names = enum_field_names,
36440 .values = &.{},36455 .values = &.{},
36441 .tag_mode = .auto,36456 .tag_mode = .auto,
...@@ -36502,6 +36517,7 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {...@@ -36502,6 +36517,7 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
36502 .single_const_pointer_to_comptime_int_type,36517 .single_const_pointer_to_comptime_int_type,
36503 .slice_const_u8_type,36518 .slice_const_u8_type,
36504 .slice_const_u8_sentinel_0_type,36519 .slice_const_u8_sentinel_0_type,
36520 .vector_8_i8_type,
36505 .vector_16_i8_type,36521 .vector_16_i8_type,
36506 .vector_32_i8_type,36522 .vector_32_i8_type,
36507 .vector_1_u8_type,36523 .vector_1_u8_type,
...@@ -36510,8 +36526,10 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {...@@ -36510,8 +36526,10 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
36510 .vector_8_u8_type,36526 .vector_8_u8_type,
36511 .vector_16_u8_type,36527 .vector_16_u8_type,
36512 .vector_32_u8_type,36528 .vector_32_u8_type,
36529 .vector_4_i16_type,
36513 .vector_8_i16_type,36530 .vector_8_i16_type,
36514 .vector_16_i16_type,36531 .vector_16_i16_type,
36532 .vector_4_u16_type,
36515 .vector_8_u16_type,36533 .vector_8_u16_type,
36516 .vector_16_u16_type,36534 .vector_16_u16_type,
36517 .vector_4_i32_type,36535 .vector_4_i32_type,
...@@ -36522,6 +36540,7 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {...@@ -36522,6 +36540,7 @@ pub fn typeHasOnePossibleValue(sema: *Sema, ty: Type) CompileError!?Value {
36522 .vector_4_i64_type,36540 .vector_4_i64_type,
36523 .vector_2_u64_type,36541 .vector_2_u64_type,
36524 .vector_4_u64_type,36542 .vector_4_u64_type,
36543 .vector_2_u128_type,
36525 .vector_4_f16_type,36544 .vector_4_f16_type,
36526 .vector_8_f16_type,36545 .vector_8_f16_type,
36527 .vector_2_f32_type,36546 .vector_2_f32_type,
src/Type.zig+8-4
...@@ -4096,6 +4096,7 @@ pub const single_const_pointer_to_comptime_int: Type = .{ .ip_index = .single_co...@@ -4096,6 +4096,7 @@ pub const single_const_pointer_to_comptime_int: Type = .{ .ip_index = .single_co
4096pub const slice_const_u8: Type = .{ .ip_index = .slice_const_u8_type };4096pub const slice_const_u8: Type = .{ .ip_index = .slice_const_u8_type };
4097pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_sentinel_0_type };4097pub const slice_const_u8_sentinel_0: Type = .{ .ip_index = .slice_const_u8_sentinel_0_type };
40984098
4099pub const vector_8_i8: Type = .{ .ip_index = .vector_8_i8_type };
4099pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };4100pub const vector_16_i8: Type = .{ .ip_index = .vector_16_i8_type };
4100pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type };4101pub const vector_32_i8: Type = .{ .ip_index = .vector_32_i8_type };
4101pub const vector_1_u8: Type = .{ .ip_index = .vector_1_u8_type };4102pub const vector_1_u8: Type = .{ .ip_index = .vector_1_u8_type };
...@@ -4104,8 +4105,10 @@ pub const vector_4_u8: Type = .{ .ip_index = .vector_4_u8_type };...@@ -4104,8 +4105,10 @@ pub const vector_4_u8: Type = .{ .ip_index = .vector_4_u8_type };
4104pub const vector_8_u8: Type = .{ .ip_index = .vector_8_u8_type };4105pub const vector_8_u8: Type = .{ .ip_index = .vector_8_u8_type };
4105pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type };4106pub const vector_16_u8: Type = .{ .ip_index = .vector_16_u8_type };
4106pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type };4107pub const vector_32_u8: Type = .{ .ip_index = .vector_32_u8_type };
4108pub const vector_4_i16: Type = .{ .ip_index = .vector_4_i16_type };
4107pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type };4109pub const vector_8_i16: Type = .{ .ip_index = .vector_8_i16_type };
4108pub const vector_16_i16: Type = .{ .ip_index = .vector_16_i16_type };4110pub const vector_16_i16: Type = .{ .ip_index = .vector_16_i16_type };
4111pub const vector_4_u16: Type = .{ .ip_index = .vector_4_u16_type };
4109pub const vector_8_u16: Type = .{ .ip_index = .vector_8_u16_type };4112pub const vector_8_u16: Type = .{ .ip_index = .vector_8_u16_type };
4110pub const vector_16_u16: Type = .{ .ip_index = .vector_16_u16_type };4113pub const vector_16_u16: Type = .{ .ip_index = .vector_16_u16_type };
4111pub const vector_4_i32: Type = .{ .ip_index = .vector_4_i32_type };4114pub const vector_4_i32: Type = .{ .ip_index = .vector_4_i32_type };
...@@ -4116,6 +4119,7 @@ pub const vector_2_i64: Type = .{ .ip_index = .vector_2_i64_type };...@@ -4116,6 +4119,7 @@ pub const vector_2_i64: Type = .{ .ip_index = .vector_2_i64_type };
4116pub const vector_4_i64: Type = .{ .ip_index = .vector_4_i64_type };4119pub const vector_4_i64: Type = .{ .ip_index = .vector_4_i64_type };
4117pub const vector_2_u64: Type = .{ .ip_index = .vector_2_u64_type };4120pub const vector_2_u64: Type = .{ .ip_index = .vector_2_u64_type };
4118pub const vector_4_u64: Type = .{ .ip_index = .vector_4_u64_type };4121pub const vector_4_u64: Type = .{ .ip_index = .vector_4_u64_type };
4122pub const vector_2_u128: Type = .{ .ip_index = .vector_2_u128_type };
4119pub const vector_4_f16: Type = .{ .ip_index = .vector_4_f16_type };4123pub const vector_4_f16: Type = .{ .ip_index = .vector_4_f16_type };
4120pub const vector_8_f16: Type = .{ .ip_index = .vector_8_f16_type };4124pub const vector_8_f16: Type = .{ .ip_index = .vector_8_f16_type };
4121pub const vector_2_f32: Type = .{ .ip_index = .vector_2_f32_type };4125pub const vector_2_f32: Type = .{ .ip_index = .vector_2_f32_type };
...@@ -4129,10 +4133,10 @@ pub const empty_tuple: Type = .{ .ip_index = .empty_tuple_type };...@@ -4129,10 +4133,10 @@ pub const empty_tuple: Type = .{ .ip_index = .empty_tuple_type };
4129pub const generic_poison: Type = .{ .ip_index = .generic_poison_type };4133pub const generic_poison: Type = .{ .ip_index = .generic_poison_type };
41304134
4131pub fn smallestUnsignedBits(max: u64) u16 {4135pub fn smallestUnsignedBits(max: u64) u16 {
4132 if (max == 0) return 0;4136 return switch (max) {
4133 const base = std.math.log2(max);4137 0 => 0,
4134 const upper = (@as(u64, 1) << @as(u6, @intCast(base))) - 1;4138 else => 1 + std.math.log2_int(u64, max),
4135 return @as(u16, @intCast(base + @intFromBool(upper < max)));4139 };
4136}4140}
41374141
4138/// This is only used for comptime asserts. Bump this number when you make a change4142/// This is only used for comptime asserts. Bump this number when you make a change
src/arch/x86_64/CodeGen.zig+88977-59046
...@@ -73,10 +73,6 @@ mir_instructions: std.MultiArrayList(Mir.Inst) = .empty,...@@ -73,10 +73,6 @@ mir_instructions: std.MultiArrayList(Mir.Inst) = .empty,
73mir_extra: std.ArrayListUnmanaged(u32) = .empty,73mir_extra: std.ArrayListUnmanaged(u32) = .empty,
74mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,74mir_table: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty,
7575
76/// Byte offset within the source file of the ending curly.
77end_di_line: u32,
78end_di_column: u32,
79
80/// The value is an offset into the `Function` `code` from the beginning.76/// The value is an offset into the `Function` `code` from the beginning.
81/// To perform the reloc, write 32-bit signed little-endian integer77/// To perform the reloc, write 32-bit signed little-endian integer
82/// which is a relative jump, based on the address following the reloc.78/// which is a relative jump, based on the address following the reloc.
...@@ -490,7 +486,7 @@ pub const MCValue = union(enum) {...@@ -490,7 +486,7 @@ pub const MCValue = union(enum) {
490 } },486 } },
491 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },487 } else .{ .base = .{ .reg = .ds }, .mod = .{ .off = addr } },
492 .indirect => |reg_off| .{488 .indirect => |reg_off| .{
493 .base = .{ .reg = registerAlias(reg_off.reg, @divExact(function.target.ptrBitWidth(), 8)) },489 .base = .{ .reg = reg_off.reg.toSize(.ptr, function.target) },
494 .mod = .{ .rm = .{490 .mod = .{ .rm = .{
495 .size = mod_rm.size,491 .size = mod_rm.size,
496 .index = mod_rm.index,492 .index = mod_rm.index,
...@@ -912,8 +908,6 @@ pub fn generate(...@@ -912,8 +908,6 @@ pub fn generate(
912 .err_ret_trace_reg = undefined, // populated after `resolveCallingConventionValues`908 .err_ret_trace_reg = undefined, // populated after `resolveCallingConventionValues`
913 .fn_type = fn_type,909 .fn_type = fn_type,
914 .src_loc = src_loc,910 .src_loc = src_loc,
915 .end_di_line = func.rbrace_line,
916 .end_di_column = func.rbrace_column,
917 };911 };
918 defer {912 defer {
919 function.frame_allocs.deinit(gpa);913 function.frame_allocs.deinit(gpa);
...@@ -1000,6 +994,16 @@ pub fn generate(...@@ -1000,6 +994,16 @@ pub fn generate(
1000 else => |e| return e,994 else => |e| return e,
1001 };995 };
1002996
997 // Drop them off at the rbrace.
998 if (debug_output != .none) _ = try function.addInst(.{
999 .tag = .pseudo,
1000 .ops = .pseudo_dbg_line_line_column,
1001 .data = .{ .line_column = .{
1002 .line = func.rbrace_line,
1003 .column = func.rbrace_column,
1004 } },
1005 });
1006
1003 var mir: Mir = .{1007 var mir: Mir = .{
1004 .instructions = function.mir_instructions.toOwnedSlice(),1008 .instructions = function.mir_instructions.toOwnedSlice(),
1005 .extra = try function.mir_extra.toOwnedSlice(gpa),1009 .extra = try function.mir_extra.toOwnedSlice(gpa),
...@@ -1076,8 +1080,6 @@ pub fn generateLazy(...@@ -1076,8 +1080,6 @@ pub fn generateLazy(
1076 .err_ret_trace_reg = undefined,1080 .err_ret_trace_reg = undefined,
1077 .fn_type = undefined,1081 .fn_type = undefined,
1078 .src_loc = src_loc,1082 .src_loc = src_loc,
1079 .end_di_line = undefined, // no debug info yet
1080 .end_di_column = undefined, // no debug info yet
1081 };1083 };
1082 defer {1084 defer {
1083 function.inst_tracking.deinit(gpa);1085 function.inst_tracking.deinit(gpa);
...@@ -2381,16 +2383,6 @@ fn gen(self: *CodeGen) InnerError!void {...@@ -2381,16 +2383,6 @@ fn gen(self: *CodeGen) InnerError!void {
2381 try self.genBody(self.air.getMainBody());2383 try self.genBody(self.air.getMainBody());
2382 if (self.debug_output != .none) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);2384 if (self.debug_output != .none) try self.asmPseudo(.pseudo_dbg_epilogue_begin_none);
2383 }2385 }
2384
2385 // Drop them off at the rbrace.
2386 if (self.debug_output != .none) _ = try self.addInst(.{
2387 .tag = .pseudo,
2388 .ops = .pseudo_dbg_line_line_column,
2389 .data = .{ .line_column = .{
2390 .line = self.end_di_line,
2391 .column = self.end_di_column,
2392 } },
2393 });
2394}2386}
23952387
2396fn checkInvariantsAfterAirInst(self: *CodeGen) void {2388fn checkInvariantsAfterAirInst(self: *CodeGen) void {
...@@ -2416,7 +2408,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2416,7 +2408,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2416}2408}
24172409
2418fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {2410fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2419 @setEvalBranchQuota(15_100);2411 @setEvalBranchQuota(20_200);
2420 const pt = cg.pt;2412 const pt = cg.pt;
2421 const zcu = pt.zcu;2413 const zcu = pt.zcu;
2422 const ip = &zcu.intern_pool;2414 const ip = &zcu.intern_pool;
...@@ -2452,16 +2444,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2452,16 +2444,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2452 try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);2444 try cg.inst_tracking.ensureUnusedCapacity(cg.gpa, 1);
2453 switch (air_tags[@intFromEnum(inst)]) {2445 switch (air_tags[@intFromEnum(inst)]) {
2454 // zig fmt: off2446 // zig fmt: off
2455 .add_sat => try cg.airAddSat(inst),
2456 .sub_sat => try cg.airSubSat(inst),
2457 .mul_sat => try cg.airMulSat(inst),
2458 .shl_sat => try cg.airShlSat(inst),
2459
2460 .shl_with_overflow => try cg.airShlWithOverflow(inst),
2461
2462 .bitcast => try cg.airBitCast(inst),
2463
2464 .splat => try cg.airSplat(inst),
2465 .select => try cg.airSelect(inst),2447 .select => try cg.airSelect(inst),
2466 .shuffle => try cg.airShuffle(inst),2448 .shuffle => try cg.airShuffle(inst),
2467 .reduce => try cg.airReduce(inst),2449 .reduce => try cg.airReduce(inst),
...@@ -2702,6 +2684,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2702,6 +2684,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2702 .unused,2684 .unused,
2703 },2685 },
2704 .dst_temps = .{ .mem, .unused },2686 .dst_temps = .{ .mem, .unused },
2687 .clobbers = .{ .eflags = true },
2705 .each = .{ .once = &.{2688 .each = .{ .once = &.{
2706 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2689 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2707 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },2690 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2734,6 +2717,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2734,6 +2717,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2734 .unused,2717 .unused,
2735 },2718 },
2736 .dst_temps = .{ .mem, .unused },2719 .dst_temps = .{ .mem, .unused },
2720 .clobbers = .{ .eflags = true },
2737 .each = .{ .once = &.{2721 .each = .{ .once = &.{
2738 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2722 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2739 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },2723 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2766,6 +2750,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2766,6 +2750,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2766 .unused,2750 .unused,
2767 },2751 },
2768 .dst_temps = .{ .mem, .unused },2752 .dst_temps = .{ .mem, .unused },
2753 .clobbers = .{ .eflags = true },
2769 .each = .{ .once = &.{2754 .each = .{ .once = &.{
2770 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2755 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2771 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },2756 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2798,6 +2783,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2798,6 +2783,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2798 .unused,2783 .unused,
2799 },2784 },
2800 .dst_temps = .{ .mem, .unused },2785 .dst_temps = .{ .mem, .unused },
2786 .clobbers = .{ .eflags = true },
2801 .each = .{ .once = &.{2787 .each = .{ .once = &.{
2802 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2788 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2803 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },2789 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2829,6 +2815,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2829,6 +2815,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2829 .unused,2815 .unused,
2830 },2816 },
2831 .dst_temps = .{ .mem, .unused },2817 .dst_temps = .{ .mem, .unused },
2818 .clobbers = .{ .eflags = true },
2832 .each = .{ .once = &.{2819 .each = .{ .once = &.{
2833 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2820 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2834 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },2821 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2909,6 +2896,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2909,6 +2896,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2909 .unused,2896 .unused,
2910 },2897 },
2911 .dst_temps = .{ .mem, .unused },2898 .dst_temps = .{ .mem, .unused },
2899 .clobbers = .{ .eflags = true },
2912 .each = .{ .once = &.{2900 .each = .{ .once = &.{
2913 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2901 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2914 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },2902 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2941,6 +2929,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2941,6 +2929,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2941 .unused,2929 .unused,
2942 },2930 },
2943 .dst_temps = .{ .mem, .unused },2931 .dst_temps = .{ .mem, .unused },
2932 .clobbers = .{ .eflags = true },
2944 .each = .{ .once = &.{2933 .each = .{ .once = &.{
2945 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2934 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2946 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },2935 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -2973,6 +2962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -2973,6 +2962,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
2973 .unused,2962 .unused,
2974 },2963 },
2975 .dst_temps = .{ .mem, .unused },2964 .dst_temps = .{ .mem, .unused },
2965 .clobbers = .{ .eflags = true },
2976 .each = .{ .once = &.{2966 .each = .{ .once = &.{
2977 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2967 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
2978 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },2968 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3004,6 +2994,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3004,6 +2994,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3004 .unused,2994 .unused,
3005 },2995 },
3006 .dst_temps = .{ .mem, .unused },2996 .dst_temps = .{ .mem, .unused },
2997 .clobbers = .{ .eflags = true },
3007 .each = .{ .once = &.{2998 .each = .{ .once = &.{
3008 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },2999 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3009 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },3000 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3084,6 +3075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3084,6 +3075,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3084 .unused,3075 .unused,
3085 },3076 },
3086 .dst_temps = .{ .mem, .unused },3077 .dst_temps = .{ .mem, .unused },
3078 .clobbers = .{ .eflags = true },
3087 .each = .{ .once = &.{3079 .each = .{ .once = &.{
3088 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3080 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3089 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },3081 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3116,6 +3108,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3116,6 +3108,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3116 .unused,3108 .unused,
3117 },3109 },
3118 .dst_temps = .{ .mem, .unused },3110 .dst_temps = .{ .mem, .unused },
3111 .clobbers = .{ .eflags = true },
3119 .each = .{ .once = &.{3112 .each = .{ .once = &.{
3120 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3113 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3121 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },3114 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3148,6 +3141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3148,6 +3141,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3148 .unused,3141 .unused,
3149 },3142 },
3150 .dst_temps = .{ .mem, .unused },3143 .dst_temps = .{ .mem, .unused },
3144 .clobbers = .{ .eflags = true },
3151 .each = .{ .once = &.{3145 .each = .{ .once = &.{
3152 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3146 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3153 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },3147 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3179,6 +3173,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3179,6 +3173,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3179 .unused,3173 .unused,
3180 },3174 },
3181 .dst_temps = .{ .mem, .unused },3175 .dst_temps = .{ .mem, .unused },
3176 .clobbers = .{ .eflags = true },
3182 .each = .{ .once = &.{3177 .each = .{ .once = &.{
3183 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3178 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3184 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },3179 .{ .@"0:", ._, .mov, .tmp1d, .memia(.src0d, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3259,6 +3254,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3259,6 +3254,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3259 .unused,3254 .unused,
3260 },3255 },
3261 .dst_temps = .{ .mem, .unused },3256 .dst_temps = .{ .mem, .unused },
3257 .clobbers = .{ .eflags = true },
3262 .each = .{ .once = &.{3258 .each = .{ .once = &.{
3263 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3259 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3264 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },3260 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3291,6 +3287,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3291,6 +3287,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3291 .unused,3287 .unused,
3292 },3288 },
3293 .dst_temps = .{ .mem, .unused },3289 .dst_temps = .{ .mem, .unused },
3290 .clobbers = .{ .eflags = true },
3294 .each = .{ .once = &.{3291 .each = .{ .once = &.{
3295 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3292 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3296 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },3293 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3323,6 +3320,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3323,6 +3320,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3323 .unused,3320 .unused,
3324 },3321 },
3325 .dst_temps = .{ .mem, .unused },3322 .dst_temps = .{ .mem, .unused },
3323 .clobbers = .{ .eflags = true },
3326 .each = .{ .once = &.{3324 .each = .{ .once = &.{
3327 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3325 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3328 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },3326 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3355,6 +3353,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3355,6 +3353,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3355 .unused,3353 .unused,
3356 },3354 },
3357 .dst_temps = .{ .mem, .unused },3355 .dst_temps = .{ .mem, .unused },
3356 .clobbers = .{ .eflags = true },
3358 .each = .{ .once = &.{3357 .each = .{ .once = &.{
3359 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3358 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3360 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },3359 .{ .@"0:", ._, .mov, .tmp1q, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._ },
...@@ -3387,6 +3386,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3387,6 +3386,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3387 .unused,3386 .unused,
3388 },3387 },
3389 .dst_temps = .{ .mem, .unused },3388 .dst_temps = .{ .mem, .unused },
3389 .clobbers = .{ .eflags = true },
3390 .each = .{ .once = &.{3390 .each = .{ .once = &.{
3391 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3391 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3392 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },3392 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
...@@ -3425,6 +3425,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -3425,6 +3425,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
3425 .unused,3425 .unused,
3426 },3426 },
3427 .dst_temps = .{ .mem, .unused },3427 .dst_temps = .{ .mem, .unused },
3428 .clobbers = .{ .eflags = true },
3428 .each = .{ .once = &.{3429 .each = .{ .once = &.{
3429 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },3430 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
3430 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },3431 .{ .@"0:", ._, .lea, .tmp1p, .memia(.src0, .tmp0, .add_unaligned_size_add_elem_size), ._, ._ },
...@@ -4096,6 +4097,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4096,6 +4097,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4096 .unused,4097 .unused,
4097 },4098 },
4098 .dst_temps = .{ .mem, .unused },4099 .dst_temps = .{ .mem, .unused },
4100 .clobbers = .{ .eflags = true },
4099 .each = .{ .once = &.{4101 .each = .{ .once = &.{
4100 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4102 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4101 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },4103 .{ .@"0:", .f_, .ld, .memia(.src0q, .tmp0, .add_unaligned_size), ._, ._, ._ },
...@@ -4189,6 +4191,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4189,6 +4191,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4189 .unused,4191 .unused,
4190 },4192 },
4191 .dst_temps = .{ .mem, .unused },4193 .dst_temps = .{ .mem, .unused },
4194 .clobbers = .{ .eflags = true },
4192 .each = .{ .once = &.{4195 .each = .{ .once = &.{
4193 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4196 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4194 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },4197 .{ .@"0:", .f_, .ld, .memia(.src0t, .tmp0, .add_unaligned_size), ._, ._, ._ },
...@@ -4356,84 +4359,136 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4356,84 +4359,136 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4356 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);4359 try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg);
4357 },4360 },
4358 .add_safe => unreachable,4361 .add_safe => unreachable,
4359 .sub, .sub_optimized, .sub_wrap => |air_tag| if (use_old) try cg.airBinOp(inst, switch (air_tag) {4362 .add_sat => |air_tag| if (use_old) try cg.airAddSat(inst) else {
4360 else => unreachable,
4361 .sub, .sub_optimized => .sub,
4362 .sub_wrap => .sub_wrap,
4363 }) else {
4364 const bin_op = air_datas[@intFromEnum(inst)].bin_op;4363 const bin_op = air_datas[@intFromEnum(inst)].bin_op;
4365 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });4364 var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs });
4366 var res: [1]Temp = undefined;4365 var res: [1]Temp = undefined;
4367 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{4366 cg.select(&res, &.{cg.typeOf(bin_op.lhs)}, &ops, comptime &.{ .{
4368 .src_constraints = .{ .{ .int = .byte }, .{ .int = .byte }, .any },4367 .src_constraints = .{ .{ .exact_int = 1 }, .{ .exact_int = 1 }, .any },
4369 .patterns = &.{4368 .patterns = &.{
4370 .{ .src = .{ .mut_mem, .imm8, .none } },
4371 .{ .src = .{ .to_mut_gpr, .imm8, .none } },4369 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4372 .{ .src = .{ .mut_mem, .to_gpr, .none } },4370 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4373 .{ .src = .{ .to_mut_gpr, .mem, .none } },4371 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4372 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4374 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },4373 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4375 },4374 },
4376 .dst_temps = .{ .{ .ref = .src0 }, .unused },4375 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4377 .clobbers = .{ .eflags = true },4376 .clobbers = .{ .eflags = true },
4378 .each = .{ .once = &.{4377 .each = .{ .once = &.{
4379 .{ ._, ._, .sub, .dst0b, .src1b, ._, ._ },4378 .{ ._, ._, .@"or", .dst0b, .src1b, ._, ._ },
4380 } },4379 } },
4381 }, .{4380 }, .{
4382 .src_constraints = .{ .{ .int = .word }, .{ .int = .word }, .any },4381 .required_features = .{ .cmov, null, null, null },
4382 .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any },
4383 .patterns = &.{4383 .patterns = &.{
4384 .{ .src = .{ .mut_mem, .imm16, .none } },4384 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4385 .{ .src = .{ .to_mut_gpr, .imm16, .none } },4385 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4386 .{ .src = .{ .mut_mem, .to_gpr, .none } },
4387 .{ .src = .{ .to_mut_gpr, .mem, .none } },4386 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4387 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4388 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },4388 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4389 },4389 },
4390 .extra_temps = .{
4391 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
4392 .unused,
4393 .unused,
4394 .unused,
4395 .unused,
4396 .unused,
4397 .unused,
4398 .unused,
4399 .unused,
4400 .unused,
4401 .unused,
4402 },
4390 .dst_temps = .{ .{ .ref = .src0 }, .unused },4403 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4391 .clobbers = .{ .eflags = true },4404 .clobbers = .{ .eflags = true },
4392 .each = .{ .once = &.{4405 .each = .{ .once = &.{
4393 .{ ._, ._, .sub, .dst0w, .src1w, ._, ._ },4406 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4407 .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ },
4408 .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ },
4409 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4410 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4394 } },4411 } },
4395 }, .{4412 }, .{
4396 .src_constraints = .{ .{ .int = .dword }, .{ .int = .dword }, .any },4413 .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any },
4397 .patterns = &.{4414 .patterns = &.{
4398 .{ .src = .{ .mut_mem, .imm32, .none } },4415 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4399 .{ .src = .{ .to_mut_gpr, .imm32, .none } },4416 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4400 .{ .src = .{ .mut_mem, .to_gpr, .none } },
4401 .{ .src = .{ .to_mut_gpr, .mem, .none } },4417 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4418 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4402 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },4419 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4403 },4420 },
4404 .dst_temps = .{ .{ .ref = .src0 }, .unused },4421 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4405 .clobbers = .{ .eflags = true },4422 .clobbers = .{ .eflags = true },
4406 .each = .{ .once = &.{4423 .each = .{ .once = &.{
4407 .{ ._, ._, .sub, .dst0d, .src1d, ._, ._ },4424 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4425 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4426 .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ },
4427 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smin), ._, ._ },
4408 } },4428 } },
4409 }, .{4429 }, .{
4410 .required_features = .{ .@"64bit", null, null, null },4430 .required_features = .{ .cmov, null, null, null },
4411 .src_constraints = .{ .{ .int = .qword }, .{ .int = .qword }, .any },4431 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
4412 .patterns = &.{4432 .patterns = &.{
4413 .{ .src = .{ .mut_mem, .simm32, .none } },4433 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4414 .{ .src = .{ .to_mut_gpr, .simm32, .none } },4434 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4415 .{ .src = .{ .mut_mem, .to_gpr, .none } },
4416 .{ .src = .{ .to_mut_gpr, .mem, .none } },4435 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4436 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4417 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },4437 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4418 },4438 },
4439 .extra_temps = .{
4440 .{ .type = .i8, .kind = .{ .rc = .general_purpose } },
4441 .unused,
4442 .unused,
4443 .unused,
4444 .unused,
4445 .unused,
4446 .unused,
4447 .unused,
4448 .unused,
4449 .unused,
4450 .unused,
4451 },
4419 .dst_temps = .{ .{ .ref = .src0 }, .unused },4452 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4420 .clobbers = .{ .eflags = true },4453 .clobbers = .{ .eflags = true },
4421 .each = .{ .once = &.{4454 .each = .{ .once = &.{
4422 .{ ._, ._, .sub, .dst0q, .src1q, ._, ._ },4455 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4456 .{ ._, ._r, .sa, .tmp0b, .ui(7), ._, ._ },
4457 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4458 .{ ._, ._, .xor, .tmp0b, .sa(.src0, .add_smax), ._, ._ },
4459 .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ },
4460 .{ ._, ._po, .cmov, .dst0d, .tmp0d, ._, ._ },
4423 } },4461 } },
4424 }, .{4462 }, .{
4425 .required_features = .{ .@"64bit", null, null, null },4463 .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any },
4426 .src_constraints = .{4464 .patterns = &.{
4427 .{ .remainder_int = .{ .of = .qword, .is = .qword } },4465 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4428 .{ .remainder_int = .{ .of = .qword, .is = .qword } },4466 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4429 .any,4467 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4468 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4469 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4430 },4470 },
4471 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4472 .clobbers = .{ .eflags = true },
4473 .each = .{ .once = &.{
4474 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4475 .{ ._, ._, .@"test", .dst0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ },
4476 .{ ._, ._pe, .j, .@"0f", ._, ._, ._ },
4477 .{ ._, ._r, .sa, .dst0b, .ui(7), ._, ._ },
4478 .{ ._, ._, .xor, .dst0b, .sa(.src0, .add_smax), ._, ._ },
4479 } },
4480 }, .{
4481 .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any },
4431 .patterns = &.{4482 .patterns = &.{
4432 .{ .src = .{ .to_mem, .to_mem, .none } },4483 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4484 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4485 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4486 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4487 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4433 },4488 },
4434 .extra_temps = .{4489 .extra_temps = .{
4435 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4490 .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4436 .{ .type = .u64, .kind = .{ .rc = .general_purpose } },4491 .unused,
4437 .unused,4492 .unused,
4438 .unused,4493 .unused,
4439 .unused,4494 .unused,
...@@ -4444,29 +4499,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4444,29 +4499,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4444 .unused,4499 .unused,
4445 .unused,4500 .unused,
4446 },4501 },
4447 .dst_temps = .{ .mem, .unused },4502 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4448 .clobbers = .{ .eflags = true },4503 .clobbers = .{ .eflags = true },
4449 .each = .{ .once = &.{4504 .each = .{ .once = &.{
4450 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ },4505 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4451 .{ ._, ._c, .cl, ._, ._, ._, ._ },4506 .{ ._, ._, .sbb, .tmp0d, .tmp0d, ._, ._ },
4452 .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ },4507 .{ ._, ._, .@"or", .dst0b, .tmp0b, ._, ._ },
4453 .{ ._, ._, .sbb, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ },
4454 .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_size), .tmp1q, ._, ._ },
4455 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
4456 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4457 } },4508 } },
4458 }, .{4509 }, .{
4459 .src_constraints = .{4510 .required_features = .{ .cmov, null, null, null },
4460 .{ .remainder_int = .{ .of = .dword, .is = .dword } },4511 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
4461 .{ .remainder_int = .{ .of = .dword, .is = .dword } },
4462 .any,
4463 },
4464 .patterns = &.{4512 .patterns = &.{
4465 .{ .src = .{ .to_mem, .to_mem, .none } },4513 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4514 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4515 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4516 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4517 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4466 },4518 },
4467 .extra_temps = .{4519 .extra_temps = .{
4468 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4520 .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4469 .{ .type = .u32, .kind = .{ .rc = .general_purpose } },4521 .unused,
4470 .unused,4522 .unused,
4471 .unused,4523 .unused,
4472 .unused,4524 .unused,
...@@ -4477,75 +4529,44 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4477,75 +4529,44 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4477 .unused,4529 .unused,
4478 .unused,4530 .unused,
4479 },4531 },
4480 .dst_temps = .{ .mem, .unused },4532 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4481 .clobbers = .{ .eflags = true },4533 .clobbers = .{ .eflags = true },
4482 .each = .{ .once = &.{4534 .each = .{ .once = &.{
4483 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_4), ._, ._ },4535 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4484 .{ ._, ._c, .cl, ._, ._, ._, ._ },4536 .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_umax), ._, ._ },
4485 .{ .@"0:", ._, .mov, .tmp1d, .memsia(.src0d, .@"4", .tmp0, .add_size), ._, ._ },4537 .{ ._, ._, .cmp, .dst0b, .tmp0b, ._, ._ },
4486 .{ ._, ._, .sbb, .tmp1d, .memsia(.src1d, .@"4", .tmp0, .add_size), ._, ._ },4538 .{ ._, ._a, .cmov, .dst0d, .tmp0d, ._, ._ },
4487 .{ ._, ._, .mov, .memsia(.dst0d, .@"4", .tmp0, .add_size), .tmp1d, ._, ._ },
4488 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },
4489 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },
4490 } },
4491 }, .{
4492 .required_features = .{ .avx, null, null, null },
4493 .src_constraints = .{
4494 .{ .scalar_int = .{ .of = .xword, .is = .byte } },
4495 .{ .scalar_int = .{ .of = .xword, .is = .byte } },
4496 .any,
4497 },
4498 .patterns = &.{
4499 .{ .src = .{ .to_sse, .mem, .none } },
4500 .{ .src = .{ .to_sse, .to_sse, .none } },
4501 },
4502 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
4503 .each = .{ .once = &.{
4504 .{ ._, .vp_b, .sub, .dst0x, .src0x, .src1x, ._ },
4505 } },4539 } },
4506 }, .{4540 }, .{
4507 .required_features = .{ .sse2, null, null, null },4541 .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any },
4508 .src_constraints = .{
4509 .{ .scalar_int = .{ .of = .xword, .is = .byte } },
4510 .{ .scalar_int = .{ .of = .xword, .is = .byte } },
4511 .any,
4512 },
4513 .patterns = &.{4542 .patterns = &.{
4514 .{ .src = .{ .to_mut_sse, .mem, .none } },4543 .{ .src = .{ .to_mut_gpr, .imm8, .none } },
4515 .{ .src = .{ .to_mut_sse, .to_sse, .none } },4544 .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4545 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4546 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4547 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4516 },4548 },
4517 .dst_temps = .{ .{ .ref = .src0 }, .unused },4549 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4550 .clobbers = .{ .eflags = true },
4518 .each = .{ .once = &.{4551 .each = .{ .once = &.{
4519 .{ ._, .p_b, .sub, .dst0x, .src1x, ._, ._ },4552 .{ ._, ._, .add, .dst0b, .src1b, ._, ._ },
4520 } },4553 .{ ._, ._, .cmp, .dst0b, .ua(.src0, .add_umax), ._, ._ },
4521 }, .{4554 .{ ._, ._na, .j, .@"0f", ._, ._, ._ },
4522 .required_features = .{ .avx2, null, null, null },4555 .{ ._, ._, .mov, .dst0b, .ua(.src0, .add_umax), ._, ._ },
4523 .src_constraints = .{
4524 .{ .scalar_int = .{ .of = .yword, .is = .byte } },
4525 .{ .scalar_int = .{ .of = .yword, .is = .byte } },
4526 .any,
4527 },
4528 .patterns = &.{
4529 .{ .src = .{ .to_sse, .mem, .none } },
4530 .{ .src = .{ .to_sse, .to_sse, .none } },
4531 },
4532 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
4533 .each = .{ .once = &.{
4534 .{ ._, .vp_b, .sub, .dst0y, .src0y, .src1y, ._ },
4535 } },4556 } },
4536 }, .{4557 }, .{
4537 .required_features = .{ .avx2, null, null, null },4558 .required_features = .{ .cmov, .fast_imm16, null, null },
4538 .src_constraints = .{4559 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
4539 .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } },
4540 .{ .multiple_scalar_int = .{ .of = .yword, .is = .byte } },
4541 .any,
4542 },
4543 .patterns = &.{4560 .patterns = &.{
4544 .{ .src = .{ .to_mem, .to_mem, .none } },4561 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4562 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4563 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4564 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4565 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4545 },4566 },
4546 .extra_temps = .{4567 .extra_temps = .{
4547 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4568 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4548 .{ .type = .vector_32_u8, .kind = .{ .rc = .sse } },4569 .unused,
4549 .unused,4570 .unused,
4550 .unused,4571 .unused,
4551 .unused,4572 .unused,
...@@ -4556,28 +4577,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4556,28 +4577,28 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4556 .unused,4577 .unused,
4557 .unused,4578 .unused,
4558 },4579 },
4559 .dst_temps = .{ .mem, .unused },4580 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4581 .clobbers = .{ .eflags = true },
4560 .each = .{ .once = &.{4582 .each = .{ .once = &.{
4561 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4583 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4562 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },4584 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4563 .{ ._, .vp_b, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },4585 .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ },
4564 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },4586 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4565 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },4587 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4566 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4567 } },4588 } },
4568 }, .{4589 }, .{
4569 .required_features = .{ .avx, null, null, null },4590 .required_features = .{ .cmov, null, null, null },
4570 .src_constraints = .{4591 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
4571 .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } },
4572 .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } },
4573 .any,
4574 },
4575 .patterns = &.{4592 .patterns = &.{
4576 .{ .src = .{ .to_mem, .to_mem, .none } },4593 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4594 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4595 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4596 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4597 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4577 },4598 },
4578 .extra_temps = .{4599 .extra_temps = .{
4579 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4600 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4580 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },4601 .unused,
4581 .unused,4602 .unused,
4582 .unused,4603 .unused,
4583 .unused,4604 .unused,
...@@ -4588,28 +4609,62 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4588,28 +4609,62 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4588 .unused,4609 .unused,
4589 .unused,4610 .unused,
4590 },4611 },
4591 .dst_temps = .{ .mem, .unused },4612 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4613 .clobbers = .{ .eflags = true },
4592 .each = .{ .once = &.{4614 .each = .{ .once = &.{
4593 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4615 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4594 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },4616 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4595 .{ ._, .vp_b, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },4617 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ },
4596 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },4618 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4597 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },4619 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4598 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4599 } },4620 } },
4600 }, .{4621 }, .{
4601 .required_features = .{ .sse2, null, null, null },4622 .required_features = .{ .fast_imm16, null, null, null },
4602 .src_constraints = .{4623 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
4603 .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } },4624 .patterns = &.{
4604 .{ .multiple_scalar_int = .{ .of = .xword, .is = .byte } },4625 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4605 .any,4626 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4627 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4628 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4629 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4630 },
4631 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4632 .clobbers = .{ .eflags = true },
4633 .each = .{ .once = &.{
4634 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4635 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4636 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
4637 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smin), ._, ._ },
4638 } },
4639 }, .{
4640 .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any },
4641 .patterns = &.{
4642 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4643 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4644 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4645 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4646 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4606 },4647 },
4648 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4649 .clobbers = .{ .eflags = true },
4650 .each = .{ .once = &.{
4651 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4652 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4653 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
4654 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smin), ._, ._ },
4655 } },
4656 }, .{
4657 .required_features = .{ .cmov, .fast_imm16, null, null },
4658 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4607 .patterns = &.{4659 .patterns = &.{
4608 .{ .src = .{ .to_mem, .to_mem, .none } },4660 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4661 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4662 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4663 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4609 },4664 },
4610 .extra_temps = .{4665 .extra_temps = .{
4611 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4666 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4612 .{ .type = .vector_16_u8, .kind = .{ .rc = .sse } },4667 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4613 .unused,4668 .unused,
4614 .unused,4669 .unused,
4615 .unused,4670 .unused,
...@@ -4620,28 +4675,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4620,28 +4675,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4620 .unused,4675 .unused,
4621 .unused,4676 .unused,
4622 },4677 },
4623 .dst_temps = .{ .mem, .unused },4678 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4679 .clobbers = .{ .eflags = true },
4624 .each = .{ .once = &.{4680 .each = .{ .once = &.{
4625 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4681 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4626 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },4682 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4627 .{ ._, .p_b, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },4683 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4628 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },4684 .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ },
4629 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },4685 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
4630 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },4686 .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ },
4687 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4631 } },4688 } },
4632 }, .{4689 }, .{
4633 .required_features = .{ .slow_incdec, null, null, null },4690 .required_features = .{ .cmov, .fast_imm16, null, null },
4634 .src_constraints = .{4691 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4635 .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } },
4636 .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } },
4637 .any,
4638 },
4639 .patterns = &.{4692 .patterns = &.{
4640 .{ .src = .{ .to_mem, .to_mem, .none } },4693 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4641 },4694 },
4642 .extra_temps = .{4695 .extra_temps = .{
4643 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4696 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4644 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },4697 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4645 .unused,4698 .unused,
4646 .unused,4699 .unused,
4647 .unused,4700 .unused,
...@@ -4652,27 +4705,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4652,27 +4705,27 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4652 .unused,4705 .unused,
4653 .unused,4706 .unused,
4654 },4707 },
4655 .dst_temps = .{ .mem, .unused },4708 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4709 .clobbers = .{ .eflags = true },
4656 .each = .{ .once = &.{4710 .each = .{ .once = &.{
4657 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4711 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4658 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },4712 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4659 .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ },4713 .{ ._, ._, .add, .dst0d, .src1d, ._, ._ },
4660 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },4714 .{ ._, ._, .xor, .tmp0w, .sa(.src0, .add_smax), ._, ._ },
4661 .{ ._, ._, .add, .tmp0p, .si(1), ._, ._ },4715 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
4662 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },4716 .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ },
4717 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4663 } },4718 } },
4664 }, .{4719 }, .{
4665 .src_constraints = .{4720 .required_features = .{ .cmov, null, null, null },
4666 .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } },4721 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4667 .{ .multiple_scalar_int = .{ .of = .byte, .is = .byte } },
4668 .any,
4669 },
4670 .patterns = &.{4722 .patterns = &.{
4671 .{ .src = .{ .to_mem, .to_mem, .none } },4723 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4724 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4672 },4725 },
4673 .extra_temps = .{4726 .extra_temps = .{
4674 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4727 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4675 .{ .type = .u8, .kind = .{ .rc = .general_purpose } },4728 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4676 .unused,4729 .unused,
4677 .unused,4730 .unused,
4678 .unused,4731 .unused,
...@@ -4683,73 +4736,61 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4683,73 +4736,61 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4683 .unused,4736 .unused,
4684 .unused,4737 .unused,
4685 },4738 },
4686 .dst_temps = .{ .mem, .unused },4739 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4740 .clobbers = .{ .eflags = true },
4687 .each = .{ .once = &.{4741 .each = .{ .once = &.{
4688 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4742 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4689 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0b, .tmp0, .add_unaligned_size), ._, ._ },4743 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4690 .{ ._, ._, .sub, .tmp1b, .memia(.src1b, .tmp0, .add_unaligned_size), ._, ._ },4744 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4691 .{ ._, ._, .mov, .memia(.dst0b, .tmp0, .add_unaligned_size), .tmp1b, ._, ._ },4745 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ },
4692 .{ ._, ._c, .in, .tmp0p, ._, ._, ._ },4746 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
4693 .{ ._, ._nz, .j, .@"0b", ._, ._, ._ },4747 .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ },
4748 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4694 } },4749 } },
4695 }, .{4750 }, .{
4696 .required_features = .{ .avx, null, null, null },4751 .required_features = .{ .cmov, null, null, null },
4697 .src_constraints = .{4752 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4698 .{ .scalar_int = .{ .of = .xword, .is = .word } },
4699 .{ .scalar_int = .{ .of = .xword, .is = .word } },
4700 .any,
4701 },
4702 .patterns = &.{4753 .patterns = &.{
4703 .{ .src = .{ .to_sse, .mem, .none } },4754 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4704 .{ .src = .{ .to_sse, .to_sse, .none } },4755 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4705 },4756 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4706 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
4707 .each = .{ .once = &.{
4708 .{ ._, .vp_w, .sub, .dst0x, .src0x, .src1x, ._ },
4709 } },
4710 }, .{
4711 .required_features = .{ .sse2, null, null, null },
4712 .src_constraints = .{
4713 .{ .scalar_int = .{ .of = .xword, .is = .word } },
4714 .{ .scalar_int = .{ .of = .xword, .is = .word } },
4715 .any,
4716 },4757 },
4717 .patterns = &.{4758 .extra_temps = .{
4718 .{ .src = .{ .to_mut_sse, .mem, .none } },4759 .{ .type = .i16, .kind = .{ .rc = .general_purpose } },
4719 .{ .src = .{ .to_mut_sse, .to_sse, .none } },4760 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4761 .unused,
4762 .unused,
4763 .unused,
4764 .unused,
4765 .unused,
4766 .unused,
4767 .unused,
4768 .unused,
4769 .unused,
4720 },4770 },
4721 .dst_temps = .{ .{ .ref = .src0 }, .unused },4771 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4772 .clobbers = .{ .eflags = true },
4722 .each = .{ .once = &.{4773 .each = .{ .once = &.{
4723 .{ ._, .p_w, .sub, .dst0x, .src1x, ._, ._ },4774 .{ ._, ._, .mov, .tmp0d, .src0d, ._, ._ },
4724 } },4775 .{ ._, ._r, .sa, .tmp0w, .ui(15), ._, ._ },
4725 }, .{4776 .{ ._, ._, .add, .dst0d, .src1d, ._, ._ },
4726 .required_features = .{ .avx2, null, null, null },4777 .{ ._, ._, .xor, .tmp0d, .sa(.src0, .add_smax), ._, ._ },
4727 .src_constraints = .{4778 .{ ._, ._, .mov, .tmp1d, .dst0d, ._, ._ },
4728 .{ .scalar_int = .{ .of = .yword, .is = .word } },4779 .{ ._, ._, .add, .tmp1w, .tmp1w, ._, ._ },
4729 .{ .scalar_int = .{ .of = .yword, .is = .word } },4780 .{ ._, ._o, .cmov, .dst0d, .tmp0d, ._, ._ },
4730 .any,
4731 },
4732 .patterns = &.{
4733 .{ .src = .{ .to_sse, .mem, .none } },
4734 .{ .src = .{ .to_sse, .to_sse, .none } },
4735 },
4736 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
4737 .each = .{ .once = &.{
4738 .{ ._, .vp_w, .sub, .dst0y, .src0y, .src1y, ._ },
4739 } },4781 } },
4740 }, .{4782 }, .{
4741 .required_features = .{ .avx2, null, null, null },4783 .required_features = .{ .fast_imm16, null, null, null },
4742 .src_constraints = .{4784 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4743 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
4744 .{ .multiple_scalar_int = .{ .of = .yword, .is = .word } },
4745 .any,
4746 },
4747 .patterns = &.{4785 .patterns = &.{
4748 .{ .src = .{ .to_mem, .to_mem, .none } },4786 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4787 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4788 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4789 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4749 },4790 },
4750 .extra_temps = .{4791 .extra_temps = .{
4751 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4792 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4752 .{ .type = .vector_16_u16, .kind = .{ .rc = .sse } },4793 .unused,
4753 .unused,4794 .unused,
4754 .unused,4795 .unused,
4755 .unused,4796 .unused,
...@@ -4760,28 +4801,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4760,28 +4801,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4760 .unused,4801 .unused,
4761 .unused,4802 .unused,
4762 },4803 },
4763 .dst_temps = .{ .mem, .unused },4804 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4805 .clobbers = .{ .eflags = true },
4764 .each = .{ .once = &.{4806 .each = .{ .once = &.{
4765 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4807 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4766 .{ .@"0:", .v_dqa, .mov, .tmp1y, .memia(.src0y, .tmp0, .add_unaligned_size), ._, ._ },4808 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
4767 .{ ._, .vp_w, .sub, .tmp1y, .tmp1y, .memia(.src1y, .tmp0, .add_unaligned_size), ._ },4809 .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ },
4768 .{ ._, .v_dqa, .mov, .memia(.dst0y, .tmp0, .add_unaligned_size), .tmp1y, ._, ._ },4810 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4769 .{ ._, ._, .add, .tmp0p, .si(32), ._, ._ },4811 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
4770 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },4812 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
4771 } },4813 } },
4772 }, .{4814 }, .{
4773 .required_features = .{ .avx, null, null, null },4815 .required_features = .{ .fast_imm16, null, null, null },
4774 .src_constraints = .{4816 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4775 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
4776 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
4777 .any,
4778 },
4779 .patterns = &.{4817 .patterns = &.{
4780 .{ .src = .{ .to_mem, .to_mem, .none } },4818 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4781 },4819 },
4782 .extra_temps = .{4820 .extra_temps = .{
4783 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4821 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4784 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },4822 .unused,
4785 .unused,4823 .unused,
4786 .unused,4824 .unused,
4787 .unused,4825 .unused,
...@@ -4792,28 +4830,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4792,28 +4830,25 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4792 .unused,4830 .unused,
4793 .unused,4831 .unused,
4794 },4832 },
4795 .dst_temps = .{ .mem, .unused },4833 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4834 .clobbers = .{ .eflags = true },
4796 .each = .{ .once = &.{4835 .each = .{ .once = &.{
4797 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4836 .{ ._, ._, .add, .dst0d, .src1d, ._, ._ },
4798 .{ .@"0:", .v_dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },4837 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
4799 .{ ._, .vp_w, .sub, .tmp1x, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._ },4838 .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ },
4800 .{ ._, .v_dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },4839 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4801 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },4840 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
4802 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },4841 .{ ._, ._, .xor, .dst0w, .sa(.src0, .add_smax), ._, ._ },
4803 } },4842 } },
4804 }, .{4843 }, .{
4805 .required_features = .{ .sse2, null, null, null },4844 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4806 .src_constraints = .{
4807 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
4808 .{ .multiple_scalar_int = .{ .of = .xword, .is = .word } },
4809 .any,
4810 },
4811 .patterns = &.{4845 .patterns = &.{
4812 .{ .src = .{ .to_mem, .to_mem, .none } },4846 .{ .src = .{ .to_mut_gpr, .mem, .none } },
4847 .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4813 },4848 },
4814 .extra_temps = .{4849 .extra_temps = .{
4815 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4850 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4816 .{ .type = .vector_8_u16, .kind = .{ .rc = .sse } },4851 .unused,
4817 .unused,4852 .unused,
4818 .unused,4853 .unused,
4819 .unused,4854 .unused,
...@@ -4824,27 +4859,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4824,27 +4859,26 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4824 .unused,4859 .unused,
4825 .unused,4860 .unused,
4826 },4861 },
4827 .dst_temps = .{ .mem, .unused },4862 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4863 .clobbers = .{ .eflags = true },
4828 .each = .{ .once = &.{4864 .each = .{ .once = &.{
4829 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },4865 .{ ._, ._, .add, .dst0w, .src1w, ._, ._ },
4830 .{ .@"0:", ._dqa, .mov, .tmp1x, .memia(.src0x, .tmp0, .add_unaligned_size), ._, ._ },4866 .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ },
4831 .{ ._, .p_w, .sub, .tmp1x, .memia(.src1x, .tmp0, .add_unaligned_size), ._, ._ },4867 .{ ._, ._, .add, .tmp0w, .tmp0w, ._, ._ },
4832 .{ ._, ._dqa, .mov, .memia(.dst0x, .tmp0, .add_unaligned_size), .tmp1x, ._, ._ },4868 .{ ._, ._no, .j, .@"0f", ._, ._, ._ },
4833 .{ ._, ._, .add, .tmp0p, .si(16), ._, ._ },4869 .{ ._, ._r, .sa, .dst0w, .ui(15), ._, ._ },
4834 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },4870 .{ ._, ._, .xor, .dst0d, .sa(.src0, .add_smax), ._, ._ },
4835 } },4871 } },
4836 }, .{4872 }, .{
4837 .src_constraints = .{4873 .src_constraints = .{ .{ .exact_signed_int = 15 }, .{ .exact_signed_int = 15 }, .any },
4838 .{ .multiple_scalar_int = .{ .of = .word, .is = .word } },
4839 .{ .multiple_scalar_int = .{ .of = .word, .is = .word } },
4840 .any,
4841 },
4842 .patterns = &.{4874 .patterns = &.{
4843 .{ .src = .{ .to_mem, .to_mem, .none } },4875 .{ .src = .{ .to_mut_gpr, .imm16, .none } },
4876 .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } },
4877 .{ .src = .{ .to_mut_gpr, .to_gpr, .none } },
4844 },4878 },
4845 .extra_temps = .{4879 .extra_temps = .{
4846 .{ .type = .isize, .kind = .{ .rc = .general_purpose } },4880 .{ .type = .i16, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } },
4847 .{ .type = .u16, .kind = .{ .rc = .general_purpose } },4881 .unused,
4848 .unused,4882 .unused,
4849 .unused,4883 .unused,
4850 .unused,4884 .unused,
...@@ -4854,74 +4888,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {...@@ -4854,74 +4888,29 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
4854 .unused,4888 .unused,
4855 .unused,4889 .unused,
4856 .unused,4890 .unused,
4857 },
4858 .dst_temps = .{ .mem, .unused },
4859 .each = .{ .once = &.{
4860 .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_unaligned_size), ._, ._ },
4861 .{ .@"0:", ._, .movzx, .tmp1d, .memia(.src0w, .tmp0, .add_unaligned_size), ._, ._ },
4862 .{ ._, ._, .sub, .tmp1w, .memia(.src1w, .tmp0, .add_unaligned_size), ._, ._ },
4863 .{ ._, ._, .mov, .memia(.dst0w, .tmp0, .add_unaligned_size), .tmp1w, ._, ._ },
4864 .{ ._, ._, .add, .tmp0p, .si(2), ._, ._ },
4865 .{ ._, ._nc, .j, .@"0b", ._, ._, ._ },
4866 } },
4867 }, .{
4868 .required_features = .{ .avx, null, null, null },
4869 .src_constraints = .{
4870 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
4871 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
4872 .any,
4873 },
4874 .patterns = &.{
4875 .{ .src = .{ .to_sse, .mem, .none } },
4876 .{ .src = .{ .to_sse, .to_sse, .none } },
4877 },
4878 .dst_temps = .{ .{ .mut_rc = .{ .ref = .src0, .rc = .sse } }, .unused },
4879 .each = .{ .once = &.{
4880 .{ ._, .vp_d, .sub, .dst0x, .src0x, .src1x, ._ },
4881 } },
4882 }, .{
4883 .required_features = .{ .sse2, null, null, null },
4884 .src_constraints = .{
4885 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
4886 .{ .scalar_int = .{ .of = .xword, .is = .dword } },
4887 .any,
4888 },
4889 .patterns = &.{
4890 .{ .src = .{ .to_mut_sse, .mem, .none } },
4891 .{ .src = .{ .to_mut_sse, .to_sse, .none } },
4892 },4891 },
4893 .dst_temps = .{ .{ .ref = .src0 }, .unused },4892 .dst_temps = .{ .{ .ref = .src0 }, .unused },
4893 .clobbers = .{ .eflags = true },
4894 .each = .{ .once = &.{4894 .each = .{ .once = &.{
4895 .{ ._, .p_d, .sub, .dst0x, .src1x, ._, ._ },4895