| author |  Jacob Young <jacobly@ziglang.org> 2023-05-10 01:56:48-04:00 |
| committer | Jacob Young <jacobly@ziglang.org> 2023-05-10 15:11:52-04:00 |
| log | 2e6a6d7564901009aace5fef99e647959ad5bb90 |
| tree | 34e42699258c0c5869c99bbb6440fbf9e0ca07e6 |
| parent | 31a13ce2e3fa16de94954f5a6c22d2d650cd8262 |
llvm/cbe: fix signed `@mod`/`@divFloor` computations
Closes #156363 files changed, 61 insertions(+), 38 deletions(-)
lib/zig.h+17-20
| ... | ... | @@ -487,14 +487,14 @@ typedef ptrdiff_t intptr_t; |
| 487 | 487 | zig_basic_operator(uint##w##_t, div_floor_u##w, /) \ |
| 488 | 488 | \ |
| 489 | 489 | static inline int##w##_t zig_div_floor_i##w(int##w##_t lhs, int##w##_t rhs) { \ |
| 490 | return lhs / rhs - (((lhs ^ rhs) & (lhs % rhs)) < INT##w##_C(0)); \ | |
| 490 | return lhs / rhs + (lhs % rhs != INT##w##_C(0) ? zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ | |
| 491 | 491 | } \ |
| 492 | 492 | \ |
| 493 | 493 | zig_basic_operator(uint##w##_t, mod_u##w, %) \ |
| 494 | 494 | \ |
| 495 | 495 | static inline int##w##_t zig_mod_i##w(int##w##_t lhs, int##w##_t rhs) { \ |
| 496 | 496 | int##w##_t rem = lhs % rhs; \ |
| 497 | return rem + (((lhs ^ rhs) & rem) < INT##w##_C(0) ? rhs : INT##w##_C(0)); \ | |
| 497 | return rem + (rem != INT##w##_C(0) ? rhs & zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ | |
| 498 | 498 | } \ |
| 499 | 499 | \ |
| 500 | 500 | static inline uint##w##_t zig_shlw_u##w(uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \ |
| ... | ... | @@ -1078,7 +1078,7 @@ static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) { |
| 1078 | 1078 | uint##w##_t temp = val - ((val >> 1) & (UINT##w##_MAX / 3)); \ |
| 1079 | 1079 | temp = (temp & (UINT##w##_MAX / 5)) + ((temp >> 2) & (UINT##w##_MAX / 5)); \ |
| 1080 | 1080 | temp = (temp + (temp >> 4)) & (UINT##w##_MAX / 17); \ |
| 1081 | return temp * (UINT##w##_MAX / 255) >> (w - 8); \ | |
| 1081 | return temp * (UINT##w##_MAX / 255) >> (UINT8_C(w) - UINT8_C(8)); \ | |
| 1082 | 1082 | } \ |
| 1083 | 1083 | \ |
| 1084 | 1084 | zig_builtin_popcount_common(w) |
| ... | ... | @@ -1298,15 +1298,6 @@ static inline zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { |
| 1298 | 1298 | return lhs % rhs; |
| 1299 | 1299 | } |
| 1300 | 1300 | |
| 1301 | static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) { | |
| 1302 | return zig_div_trunc_i128(lhs, rhs) - (((lhs ^ rhs) & zig_rem_i128(lhs, rhs)) < zig_make_i128(0, 0)); | |
| 1303 | } | |
| 1304 | ||
| 1305 | static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) { | |
| 1306 | zig_i128 rem = zig_rem_i128(lhs, rhs); | |
| 1307 | return rem + (((lhs ^ rhs) & rem) < zig_make_i128(0, 0) ? rhs : zig_make_i128(0, 0)); | |
| 1308 | } | |
| 1309 | ||
| 1310 | 1301 | #else /* zig_has_int128 */ |
| 1311 | 1302 | |
| 1312 | 1303 | static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) { |
| ... | ... | @@ -1394,20 +1385,26 @@ static zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { |
| 1394 | 1385 | return __modti3(lhs, rhs); |
| 1395 | 1386 | } |
| 1396 | 1387 | |
| 1397 | static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) { | |
| 1398 | zig_i128 rem = zig_rem_i128(lhs, rhs); | |
| 1399 | return zig_add_i128(rem, ((lhs.hi ^ rhs.hi) & rem.hi) < INT64_C(0) ? rhs : zig_make_i128(0, 0)); | |
| 1400 | } | |
| 1388 | #endif /* zig_has_int128 */ | |
| 1389 | ||
| 1390 | #define zig_div_floor_u128 zig_div_trunc_u128 | |
| 1401 | 1391 | |
| 1402 | 1392 | static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) { |
| 1403 | return zig_sub_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(0, zig_cmp_i128(zig_and_i128(zig_xor_i128(lhs, rhs), zig_rem_i128(lhs, rhs)), zig_make_i128(0, 0)) < INT32_C(0))); | |
| 1393 | zig_i128 rem = zig_rem_i128(lhs, rhs); | |
| 1394 | int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) | |
| 1395 | ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); | |
| 1396 | return zig_add_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(mask, (uint64_t)mask)); | |
| 1404 | 1397 | } |
| 1405 | 1398 | |
| 1406 | #endif /* zig_has_int128 */ | |
| 1407 | ||
| 1408 | #define zig_div_floor_u128 zig_div_trunc_u128 | |
| 1409 | 1399 | #define zig_mod_u128 zig_rem_u128 |
| 1410 | 1400 | |
| 1401 | static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) { | |
| 1402 | zig_i128 rem = zig_rem_i128(lhs, rhs); | |
| 1403 | int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) | |
| 1404 | ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); | |
| 1405 | return zig_add_i128(rem, zig_and_i128(rhs, zig_make_i128(mask, (uint64_t)mask))); | |
| 1406 | } | |
| 1407 | ||
| 1411 | 1408 | static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) { |
| 1412 | 1409 | return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs; |
| 1413 | 1410 | } |
src/codegen/llvm.zig+41-18
| ... | ... | @@ -7215,20 +7215,28 @@ pub const FuncGen = struct { |
| 7215 | 7215 | return self.buildFloatOp(.floor, inst_ty, 1, .{result}); |
| 7216 | 7216 | } |
| 7217 | 7217 | if (scalar_ty.isSignedInt()) { |
| 7218 | // const d = @divTrunc(a, b); | |
| 7219 | // const r = @rem(a, b); | |
| 7220 | // return if (r == 0) d else d - ((a < 0) ^ (b < 0)); | |
| 7221 | const result_llvm_ty = try self.dg.lowerType(inst_ty); | |
| 7222 | const zero = result_llvm_ty.constNull(); | |
| 7223 | const div_trunc = self.builder.buildSDiv(lhs, rhs, ""); | |
| 7218 | const target = self.dg.module.getTarget(); | |
| 7219 | const inst_llvm_ty = try self.dg.lowerType(inst_ty); | |
| 7220 | const scalar_bit_size_minus_one = scalar_ty.bitSize(target) - 1; | |
| 7221 | const bit_size_minus_one = if (inst_ty.zigTypeTag() == .Vector) const_vector: { | |
| 7222 | const vec_len = inst_ty.vectorLen(); | |
| 7223 | const scalar_llvm_ty = try self.dg.lowerType(scalar_ty); | |
| 7224 | ||
| 7225 | const shifts = try self.gpa.alloc(*llvm.Value, vec_len); | |
| 7226 | defer self.gpa.free(shifts); | |
| 7227 | ||
| 7228 | @memset(shifts, scalar_llvm_ty.constInt(scalar_bit_size_minus_one, .False)); | |
| 7229 | break :const_vector llvm.constVector(shifts.ptr, vec_len); | |
| 7230 | } else inst_llvm_ty.constInt(scalar_bit_size_minus_one, .False); | |
| 7231 | ||
| 7232 | const div = self.builder.buildSDiv(lhs, rhs, ""); | |
| 7224 | 7233 | const rem = self.builder.buildSRem(lhs, rhs, ""); |
| 7225 | const rem_eq_0 = self.builder.buildICmp(.EQ, rem, zero, ""); | |
| 7226 | const a_lt_0 = self.builder.buildICmp(.SLT, lhs, zero, ""); | |
| 7227 | const b_lt_0 = self.builder.buildICmp(.SLT, rhs, zero, ""); | |
| 7228 | const a_b_xor = self.builder.buildXor(a_lt_0, b_lt_0, ""); | |
| 7229 | const a_b_xor_ext = self.builder.buildZExt(a_b_xor, div_trunc.typeOf(), ""); | |
| 7230 | const d_sub_xor = self.builder.buildSub(div_trunc, a_b_xor_ext, ""); | |
| 7231 | return self.builder.buildSelect(rem_eq_0, div_trunc, d_sub_xor, ""); | |
| 7234 | const div_sign = self.builder.buildXor(lhs, rhs, ""); | |
| 7235 | const div_sign_mask = self.builder.buildAShr(div_sign, bit_size_minus_one, ""); | |
| 7236 | const zero = inst_llvm_ty.constNull(); | |
| 7237 | const rem_nonzero = self.builder.buildICmp(.NE, rem, zero, ""); | |
| 7238 | const correction = self.builder.buildSelect(rem_nonzero, div_sign_mask, zero, ""); | |
| 7239 | return self.builder.buildNSWAdd(div, correction, ""); | |
| 7232 | 7240 | } |
| 7233 | 7241 | return self.builder.buildUDiv(lhs, rhs, ""); |
| 7234 | 7242 | } |
| ... | ... | @@ -7280,12 +7288,27 @@ pub const FuncGen = struct { |
| 7280 | 7288 | return self.builder.buildSelect(ltz, c, a, ""); |
| 7281 | 7289 | } |
| 7282 | 7290 | if (scalar_ty.isSignedInt()) { |
| 7283 | const a = self.builder.buildSRem(lhs, rhs, ""); | |
| 7284 | const b = self.builder.buildNSWAdd(a, rhs, ""); | |
| 7285 | const c = self.builder.buildSRem(b, rhs, ""); | |
| 7291 | const target = self.dg.module.getTarget(); | |
| 7292 | const scalar_bit_size_minus_one = scalar_ty.bitSize(target) - 1; | |
| 7293 | const bit_size_minus_one = if (inst_ty.zigTypeTag() == .Vector) const_vector: { | |
| 7294 | const vec_len = inst_ty.vectorLen(); | |
| 7295 | const scalar_llvm_ty = try self.dg.lowerType(scalar_ty); | |
| 7296 | ||
| 7297 | const shifts = try self.gpa.alloc(*llvm.Value, vec_len); | |
| 7298 | defer self.gpa.free(shifts); | |
| 7299 | ||
| 7300 | @memset(shifts, scalar_llvm_ty.constInt(scalar_bit_size_minus_one, .False)); | |
| 7301 | break :const_vector llvm.constVector(shifts.ptr, vec_len); | |
| 7302 | } else inst_llvm_ty.constInt(scalar_bit_size_minus_one, .False); | |
| 7303 | ||
| 7304 | const rem = self.builder.buildSRem(lhs, rhs, ""); | |
| 7305 | const div_sign = self.builder.buildXor(lhs, rhs, ""); | |
| 7306 | const div_sign_mask = self.builder.buildAShr(div_sign, bit_size_minus_one, ""); | |
| 7307 | const rhs_masked = self.builder.buildAnd(rhs, div_sign_mask, ""); | |
| 7286 | 7308 | const zero = inst_llvm_ty.constNull(); |
| 7287 | const ltz = self.builder.buildICmp(.SLT, lhs, zero, ""); | |
| 7288 | return self.builder.buildSelect(ltz, c, a, ""); | |
| 7309 | const rem_nonzero = self.builder.buildICmp(.NE, rem, zero, ""); | |
| 7310 | const correction = self.builder.buildSelect(rem_nonzero, rhs_masked, zero, ""); | |
| 7311 | return self.builder.buildNSWAdd(rem, correction, ""); | |
| 7289 | 7312 | } |
| 7290 | 7313 | return self.builder.buildURem(lhs, rhs, ""); |
| 7291 | 7314 | } |
test/behavior/math.zig+3
| ... | ... | @@ -449,6 +449,9 @@ fn testDivision() !void { |
| 449 | 449 | try expect(mod(i32, 10, 12) == 10); |
| 450 | 450 | try expect(mod(i32, -14, 12) == 10); |
| 451 | 451 | try expect(mod(i32, -2, 12) == 10); |
| 452 | try expect(mod(i32, 10, -12) == -2); | |
| 453 | try expect(mod(i32, -14, -12) == -2); | |
| 454 | try expect(mod(i32, -2, -12) == -2); | |
| 452 | 455 | |
| 453 | 456 | comptime { |
| 454 | 457 | try expect( |