| ... | @@ -658,7 +658,7 @@ pub const Mutable = struct { | ... | @@ -658,7 +658,7 @@ pub const Mutable = struct { |
| 658 | | 658 | |
| 659 | mem.set(Limb, rma.limbs[0 .. a.limbs.len + b.limbs.len + 1], 0); | 659 | mem.set(Limb, rma.limbs[0 .. a.limbs.len + b.limbs.len + 1], 0); |
| 660 | | 660 | |
| 661 | llmulacc(.add, allocator, rma.limbs, a.limbs, b.limbs); | 661 | _ = llmulacc(.add, allocator, rma.limbs, a.limbs, b.limbs); |
| 662 | | 662 | |
| 663 | rma.normalize(a.limbs.len + b.limbs.len); | 663 | rma.normalize(a.limbs.len + b.limbs.len); |
| 664 | rma.positive = (a.positive == b.positive); | 664 | rma.positive = (a.positive == b.positive); |
| ... | @@ -2365,9 +2365,12 @@ const AccOp = enum { | ... | @@ -2365,9 +2365,12 @@ const AccOp = enum { |
| 2365 | /// | 2365 | /// |
| 2366 | /// r = r (op) a * b | 2366 | /// r = r (op) a * b |
| 2367 | /// r MUST NOT alias any of a or b. | 2367 | /// r MUST NOT alias any of a or b. |
| | 2368 | /// |
| | 2369 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2368 | fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []const Limb, b: []const Limb) void { | 2370 | fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []const Limb, b: []const Limb) void { |
| 2369 | @setRuntimeSafety(debug_safety); | 2371 | @setRuntimeSafety(debug_safety); |
| 2370 | assert(r.len >= a.len + b.len); | 2372 | assert(r.len >= a.len); |
| | 2373 | assert(r.len >= b.len); |
| 2371 | | 2374 | |
| 2372 | // Order greatest first. | 2375 | // Order greatest first. |
| 2373 | var x = a; | 2376 | var x = a; |
| ... | @@ -2395,6 +2398,8 @@ fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []cons | ... | @@ -2395,6 +2398,8 @@ fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []cons |
| 2395 | /// | 2398 | /// |
| 2396 | /// r = r (op) a * b | 2399 | /// r = r (op) a * b |
| 2397 | /// r MUST NOT alias any of a or b. | 2400 | /// r MUST NOT alias any of a or b. |
| | 2401 | /// |
| | 2402 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2398 | fn llmulaccKaratsuba( | 2403 | fn llmulaccKaratsuba( |
| 2399 | comptime op: AccOp, | 2404 | comptime op: AccOp, |
| 2400 | allocator: *Allocator, | 2405 | allocator: *Allocator, |
| ... | @@ -2403,7 +2408,7 @@ fn llmulaccKaratsuba( | ... | @@ -2403,7 +2408,7 @@ fn llmulaccKaratsuba( |
| 2403 | b: []const Limb, | 2408 | b: []const Limb, |
| 2404 | ) error{OutOfMemory}!void { | 2409 | ) error{OutOfMemory}!void { |
| 2405 | @setRuntimeSafety(debug_safety); | 2410 | @setRuntimeSafety(debug_safety); |
| 2406 | assert(r.len >= a.len + b.len); | 2411 | assert(r.len >= a.len); |
| 2407 | assert(a.len >= b.len); | 2412 | assert(a.len >= b.len); |
| 2408 | | 2413 | |
| 2409 | // Classical karatsuba algorithm: | 2414 | // Classical karatsuba algorithm: |
| ... | @@ -2437,49 +2442,84 @@ fn llmulaccKaratsuba( | ... | @@ -2437,49 +2442,84 @@ fn llmulaccKaratsuba( |
| 2437 | // | 2442 | // |
| 2438 | // Note, when B is a multiple of the limb size, multiplies by B amount to shifts or | 2443 | // Note, when B is a multiple of the limb size, multiplies by B amount to shifts or |
| 2439 | // slices of a limbs array. | 2444 | // slices of a limbs array. |
| | 2445 | // |
| | 2446 | // This function computes the result of the multiplication modulo r.len. This means: |
| | 2447 | // - p2 and p1 only need to be computed modulo r.len - B. |
| | 2448 | // - In the case of p2, p2 * B^2 needs to be added modulo r.len - 2 * B. |
| 2440 | | 2449 | |
| 2441 | const split = b.len / 2; // B | 2450 | const split = b.len / 2; // B |
| | 2451 | |
| | 2452 | const limbs_after_split = r.len - split; // Limbs to compute for p1 and p2. |
| | 2453 | const limbs_after_split2 = r.len - split * 2; // Limbs to add for p2 * B^2. |
| | 2454 | |
| | 2455 | // For a0 and b0 we need the full range. |
| 2442 | const a0 = a[0..llnormalize(a[0..split])]; | 2456 | const a0 = a[0..llnormalize(a[0..split])]; |
| 2443 | const a1 = a[split..][0..llnormalize(a[split..])]; | | |
| 2444 | const b0 = b[0..llnormalize(b[0..split])]; | 2457 | const b0 = b[0..llnormalize(b[0..split])]; |
| 2445 | const b1 = b[split..][0..llnormalize(b[split..])]; | | |
| 2446 | | 2458 | |
| 2447 | // Note that the above slices work because we have a.len > b.len. | 2459 | // For a1 and b1 we only need `limbs_after_split` limbs. |
| 2448 | // We now also have: | 2460 | const a1 = blk: { |
| 2449 | // a1.len >= a0.len | 2461 | var a1 = a[split..]; |
| 2450 | // a1.len >= b1.len >= b0.len | 2462 | a1.len = math.min(llnormalize(a1), limbs_after_split); |
| 2451 | // a0.len == b0.len | 2463 | break :blk a1; |
| | 2464 | }; |
| | 2465 | |
| | 2466 | const b1 = blk: { |
| | 2467 | var b1 = b[split..]; |
| | 2468 | b1.len = math.min(llnormalize(b1), limbs_after_split); |
| | 2469 | break :blk b1; |
| | 2470 | }; |
| | 2471 | |
| | 2472 | // Note that the above slices relative to `split` work because we have a.len > b.len. |
| 2452 | | 2473 | |
| 2453 | // We need some temporary memory to store intermediate results. | 2474 | // We need some temporary memory to store intermediate results. |
| 2454 | // Note, we can reduce the amount of temporaries we need by reordering the computation here: | 2475 | // Note, we can reduce the amount of temporaries we need by reordering the computation here: |
| 2455 | // ab = p2 * B^2 + (p0 + p1 + p2) * B + p0 | 2476 | // ab = p2 * B^2 + (p0 + p1 + p2) * B + p0 |
| 2456 | // = p2 * B^2 + (p0 * B + p1 * B + p2 * B) + p0 | 2477 | // = p2 * B^2 + (p0 * B + p1 * B + p2 * B) + p0 |
| 2457 | // = (p2 * B^2 + p2 * B) + (p0 * B + p0) + p1 * B | 2478 | // = (p2 * B^2 + p2 * B) + (p0 * B + p0) + p1 * B |
| 2458 | // By allocating a1.len * b1.len we can be sure that all the intermediary results fit. | 2479 | |
| | 2480 | // Allocate at least enough memory to be able to multiply the upper two segments of a and b, assuming |
| | 2481 | // no overflow. |
| 2459 | const tmp = try allocator.alloc(Limb, a.len - split + b.len - split); | 2482 | const tmp = try allocator.alloc(Limb, a.len - split + b.len - split); |
| 2460 | defer allocator.free(tmp); | 2483 | defer allocator.free(tmp); |
| 2461 | | 2484 | |
| 2462 | // Compute p2. | 2485 | // Compute p2. |
| 2463 | mem.set(Limb, tmp, 0); | 2486 | // Note, we don't need to compute all of p2, just enough limbs to satisfy r. |
| 2464 | llmulacc(.add, allocator, tmp, a1, b1); | 2487 | const p2_limbs = math.min(limbs_after_split, a1.len + b1.len); |
| 2465 | const p2 = tmp[0 .. llnormalize(tmp)]; | | |
| 2466 | | 2488 | |
| 2467 | // Add terms p2 * B^2 and p2 * B to the result. | 2489 | mem.set(Limb, tmp[0..p2_limbs], 0); |
| 2468 | _ = llaccum(op, r[split..], p2); | 2490 | llmulacc(.add, allocator, tmp[0..p2_limbs], a1[0..math.min(a1.len, p2_limbs)], b1[0..math.min(b1.len, p2_limbs)]); |
| 2469 | _ = llaccum(op, r[split * 2..], p2); | 2491 | const p2 = tmp[0 .. llnormalize(tmp[0..p2_limbs])]; |
| | 2492 | |
| | 2493 | // Add p2 * B to the result. |
| | 2494 | llaccum(op, r[split..], p2); |
| | 2495 | |
| | 2496 | // Add p2 * B^2 to the result if required. |
| | 2497 | if (limbs_after_split2 > 0) { |
| | 2498 | llaccum(op, r[split * 2..], p2[0..math.min(p2.len, limbs_after_split2)]); |
| | 2499 | } |
| 2470 | | 2500 | |
| 2471 | // Compute p0. | 2501 | // Compute p0. |
| 2472 | mem.set(Limb, p2, 0); | 2502 | // Since a0.len, b0.len <= split and r.len >= split * 2, the full width of p0 needs to be computed. |
| 2473 | llmulacc(.add, allocator, tmp, a0, b0); | 2503 | const p0_limbs = a0.len + b0.len; |
| 2474 | const p0 = tmp[0 .. llnormalize(tmp[0..a0.len + b0.len])]; | 2504 | mem.set(Limb, tmp[0..p0_limbs], 0); |
| | 2505 | llmulacc(.add, allocator, tmp[0..p0_limbs], a0, b0); |
| | 2506 | const p0 = tmp[0 .. llnormalize(tmp[0..p0_limbs])]; |
| | 2507 | |
| | 2508 | // Add p0 to the result. |
| | 2509 | llaccum(op, r, p0); |
| | 2510 | |
| | 2511 | // Add p0 * B to the result. In this case, we may not need all of it. |
| | 2512 | llaccum(op, r[split..], p0[0..math.min(limbs_after_split, p0.len)]); |
| 2475 | | 2513 | |
| 2476 | // Add terms p0 * B and p0 to the result. | | |
| 2477 | _ = llaccum(op, r, p0); | | |
| 2478 | _ = llaccum(op, r[split..], p0); | | |
| 2479 | | 2514 | |
| 2480 | // Finally, compute and add p1. | 2515 | // Finally, compute and add p1. |
| 2481 | const j0_sign = llcmp(a0, a1); | 2516 | // From now on we only need `limbs_after_split` limbs for a0 and b0, since the result of the |
| 2482 | const j1_sign = llcmp(b1, b0); | 2517 | // following computation will be added * B. |
| | 2518 | const a0x = a0[0..std.math.min(a0.len, limbs_after_split)]; |
| | 2519 | const b0x = b0[0..std.math.min(b0.len, limbs_after_split)]; |
| | 2520 | |
| | 2521 | const j0_sign = llcmp(a0x, a1); |
| | 2522 | const j1_sign = llcmp(b1, b0x); |
| 2483 | | 2523 | |
| 2484 | if (j0_sign * j1_sign == 0) { | 2524 | if (j0_sign * j1_sign == 0) { |
| 2485 | // p1 is zero, we don't need to do any computation at all. | 2525 | // p1 is zero, we don't need to do any computation at all. |
| ... | @@ -2492,24 +2532,24 @@ fn llmulaccKaratsuba( | ... | @@ -2492,24 +2532,24 @@ fn llmulaccKaratsuba( |
| 2492 | // Note that in this case, we again need some storage for intermediary results | 2532 | // Note that in this case, we again need some storage for intermediary results |
| 2493 | // j0 and j1. Since we have tmp.len >= 2B, we can store both | 2533 | // j0 and j1. Since we have tmp.len >= 2B, we can store both |
| 2494 | // intermediaries in the already allocated array. | 2534 | // intermediaries in the already allocated array. |
| 2495 | const j0 = tmp[0..a1.len]; | 2535 | const j0 = tmp[0..a.len - split]; |
| 2496 | const j1 = tmp[a1.len..]; | 2536 | const j1 = tmp[a.len - split..]; |
| 2497 | | 2537 | |
| 2498 | // Ensure that no subtraction overflows. | 2538 | // Ensure that no subtraction overflows. |
| 2499 | if (j0_sign == 1) { | 2539 | if (j0_sign == 1) { |
| 2500 | // a0 > a1. | 2540 | // a0 > a1. |
| 2501 | _ = llsubcarry(j0, a0, a1); | 2541 | _ = llsubcarry(j0, a0x, a1); |
| 2502 | } else { | 2542 | } else { |
| 2503 | // a0 < a1. | 2543 | // a0 < a1. |
| 2504 | _ = llsubcarry(j0, a1, a0); | 2544 | _ = llsubcarry(j0, a1, a0x); |
| 2505 | } | 2545 | } |
| 2506 | | 2546 | |
| 2507 | if (j1_sign == 1) { | 2547 | if (j1_sign == 1) { |
| 2508 | // b1 > b0. | 2548 | // b1 > b0. |
| 2509 | _ = llsubcarry(j1, b1, b0); | 2549 | _ = llsubcarry(j1, b1, b0x); |
| 2510 | } else { | 2550 | } else { |
| 2511 | // b1 > b0. | 2551 | // b1 > b0. |
| 2512 | _ = llsubcarry(j1, b0, b1); | 2552 | _ = llsubcarry(j1, b0x, b1); |
| 2513 | } | 2553 | } |
| 2514 | | 2554 | |
| 2515 | if (j0_sign * j1_sign == 1) { | 2555 | if (j0_sign * j1_sign == 1) { |
| ... | @@ -2528,11 +2568,13 @@ fn llmulaccKaratsuba( | ... | @@ -2528,11 +2568,13 @@ fn llmulaccKaratsuba( |
| 2528 | } | 2568 | } |
| 2529 | } | 2569 | } |
| 2530 | | 2570 | |
| 2531 | // r = r (op) a | 2571 | /// r = r (op) a. |
| 2532 | fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) Limb { | 2572 | /// The result is computed modulo `r.len`. |
| | 2573 | fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) void { |
| 2533 | @setRuntimeSafety(debug_safety); | 2574 | @setRuntimeSafety(debug_safety); |
| 2534 | if (op == .sub) { | 2575 | if (op == .sub) { |
| 2535 | return llsubcarry(r, r, a); | 2576 | _ = llsubcarry(r, r, a); |
| | 2577 | return; |
| 2536 | } | 2578 | } |
| 2537 | | 2579 | |
| 2538 | assert(r.len != 0 and a.len != 0); | 2580 | assert(r.len != 0 and a.len != 0); |
| ... | @@ -2551,8 +2593,6 @@ fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) Limb { | ... | @@ -2551,8 +2593,6 @@ fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) Limb { |
| 2551 | while ((carry != 0) and i < r.len) : (i += 1) { | 2593 | while ((carry != 0) and i < r.len) : (i += 1) { |
| 2552 | carry = @boolToInt(@addWithOverflow(Limb, r[i], carry, &r[i])); | 2594 | carry = @boolToInt(@addWithOverflow(Limb, r[i], carry, &r[i])); |
| 2553 | } | 2595 | } |
| 2554 | | | |
| 2555 | return carry; | | |
| 2556 | } | 2596 | } |
| 2557 | | 2597 | |
| 2558 | /// Returns -1, 0, 1 if |a| < |b|, |a| == |b| or |a| > |b| respectively for limbs. | 2598 | /// Returns -1, 0, 1 if |a| < |b|, |a| == |b| or |a| > |b| respectively for limbs. |
| ... | @@ -2583,19 +2623,21 @@ pub fn llcmp(a: []const Limb, b: []const Limb) i8 { | ... | @@ -2583,19 +2623,21 @@ pub fn llcmp(a: []const Limb, b: []const Limb) i8 { |
| 2583 | } | 2623 | } |
| 2584 | } | 2624 | } |
| 2585 | | 2625 | |
| 2586 | // r = r (op) y * xi | 2626 | /// r = r (op) y * xi |
| | 2627 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2587 | fn llmulaccLong(comptime op: AccOp, r: []Limb, a: []const Limb, b: []const Limb) void { | 2628 | fn llmulaccLong(comptime op: AccOp, r: []Limb, a: []const Limb, b: []const Limb) void { |
| 2588 | @setRuntimeSafety(debug_safety); | 2629 | @setRuntimeSafety(debug_safety); |
| 2589 | assert(r.len >= a.len + b.len); | 2630 | assert(r.len >= a.len + b.len); |
| 2590 | assert(a.len >= b.len); | 2631 | assert(a.len >= b.len); |
| 2591 | | 2632 | |
| 2592 | var i: usize = 0; | 2633 | var i: usize = 0; |
| 2593 | while (i < a.len) : (i += 1) { | 2634 | while (i < b.len) : (i += 1) { |
| 2594 | llmulLimb(op, r[i..], b, a[i]); | 2635 | llmulLimb(op, r[i..], a, b[i]); |
| 2595 | } | 2636 | } |
| 2596 | } | 2637 | } |
| 2597 | | 2638 | |
| 2598 | // r = r (op) y * xi | 2639 | /// r = r (op) y * xi |
| | 2640 | /// The result is computed modulo `r.len`. |
| 2599 | fn llmulLimb(comptime op: AccOp, acc: []Limb, y: []const Limb, xi: Limb) void { | 2641 | fn llmulLimb(comptime op: AccOp, acc: []Limb, y: []const Limb, xi: Limb) void { |
| 2600 | @setRuntimeSafety(debug_safety); | 2642 | @setRuntimeSafety(debug_safety); |
| 2601 | if (xi == 0) { | 2643 | if (xi == 0) { |
Robin Voetter <robin@voetter.nl> 2021-10-03 16:03:43+02:00