| ... | ... | @@ -658,7 +658,7 @@ pub const Mutable = struct { |
| 658 | 658 | |
| 659 | 659 | mem.set(Limb, rma.limbs[0 .. a.limbs.len + b.limbs.len + 1], 0); |
| 660 | 660 | |
| 661 | | llmulacc(.add, allocator, rma.limbs, a.limbs, b.limbs); |
| 661 | _ = llmulacc(.add, allocator, rma.limbs, a.limbs, b.limbs); |
| 662 | 662 | |
| 663 | 663 | rma.normalize(a.limbs.len + b.limbs.len); |
| 664 | 664 | rma.positive = (a.positive == b.positive); |
| ... | ... | @@ -2365,9 +2365,12 @@ const AccOp = enum { |
| 2365 | 2365 | /// |
| 2366 | 2366 | /// r = r (op) a * b |
| 2367 | 2367 | /// r MUST NOT alias any of a or b. |
| 2368 | /// |
| 2369 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2368 | 2370 | fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []const Limb, b: []const Limb) void { |
| 2369 | 2371 | @setRuntimeSafety(debug_safety); |
| 2370 | | assert(r.len >= a.len + b.len); |
| 2372 | assert(r.len >= a.len); |
| 2373 | assert(r.len >= b.len); |
| 2371 | 2374 | |
| 2372 | 2375 | // Order greatest first. |
| 2373 | 2376 | var x = a; |
| ... | ... | @@ -2395,6 +2398,8 @@ fn llmulacc(comptime op: AccOp, opt_allocator: ?*Allocator, r: []Limb, a: []cons |
| 2395 | 2398 | /// |
| 2396 | 2399 | /// r = r (op) a * b |
| 2397 | 2400 | /// r MUST NOT alias any of a or b. |
| 2401 | /// |
| 2402 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2398 | 2403 | fn llmulaccKaratsuba( |
| 2399 | 2404 | comptime op: AccOp, |
| 2400 | 2405 | allocator: *Allocator, |
| ... | ... | @@ -2403,7 +2408,7 @@ fn llmulaccKaratsuba( |
| 2403 | 2408 | b: []const Limb, |
| 2404 | 2409 | ) error{OutOfMemory}!void { |
| 2405 | 2410 | @setRuntimeSafety(debug_safety); |
| 2406 | | assert(r.len >= a.len + b.len); |
| 2411 | assert(r.len >= a.len); |
| 2407 | 2412 | assert(a.len >= b.len); |
| 2408 | 2413 | |
| 2409 | 2414 | // Classical karatsuba algorithm: |
| ... | ... | @@ -2437,49 +2442,84 @@ fn llmulaccKaratsuba( |
| 2437 | 2442 | // |
| 2438 | 2443 | // Note, when B is a multiple of the limb size, multiplies by B amount to shifts or |
| 2439 | 2444 | // slices of a limbs array. |
| 2445 | // |
| 2446 | // This function computes the result of the multiplication modulo r.len. This means: |
| 2447 | // - p2 and p1 only need to be computed modulo r.len - B. |
| 2448 | // - In the case of p2, p2 * B^2 needs to be added modulo r.len - 2 * B. |
| 2440 | 2449 | |
| 2441 | 2450 | const split = b.len / 2; // B |
| 2451 | |
| 2452 | const limbs_after_split = r.len - split; // Limbs to compute for p1 and p2. |
| 2453 | const limbs_after_split2 = r.len - split * 2; // Limbs to add for p2 * B^2. |
| 2454 | |
| 2455 | // For a0 and b0 we need the full range. |
| 2442 | 2456 | const a0 = a[0..llnormalize(a[0..split])]; |
| 2443 | | const a1 = a[split..][0..llnormalize(a[split..])]; |
| 2444 | 2457 | const b0 = b[0..llnormalize(b[0..split])]; |
| 2445 | | const b1 = b[split..][0..llnormalize(b[split..])]; |
| 2446 | 2458 | |
| 2447 | | // Note that the above slices work because we have a.len > b.len. |
| 2448 | | // We now also have: |
| 2449 | | // a1.len >= a0.len |
| 2450 | | // a1.len >= b1.len >= b0.len |
| 2451 | | // a0.len == b0.len |
| 2459 | // For a1 and b1 we only need `limbs_after_split` limbs. |
| 2460 | const a1 = blk: { |
| 2461 | var a1 = a[split..]; |
| 2462 | a1.len = math.min(llnormalize(a1), limbs_after_split); |
| 2463 | break :blk a1; |
| 2464 | }; |
| 2465 | |
| 2466 | const b1 = blk: { |
| 2467 | var b1 = b[split..]; |
| 2468 | b1.len = math.min(llnormalize(b1), limbs_after_split); |
| 2469 | break :blk b1; |
| 2470 | }; |
| 2471 | |
| 2472 | // Note that the above slices relative to `split` work because we have a.len > b.len. |
| 2452 | 2473 | |
| 2453 | 2474 | // We need some temporary memory to store intermediate results. |
| 2454 | 2475 | // Note, we can reduce the amount of temporaries we need by reordering the computation here: |
| 2455 | 2476 | // ab = p2 * B^2 + (p0 + p1 + p2) * B + p0 |
| 2456 | 2477 | // = p2 * B^2 + (p0 * B + p1 * B + p2 * B) + p0 |
| 2457 | 2478 | // = (p2 * B^2 + p2 * B) + (p0 * B + p0) + p1 * B |
| 2458 | | // By allocating a1.len * b1.len we can be sure that all the intermediary results fit. |
| 2479 | |
| 2480 | // Allocate at least enough memory to be able to multiply the upper two segments of a and b, assuming |
| 2481 | // no overflow. |
| 2459 | 2482 | const tmp = try allocator.alloc(Limb, a.len - split + b.len - split); |
| 2460 | 2483 | defer allocator.free(tmp); |
| 2461 | 2484 | |
| 2462 | 2485 | // Compute p2. |
| 2463 | | mem.set(Limb, tmp, 0); |
| 2464 | | llmulacc(.add, allocator, tmp, a1, b1); |
| 2465 | | const p2 = tmp[0 .. llnormalize(tmp)]; |
| 2486 | // Note, we don't need to compute all of p2, just enough limbs to satisfy r. |
| 2487 | const p2_limbs = math.min(limbs_after_split, a1.len + b1.len); |
| 2466 | 2488 | |
| 2467 | | // Add terms p2 * B^2 and p2 * B to the result. |
| 2468 | | _ = llaccum(op, r[split..], p2); |
| 2469 | | _ = llaccum(op, r[split * 2..], p2); |
| 2489 | mem.set(Limb, tmp[0..p2_limbs], 0); |
| 2490 | llmulacc(.add, allocator, tmp[0..p2_limbs], a1[0..math.min(a1.len, p2_limbs)], b1[0..math.min(b1.len, p2_limbs)]); |
| 2491 | const p2 = tmp[0 .. llnormalize(tmp[0..p2_limbs])]; |
| 2492 | |
| 2493 | // Add p2 * B to the result. |
| 2494 | llaccum(op, r[split..], p2); |
| 2495 | |
| 2496 | // Add p2 * B^2 to the result if required. |
| 2497 | if (limbs_after_split2 > 0) { |
| 2498 | llaccum(op, r[split * 2..], p2[0..math.min(p2.len, limbs_after_split2)]); |
| 2499 | } |
| 2470 | 2500 | |
| 2471 | 2501 | // Compute p0. |
| 2472 | | mem.set(Limb, p2, 0); |
| 2473 | | llmulacc(.add, allocator, tmp, a0, b0); |
| 2474 | | const p0 = tmp[0 .. llnormalize(tmp[0..a0.len + b0.len])]; |
| 2502 | // Since a0.len, b0.len <= split and r.len >= split * 2, the full width of p0 needs to be computed. |
| 2503 | const p0_limbs = a0.len + b0.len; |
| 2504 | mem.set(Limb, tmp[0..p0_limbs], 0); |
| 2505 | llmulacc(.add, allocator, tmp[0..p0_limbs], a0, b0); |
| 2506 | const p0 = tmp[0 .. llnormalize(tmp[0..p0_limbs])]; |
| 2507 | |
| 2508 | // Add p0 to the result. |
| 2509 | llaccum(op, r, p0); |
| 2510 | |
| 2511 | // Add p0 * B to the result. In this case, we may not need all of it. |
| 2512 | llaccum(op, r[split..], p0[0..math.min(limbs_after_split, p0.len)]); |
| 2475 | 2513 | |
| 2476 | | // Add terms p0 * B and p0 to the result. |
| 2477 | | _ = llaccum(op, r, p0); |
| 2478 | | _ = llaccum(op, r[split..], p0); |
| 2479 | 2514 | |
| 2480 | 2515 | // Finally, compute and add p1. |
| 2481 | | const j0_sign = llcmp(a0, a1); |
| 2482 | | const j1_sign = llcmp(b1, b0); |
| 2516 | // From now on we only need `limbs_after_split` limbs for a0 and b0, since the result of the |
| 2517 | // following computation will be added * B. |
| 2518 | const a0x = a0[0..std.math.min(a0.len, limbs_after_split)]; |
| 2519 | const b0x = b0[0..std.math.min(b0.len, limbs_after_split)]; |
| 2520 | |
| 2521 | const j0_sign = llcmp(a0x, a1); |
| 2522 | const j1_sign = llcmp(b1, b0x); |
| 2483 | 2523 | |
| 2484 | 2524 | if (j0_sign * j1_sign == 0) { |
| 2485 | 2525 | // p1 is zero, we don't need to do any computation at all. |
| ... | ... | @@ -2492,24 +2532,24 @@ fn llmulaccKaratsuba( |
| 2492 | 2532 | // Note that in this case, we again need some storage for intermediary results |
| 2493 | 2533 | // j0 and j1. Since we have tmp.len >= 2B, we can store both |
| 2494 | 2534 | // intermediaries in the already allocated array. |
| 2495 | | const j0 = tmp[0..a1.len]; |
| 2496 | | const j1 = tmp[a1.len..]; |
| 2535 | const j0 = tmp[0..a.len - split]; |
| 2536 | const j1 = tmp[a.len - split..]; |
| 2497 | 2537 | |
| 2498 | 2538 | // Ensure that no subtraction overflows. |
| 2499 | 2539 | if (j0_sign == 1) { |
| 2500 | 2540 | // a0 > a1. |
| 2501 | | _ = llsubcarry(j0, a0, a1); |
| 2541 | _ = llsubcarry(j0, a0x, a1); |
| 2502 | 2542 | } else { |
| 2503 | 2543 | // a0 < a1. |
| 2504 | | _ = llsubcarry(j0, a1, a0); |
| 2544 | _ = llsubcarry(j0, a1, a0x); |
| 2505 | 2545 | } |
| 2506 | 2546 | |
| 2507 | 2547 | if (j1_sign == 1) { |
| 2508 | 2548 | // b1 > b0. |
| 2509 | | _ = llsubcarry(j1, b1, b0); |
| 2549 | _ = llsubcarry(j1, b1, b0x); |
| 2510 | 2550 | } else { |
| 2511 | 2551 | // b1 > b0. |
| 2512 | | _ = llsubcarry(j1, b0, b1); |
| 2552 | _ = llsubcarry(j1, b0x, b1); |
| 2513 | 2553 | } |
| 2514 | 2554 | |
| 2515 | 2555 | if (j0_sign * j1_sign == 1) { |
| ... | ... | @@ -2528,11 +2568,13 @@ fn llmulaccKaratsuba( |
| 2528 | 2568 | } |
| 2529 | 2569 | } |
| 2530 | 2570 | |
| 2531 | | // r = r (op) a |
| 2532 | | fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) Limb { |
| 2571 | /// r = r (op) a. |
| 2572 | /// The result is computed modulo `r.len`. |
| 2573 | fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) void { |
| 2533 | 2574 | @setRuntimeSafety(debug_safety); |
| 2534 | 2575 | if (op == .sub) { |
| 2535 | | return llsubcarry(r, r, a); |
| 2576 | _ = llsubcarry(r, r, a); |
| 2577 | return; |
| 2536 | 2578 | } |
| 2537 | 2579 | |
| 2538 | 2580 | assert(r.len != 0 and a.len != 0); |
| ... | ... | @@ -2551,8 +2593,6 @@ fn llaccum(comptime op: AccOp, r: []Limb, a: []const Limb) Limb { |
| 2551 | 2593 | while ((carry != 0) and i < r.len) : (i += 1) { |
| 2552 | 2594 | carry = @boolToInt(@addWithOverflow(Limb, r[i], carry, &r[i])); |
| 2553 | 2595 | } |
| 2554 | | |
| 2555 | | return carry; |
| 2556 | 2596 | } |
| 2557 | 2597 | |
| 2558 | 2598 | /// Returns -1, 0, 1 if |a| < |b|, |a| == |b| or |a| > |b| respectively for limbs. |
| ... | ... | @@ -2583,19 +2623,21 @@ pub fn llcmp(a: []const Limb, b: []const Limb) i8 { |
| 2583 | 2623 | } |
| 2584 | 2624 | } |
| 2585 | 2625 | |
| 2586 | | // r = r (op) y * xi |
| 2626 | /// r = r (op) y * xi |
| 2627 | /// The result is computed modulo `r.len`. When `r.len >= a.len + b.len`, no overflow occurs. |
| 2587 | 2628 | fn llmulaccLong(comptime op: AccOp, r: []Limb, a: []const Limb, b: []const Limb) void { |
| 2588 | 2629 | @setRuntimeSafety(debug_safety); |
| 2589 | 2630 | assert(r.len >= a.len + b.len); |
| 2590 | 2631 | assert(a.len >= b.len); |
| 2591 | 2632 | |
| 2592 | 2633 | var i: usize = 0; |
| 2593 | | while (i < a.len) : (i += 1) { |
| 2594 | | llmulLimb(op, r[i..], b, a[i]); |
| 2634 | while (i < b.len) : (i += 1) { |
| 2635 | llmulLimb(op, r[i..], a, b[i]); |
| 2595 | 2636 | } |
| 2596 | 2637 | } |
| 2597 | 2638 | |
| 2598 | | // r = r (op) y * xi |
| 2639 | /// r = r (op) y * xi |
| 2640 | /// The result is computed modulo `r.len`. |
| 2599 | 2641 | fn llmulLimb(comptime op: AccOp, acc: []Limb, y: []const Limb, xi: Limb) void { |
| 2600 | 2642 | @setRuntimeSafety(debug_safety); |
| 2601 | 2643 | if (xi == 0) { |
Robin Voetter <robin@voetter.nl> 2021-10-03 16:03:43+02:00