crypto_bigint/uint/
div_limb.rs

1//! Implementation of constant-time division via reciprocal precomputation, as described in
2//! "Improved Division by Invariant Integers" by Niels Möller and Torbjorn Granlund
3//! (DOI: 10.1109/TC.2010.143, <https://gmplib.org/~tege/division-paper.pdf>).
4use subtle::{Choice, ConditionallySelectable, CtOption};
5
6use crate::{CtChoice, Limb, Uint, WideWord, Word};
7
8/// Calculates the reciprocal of the given 32-bit divisor with the highmost bit set.
9#[cfg(target_pointer_width = "32")]
10pub const fn reciprocal(d: Word) -> Word {
11    debug_assert!(d >= (1 << (Word::BITS - 1)));
12
13    let d0 = d & 1;
14    let d10 = d >> 22;
15    let d21 = (d >> 11) + 1;
16    let d31 = (d >> 1) + d0;
17    let v0 = short_div((1 << 24) - (1 << 14) + (1 << 9), 24, d10, 10);
18    let (hi, _lo) = mulhilo(v0 * v0, d21);
19    let v1 = (v0 << 4) - hi - 1;
20
21    // Checks that the expression for `e` can be simplified in the way we did below.
22    debug_assert!(mulhilo(v1, d31).0 == (1 << 16) - 1);
23    let e = Word::MAX - v1.wrapping_mul(d31) + 1 + (v1 >> 1) * d0;
24
25    let (hi, _lo) = mulhilo(v1, e);
26    // Note: the paper does not mention a wrapping add here,
27    // but the 64-bit version has it at this stage, and the function panics without it
28    // when calculating a reciprocal for `Word::MAX`.
29    let v2 = (v1 << 15).wrapping_add(hi >> 1);
30
31    // The paper has `(v2 + 1) * d / 2^32` (there's another 2^32, but it's accounted for later).
32    // If `v2 == 2^32-1` this should give `d`, but we can't achieve this in our wrapping arithmetic.
33    // Hence the `ct_select()`.
34    let x = v2.wrapping_add(1);
35    let (hi, _lo) = mulhilo(x, d);
36    let hi = Limb::ct_select(Limb(d), Limb(hi), Limb(x).ct_is_nonzero()).0;
37
38    v2.wrapping_sub(hi).wrapping_sub(d)
39}
40
41/// Calculates the reciprocal of the given 64-bit divisor with the highmost bit set.
42#[cfg(target_pointer_width = "64")]
43pub const fn reciprocal(d: Word) -> Word {
44    debug_assert!(d >= (1 << (Word::BITS - 1)));
45
46    let d0 = d & 1;
47    let d9 = d >> 55;
48    let d40 = (d >> 24) + 1;
49    let d63 = (d >> 1) + d0;
50    let v0 = short_div((1 << 19) - 3 * (1 << 8), 19, d9 as u32, 9) as u64;
51    let v1 = (v0 << 11) - ((v0 * v0 * d40) >> 40) - 1;
52    let v2 = (v1 << 13) + ((v1 * ((1 << 60) - v1 * d40)) >> 47);
53
54    // Checks that the expression for `e` can be simplified in the way we did below.
55    debug_assert!(mulhilo(v2, d63).0 == (1 << 32) - 1);
56    let e = Word::MAX - v2.wrapping_mul(d63) + 1 + (v2 >> 1) * d0;
57
58    let (hi, _lo) = mulhilo(v2, e);
59    let v3 = (v2 << 31).wrapping_add(hi >> 1);
60
61    // The paper has `(v3 + 1) * d / 2^64` (there's another 2^64, but it's accounted for later).
62    // If `v3 == 2^64-1` this should give `d`, but we can't achieve this in our wrapping arithmetic.
63    // Hence the `ct_select()`.
64    let x = v3.wrapping_add(1);
65    let (hi, _lo) = mulhilo(x, d);
66    let hi = Limb::ct_select(Limb(d), Limb(hi), Limb(x).ct_is_nonzero()).0;
67
68    v3.wrapping_sub(hi).wrapping_sub(d)
69}
70
71/// Returns `u32::MAX` if `a < b` and `0` otherwise.
72#[inline]
73const fn ct_lt(a: u32, b: u32) -> u32 {
74    let bit = (((!a) & b) | (((!a) | b) & (a.wrapping_sub(b)))) >> (u32::BITS - 1);
75    bit.wrapping_neg()
76}
77
78/// Returns `a` if `c == 0` and `b` if `c == u32::MAX`.
79#[inline(always)]
80const fn ct_select(a: u32, b: u32, c: u32) -> u32 {
81    a ^ (c & (a ^ b))
82}
83
84/// Calculates `dividend / divisor`, given `dividend` and `divisor`
85/// along with their maximum bitsizes.
86#[inline(always)]
87const fn short_div(dividend: u32, dividend_bits: u32, divisor: u32, divisor_bits: u32) -> u32 {
88    // TODO: this may be sped up even more using the fact that `dividend` is a known constant.
89
90    // In the paper this is a table lookup, but since we want it to be constant-time,
91    // we have to access all the elements of the table, which is quite large.
92    // So this shift-and-subtract approach is actually faster.
93
94    // Passing `dividend_bits` and `divisor_bits` because calling `.leading_zeros()`
95    // causes a significant slowdown, and we know those values anyway.
96
97    let mut dividend = dividend;
98    let mut divisor = divisor << (dividend_bits - divisor_bits);
99    let mut quotient: u32 = 0;
100    let mut i = dividend_bits - divisor_bits + 1;
101
102    while i > 0 {
103        i -= 1;
104        let bit = ct_lt(dividend, divisor);
105        dividend = ct_select(dividend.wrapping_sub(divisor), dividend, bit);
106        divisor >>= 1;
107        let inv_bit = !bit;
108        quotient |= (inv_bit >> (u32::BITS - 1)) << i;
109    }
110
111    quotient
112}
113
114/// Multiplies `x` and `y`, returning the most significant
115/// and the least significant words as `(hi, lo)`.
116#[inline(always)]
117const fn mulhilo(x: Word, y: Word) -> (Word, Word) {
118    let res = (x as WideWord) * (y as WideWord);
119    ((res >> Word::BITS) as Word, res as Word)
120}
121
122/// Adds wide numbers represented by pairs of (most significant word, least significant word)
123/// and returns the result in the same format `(hi, lo)`.
124#[inline(always)]
125const fn addhilo(x_hi: Word, x_lo: Word, y_hi: Word, y_lo: Word) -> (Word, Word) {
126    let res = (((x_hi as WideWord) << Word::BITS) | (x_lo as WideWord))
127        + (((y_hi as WideWord) << Word::BITS) | (y_lo as WideWord));
128    ((res >> Word::BITS) as Word, res as Word)
129}
130
131/// Calculate the quotient and the remainder of the division of a wide word
132/// (supplied as high and low words) by `d`, with a precalculated reciprocal `v`.
133#[inline(always)]
134const fn div2by1(u1: Word, u0: Word, reciprocal: &Reciprocal) -> (Word, Word) {
135    let d = reciprocal.divisor_normalized;
136
137    debug_assert!(d >= (1 << (Word::BITS - 1)));
138    debug_assert!(u1 < d);
139
140    let (q1, q0) = mulhilo(reciprocal.reciprocal, u1);
141    let (q1, q0) = addhilo(q1, q0, u1, u0);
142    let q1 = q1.wrapping_add(1);
143    let r = u0.wrapping_sub(q1.wrapping_mul(d));
144
145    let r_gt_q0 = Limb::ct_lt(Limb(q0), Limb(r));
146    let q1 = Limb::ct_select(Limb(q1), Limb(q1.wrapping_sub(1)), r_gt_q0).0;
147    let r = Limb::ct_select(Limb(r), Limb(r.wrapping_add(d)), r_gt_q0).0;
148
149    // If this was a normal `if`, we wouldn't need wrapping ops, because there would be no overflow.
150    // But since we calculate both results either way, we have to wrap.
151    // Added an assert to still check the lack of overflow in debug mode.
152    debug_assert!(r < d || q1 < Word::MAX);
153    let r_ge_d = Limb::ct_le(Limb(d), Limb(r));
154    let q1 = Limb::ct_select(Limb(q1), Limb(q1.wrapping_add(1)), r_ge_d).0;
155    let r = Limb::ct_select(Limb(r), Limb(r.wrapping_sub(d)), r_ge_d).0;
156
157    (q1, r)
158}
159
160/// A pre-calculated reciprocal for division by a single limb.
161#[derive(Copy, Clone, Debug, PartialEq, Eq)]
162pub struct Reciprocal {
163    divisor_normalized: Word,
164    shift: u32,
165    reciprocal: Word,
166}
167
168impl Reciprocal {
169    /// Pre-calculates a reciprocal for a known divisor,
170    /// to be used in the single-limb division later.
171    /// Returns the reciprocal, and the truthy value if `divisor != 0`
172    /// and the falsy value otherwise.
173    ///
174    /// Note: if the returned flag is falsy, the returned reciprocal object is still self-consistent
175    /// and can be passed to functions here without causing them to panic,
176    /// but the results are naturally not to be used.
177    pub const fn ct_new(divisor: Limb) -> (Self, CtChoice) {
178        // Assuming this is constant-time for primitive types.
179        let shift = divisor.0.leading_zeros();
180
181        #[allow(trivial_numeric_casts)]
182        let is_some = Limb((Word::BITS - shift) as Word).ct_is_nonzero();
183
184        // If `divisor = 0`, shifting `divisor` by `leading_zeros == Word::BITS` will cause a panic.
185        // Have to substitute a "bogus" shift in that case.
186        #[allow(trivial_numeric_casts)]
187        let shift_limb = Limb::ct_select(Limb::ZERO, Limb(shift as Word), is_some);
188
189        // Need to provide bogus normalized divisor and reciprocal too,
190        // so that we don't get a panic in low-level functions.
191        let divisor_normalized = divisor.shl(shift_limb);
192        let divisor_normalized = Limb::ct_select(Limb::MAX, divisor_normalized, is_some).0;
193
194        #[allow(trivial_numeric_casts)]
195        let shift = shift_limb.0 as u32;
196
197        (
198            Self {
199                divisor_normalized,
200                shift,
201                reciprocal: reciprocal(divisor_normalized),
202            },
203            is_some,
204        )
205    }
206
207    /// Returns a default instance of this object.
208    /// It is a self-consistent `Reciprocal` that will not cause panics in functions that take it.
209    ///
210    /// NOTE: intended for using it as a placeholder during compile-time array generation,
211    /// don't rely on the contents.
212    pub const fn default() -> Self {
213        Self {
214            divisor_normalized: Word::MAX,
215            shift: 0,
216            // The result of calling `reciprocal(Word::MAX)`
217            // This holds both for 32- and 64-bit versions.
218            reciprocal: 1,
219        }
220    }
221
222    /// A non-const-fn version of `new_const()`, wrapping the result in a `CtOption`.
223    pub fn new(divisor: Limb) -> CtOption<Self> {
224        let (rec, is_some) = Self::ct_new(divisor);
225        CtOption::new(rec, is_some.into())
226    }
227}
228
229impl ConditionallySelectable for Reciprocal {
230    fn conditional_select(a: &Self, b: &Self, choice: Choice) -> Self {
231        Self {
232            divisor_normalized: Word::conditional_select(
233                &a.divisor_normalized,
234                &b.divisor_normalized,
235                choice,
236            ),
237            shift: u32::conditional_select(&a.shift, &b.shift, choice),
238            reciprocal: Word::conditional_select(&a.reciprocal, &b.reciprocal, choice),
239        }
240    }
241}
242
243// `CtOption.map()` needs this; for some reason it doesn't use the value it already has
244// for the `None` branch.
245impl Default for Reciprocal {
246    fn default() -> Self {
247        Self::default()
248    }
249}
250
251/// Divides `u` by the divisor encoded in the `reciprocal`, and returns
252/// the quotient and the remainder.
253#[inline(always)]
254pub(crate) const fn div_rem_limb_with_reciprocal<const L: usize>(
255    u: &Uint<L>,
256    reciprocal: &Reciprocal,
257) -> (Uint<L>, Limb) {
258    let (u_shifted, u_hi) = u.shl_limb(reciprocal.shift as usize);
259    let mut r = u_hi.0;
260    let mut q = [Limb::ZERO; L];
261
262    let mut j = L;
263    while j > 0 {
264        j -= 1;
265        let (qj, rj) = div2by1(r, u_shifted.as_limbs()[j].0, reciprocal);
266        q[j] = Limb(qj);
267        r = rj;
268    }
269    (Uint::<L>::new(q), Limb(r >> reciprocal.shift))
270}
271
272#[cfg(test)]
273mod tests {
274    use super::{div2by1, Reciprocal};
275    use crate::{Limb, Word};
276    #[test]
277    fn div2by1_overflow() {
278        // A regression test for a situation when in div2by1() an operation (`q1 + 1`)
279        // that is protected from overflowing by a condition in the original paper (`r >= d`)
280        // still overflows because we're calculating the results for both branches.
281        let r = Reciprocal::new(Limb(Word::MAX - 1)).unwrap();
282        assert_eq!(
283            div2by1(Word::MAX - 2, Word::MAX - 63, &r),
284            (Word::MAX, Word::MAX - 65)
285        );
286    }
287}
crypto_bigint/uint/div_limb.rs

crypto_bigint/uint/
div_limb.rs
