Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 86 additions & 1 deletion banderwagon/src/element.rs
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,48 @@ fn subgroup_check(point: &EdwardsProjective) -> bool {
.is_qr()
}

/// The smallest multi-scalar multiplication the vector path ([`crate::ifma_msm`]) takes. Below
/// it the transposition and bucket setup cost more than the lanes save, and
/// [`crate::msm::msm_windowed`] runs. The cutoff is the same for every thread count.
///
/// The size compared is the MSM's own input, after a proof check has coalesced repeated
/// commitments, not the number of proof queries.
#[cfg(all(feature = "std", target_arch = "x86_64"))]
pub const LANE_MSM_MIN_POINTS: usize = 8192;

/// The multi-scalar multiplication: eight windows per vector register on a host with AVX-512
/// IFMA from [`LANE_MSM_MIN_POINTS`] points up, and [`crate::msm::msm_windowed`] everywhere else.
fn msm(bases: &[EdwardsAffine], scalars: &[Fr]) -> EdwardsProjective {
assert_eq!(
bases.len(),
scalars.len(),
"number of bases should equal number of scalars"
);
#[cfg(all(feature = "std", target_arch = "x86_64"))]
if bases.len() >= LANE_MSM_MIN_POINTS && crate::ifma::available() {
// Keep vector arithmetic across threads: the scalar fallback also parallelizes.
// Rayon can distribute chunks for one proof or run them within concurrent proofs.
#[cfg(feature = "parallel")]
{
use rayon::prelude::*;
let chunk = bases
.len()
.div_ceil(rayon::current_num_threads())
.max(LANE_MSM_MIN_POINTS);
return bases
.par_chunks(chunk)
.zip(scalars.par_chunks(chunk))
// SAFETY: `available()` checked the host's features.
.map(|(b, s)| unsafe { crate::ifma_msm::msm(b, s) })
.reduce(EdwardsProjective::zero, |a, b| a + b);
}
// SAFETY: `available()` checked the host's features.
#[cfg(not(feature = "parallel"))]
return unsafe { crate::ifma_msm::msm(bases, scalars) };
}
crate::msm::msm_windowed(bases, scalars)
}

pub fn multi_scalar_mul(bases: &[Element], scalars: &[Fr]) -> Element {
assert_eq!(
bases.len(),
Expand All @@ -514,7 +556,7 @@ pub fn multi_scalar_mul(bases: &[Element], scalars: &[Fr]) -> Element {
EdwardsProjective::batch_convert_to_mul_base(&bases_inner)
};

Element(crate::msm::msm_windowed(&bases, scalars))
Element(msm(&bases, scalars))
}

/// Multiplies an `Element` by a scalar field element.
Expand Down Expand Up @@ -609,6 +651,49 @@ impl Hash for Element {
mod tests {
extern crate std;
use self::std::println;

#[cfg(feature = "parallel")]
#[test]
fn msm_pool_crossover_preserves_group_result() {
use ark_ec::VariableBaseMSM;
use ark_ff::UniformRand;
use rand_chacha::rand_core::SeedableRng;

let mut rng = rand_chacha::ChaCha20Rng::seed_from_u64(0xf21);
let mut bases: Vec<EdwardsAffine> =
(0..16385).map(|_| EdwardsAffine::rand(&mut rng)).collect();
let mut scalars: Vec<Fr> = (0..16385).map(|_| Fr::rand(&mut rng)).collect();
bases[1] = EdwardsAffine::zero();
scalars[2] = Fr::zero();
scalars[3] = -Fr::one();

for n in [0, 127, 128, 1023, 1024, 4351, 8191, 8192, 8193, 16385] {
let reference = Element(EdwardsProjective::msm(&bases[..n], &scalars[..n]).unwrap());
let projective: Vec<_> = bases[..n]
.iter()
.map(|point| Element((*point).into()))
.collect();
for threads in [1, 2, 8] {
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(threads)
.build()
.unwrap();
pool.install(|| {
assert_eq!(
multi_scalar_mul(&projective, &scalars[..n]),
reference,
"projective n={n} threads={threads}"
);
assert_eq!(
Element(msm(&bases[..n], &scalars[..n])),
reference,
"affine n={n} threads={threads}"
);
});
}
}
}

/// `from_bytes_batch` returns `from_bytes`'s result on every input: accepted points, canonical `x`
/// with no square root, points off the prime subgroup, non-canonical bytes, and a tail shorter than
/// a chunk.
Expand Down
66 changes: 66 additions & 0 deletions banderwagon/src/ifma.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ fn from_limbs52(l: &[u64; 5]) -> BigInt<4> {

pub(crate) struct Ctx {
p: L5,
two_p: L5,
n0: __m512i,
pub(crate) mask: __m512i,
pub(crate) one: L5,
Expand Down Expand Up @@ -248,16 +249,27 @@ unsafe fn ctx() -> Ctx {
one_p_l[i] = s & MASK52;
carry = s >> 52;
}
// 2p as 52-bit limbs (< 2^257: fits)
let mut two_p_l = [0u64; 5];
let mut carry = 0u64;
for i in 0..5 {
let s = 2 * p_scalar[i] + carry;
two_p_l[i] = s & MASK52;
carry = s >> 52;
}
let mut p = [_mm512_setzero_si512(); 5];
let mut two_p = [_mm512_setzero_si512(); 5];
let mut one = [_mm512_setzero_si512(); 5];
let mut one_p = [_mm512_setzero_si512(); 5];
for i in 0..5 {
p[i] = _mm512_set1_epi64(p_scalar[i] as i64);
two_p[i] = _mm512_set1_epi64(two_p_l[i] as i64);
one[i] = _mm512_set1_epi64(one_l[i] as i64);
one_p[i] = _mm512_set1_epi64(one_p_l[i] as i64);
}
let mut c = Ctx {
p,
two_p,
n0: _mm512_set1_epi64(n0 as i64),
mask: _mm512_set1_epi64(MASK52 as i64),
one,
Expand Down Expand Up @@ -744,6 +756,38 @@ pub(crate) unsafe fn store(c: &Ctx, v: &L5) -> [Fq; 8] {
out
}

/// a + b with limb carries normalized. No modular reduction: the caller keeps the bound (inputs < 2p give
/// < 4p; `mont_mul` accepts any pair of inputs whose product is < p·2^260, e.g. 6p × 4p).
#[inline]
#[target_feature(enable = "avx512f,avx512ifma")]
pub(crate) unsafe fn add(c: &Ctx, a: &L5, b: &L5) -> L5 {
let mut r = [_mm512_setzero_si512(); 5];
let mut carry = _mm512_setzero_si512();
for j in 0..5 {
let v = _mm512_add_epi64(_mm512_add_epi64(a[j], b[j]), carry);
r[j] = _mm512_and_si512(v, c.mask);
carry = _mm512_srli_epi64(v, 52);
}
r
}

/// a - b + 2p (b < 2p keeps it non-negative), limbs normalized through a signed borrow chain.
#[inline]
#[target_feature(enable = "avx512f,avx512ifma")]
pub(crate) unsafe fn sub(c: &Ctx, a: &L5, b: &L5) -> L5 {
let mut r = [_mm512_setzero_si512(); 5];
let mut carry = _mm512_setzero_si512();
for j in 0..5 {
let v = _mm512_add_epi64(
_mm512_sub_epi64(_mm512_add_epi64(a[j], c.two_p[j]), b[j]),
carry,
);
r[j] = _mm512_and_si512(v, c.mask);
carry = _mm512_srai_epi64(v, 52); // arithmetic: -1 where the limb went negative
}
r
}

/// Lane mask: which lanes of canonical `a` are zero.
#[inline]
#[target_feature(enable = "avx512f,avx512ifma")]
Expand All @@ -763,6 +807,28 @@ pub(crate) fn bcast_limbs(x: &Fq) -> [u64; 5] {
to_limbs52(&x16.0)
}

/// The same limbs on every lane.
#[inline]
#[target_feature(enable = "avx512f,avx512ifma")]
pub(crate) unsafe fn bcast(l: &[u64; 5]) -> L5 {
let mut v = [_mm512_setzero_si512(); 5];
for i in 0..5 {
v[i] = _mm512_set1_epi64(l[i] as i64);
}
v
}

/// The eight lanes' limbs as a [5][8] array (lane-major storage of points).
#[inline]
#[target_feature(enable = "avx512f,avx512ifma")]
pub(crate) unsafe fn to_words(v: &L5) -> [[u64; 8]; 5] {
let mut w = [[0u64; 8]; 5];
for i in 0..5 {
_mm512_storeu_si512(w[i].as_mut_ptr() as *mut _, v[i]);
}
w
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
Loading
Loading