groestl: add AVX-512/GFNI backend

Author: robbie01

Cargo.lock

@@ -14,6 +14,10 @@ categories = ["cryptography", "no-std"]
 
 [dependencies]
 digest = "0.11.0-rc.0"
+cfg-if = "1"
+
+[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies]
+cpufeatures = "0.2.12"
 
 [dev-dependencies]
 digest = { version = "0.11.0-rc.0", features = ["dev"] }
@@ -25,5 +29,9 @@ default = ["alloc"]
 alloc = ["digest/alloc"]
 zeroize = ["digest/zeroize"]
 
+[lints.rust.unexpected_cfgs]
+level = "warn"
+check-cfg = ["cfg(groestl_force_soft)"]
+
 [package.metadata.docs.rs]
 all-features = true

groestl/Cargo.toml

@@ -1,4 +1,4 @@
-use core::fmt;
+use core::{fmt, slice};
 use digest::{
     HashMarker, InvalidOutputSize, Output,
     block_api::{
@@ -36,9 +36,7 @@ macro_rules! impl_variant {
             #[inline]
             fn update_blocks(&mut self, blocks: &[Block<Self>]) {
                 self.blocks_len += blocks.len() as u64;
-                for block in blocks {
-                    $compress::compress(&mut self.state, block.as_ref());
-                }
+                $compress::compress(&mut self.state, Block::<Self>::cast_slice_to_core(blocks));
             }
         }
 
@@ -72,7 +70,7 @@ macro_rules! impl_variant {
                     self.blocks_len + 1
                 };
                 buffer.len64_padding_be(blocks_len, |block| {
-                    $compress::compress(&mut self.state, block.as_ref())
+                    $compress::compress(&mut self.state, slice::from_ref(block.as_ref()))
                 });
                 let res = $compress::p(&self.state);
                 let n = $compress::COLS / 2;

groestl/src/block_api.rs

@@ -65,23 +65,25 @@ fn rndp(mut x: [u64; COLS], r: u64) -> [u64; COLS] {
     ]
 }
 
-pub(crate) fn compress(h: &mut [u64; COLS], block: &[u8; 128]) {
-    let mut q = [0u64; COLS];
-    for (chunk, v) in block.chunks_exact(8).zip(q.iter_mut()) {
-        *v = u64::from_be_bytes(chunk.try_into().unwrap());
-    }
-    let mut p = [0u64; COLS];
-    for i in 0..COLS {
-        p[i] = h[i] ^ q[i];
-    }
-    for i in 0..ROUNDS {
-        q = rndq(q, i);
-    }
-    for i in 0..ROUNDS {
-        p = rndp(p, i << 56);
-    }
-    for i in 0..COLS {
-        h[i] ^= q[i] ^ p[i];
+pub(crate) fn compress(h: &mut [u64; COLS], blocks: &[[u8; 128]]) {
+    for block in blocks {
+        let mut q = [0u64; COLS];
+        for (chunk, v) in block.chunks_exact(8).zip(q.iter_mut()) {
+            *v = u64::from_be_bytes(chunk.try_into().unwrap());
+        }
+        let mut p = [0u64; COLS];
+        for i in 0..COLS {
+            p[i] = h[i] ^ q[i];
+        }
+        for i in 0..ROUNDS {
+            q = rndq(q, i);
+        }
+        for i in 0..ROUNDS {
+            p = rndp(p, i << 56);
+        }
+        for i in 0..COLS {
+            h[i] ^= q[i] ^ p[i];
+        }
     }
 }
 

groestl/src/compress_long.rs

@@ -1,81 +1,33 @@
-#![allow(clippy::needless_range_loop)]
-use crate::table::TABLE;
-
 pub(crate) const COLS: usize = 8;
-const ROUNDS: u64 = 10;
 
-#[inline(always)]
-fn column(x: &[u64; COLS], c: [usize; 8]) -> u64 {
-    let mut t = 0;
-    for i in 0..8 {
-        let sl = 8 * (7 - i);
-        let idx = ((x[c[i]] >> sl) & 0xFF) as usize;
-        t ^= TABLE[i][idx];
-    }
-    t
-}
+mod soft;
 
-#[inline(always)]
-fn rndq(mut x: [u64; COLS], r: u64) -> [u64; COLS] {
-    for i in 0..COLS {
-        x[i] ^= u64::MAX.wrapping_sub((i as u64) << 4) ^ r;
-    }
-    [
-        column(&x, [1, 3, 5, 7, 0, 2, 4, 6]),
-        column(&x, [2, 4, 6, 0, 1, 3, 5, 7]),
-        column(&x, [3, 5, 7, 1, 2, 4, 6, 0]),
-        column(&x, [4, 6, 0, 2, 3, 5, 7, 1]),
-        column(&x, [5, 7, 1, 3, 4, 6, 0, 2]),
-        column(&x, [6, 0, 2, 4, 5, 7, 1, 3]),
-        column(&x, [7, 1, 3, 5, 6, 0, 2, 4]),
-        column(&x, [0, 2, 4, 6, 7, 1, 3, 5]),
-    ]
-}
+cfg_if::cfg_if! {
+    if #[cfg(any(not(any(target_arch = "x86_64", target_arch = "x86")), groestl_force_soft))] {
+        pub(crate) use soft::*;
+    } else {
+        mod avx512_gfni;
 
-#[inline(always)]
-fn rndp(mut x: [u64; COLS], r: u64) -> [u64; COLS] {
-    for i in 0..COLS {
-        x[i] ^= ((i as u64) << 60) ^ r;
-    }
-    [
-        column(&x, [0, 1, 2, 3, 4, 5, 6, 7]),
-        column(&x, [1, 2, 3, 4, 5, 6, 7, 0]),
-        column(&x, [2, 3, 4, 5, 6, 7, 0, 1]),
-        column(&x, [3, 4, 5, 6, 7, 0, 1, 2]),
-        column(&x, [4, 5, 6, 7, 0, 1, 2, 3]),
-        column(&x, [5, 6, 7, 0, 1, 2, 3, 4]),
-        column(&x, [6, 7, 0, 1, 2, 3, 4, 5]),
-        column(&x, [7, 0, 1, 2, 3, 4, 5, 6]),
-    ]
-}
+        cpufeatures::new!(cpuid_avx512_gfni, "avx", "avx512f", "avx512vbmi", "gfni");
 
-pub(crate) fn compress(h: &mut [u64; COLS], block: &[u8; 64]) {
-    let mut q = [0u64; COLS];
-    for (chunk, v) in block.chunks_exact(8).zip(q.iter_mut()) {
-        *v = u64::from_be_bytes(chunk.try_into().unwrap());
-    }
-    let mut p = [0u64; COLS];
-    for i in 0..COLS {
-        p[i] = h[i] ^ q[i];
-    }
-    for i in 0..ROUNDS {
-        q = rndq(q, i);
-    }
-    for i in 0..ROUNDS {
-        p = rndp(p, i << 56);
-    }
-    for i in 0..COLS {
-        h[i] ^= q[i] ^ p[i];
-    }
-}
+        #[inline(always)]
+        pub(crate) fn compress(h: &mut [u64; COLS], blocks: &[[u8; 64]]) {
+            if cpuid_avx512_gfni::get() {
+                #[allow(unsafe_code)]
+                unsafe { avx512_gfni::compress(h, blocks); }
+            } else {
+                soft::compress(h, blocks);
+            }
+        }
 
-pub(crate) fn p(h: &[u64; COLS]) -> [u64; COLS] {
-    let mut p = *h;
-    for i in 0..ROUNDS {
-        p = rndp(p, i << 56);
-    }
-    for i in 0..COLS {
-        p[i] ^= h[i];
+        #[inline(always)]
+        pub(crate) fn p(h: &[u64; COLS]) -> [u64; COLS] {
+            if cpuid_avx512_gfni::get() {
+                #[allow(unsafe_code)]
+                unsafe { avx512_gfni::p(h) }
+            } else {
+                soft::p(h)
+            }
+        }
     }
-    p
 }