ab_riscv_interpreter/rv64/b/zbc/
rv64_zbc_helpers.rs

1//! Opaque helpers for Zbc extension
2
3use crate::const_utils::ConstRange;
4use const_fn_specialization::const_fn_specialization;
5
6#[const_fn_specialization]
7#[inline(always)]
8#[doc(hidden)]
9#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
10pub fn clmul(a: u64, b: u64) -> u64 {
11    // TODO: Miri is excluded because corresponding intrinsic is not implemented there
12    cfg_select! {
13        all(not(miri), target_arch = "riscv64", target_feature = "zbkc") => {
14            // SAFETY: Compile-time checked for supported feature
15            unsafe { core::arch::riscv64::clmul(a as usize, b as usize) as u64 }
16        }
17        _ => clmul_internal(a, b) as u64,
18    }
19}
20
21#[const_fn_specialization]
22#[inline(always)]
23#[doc(hidden)]
24#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
25pub fn clmulh(a: u64, b: u64) -> u64 {
26    // TODO: Miri is excluded because corresponding intrinsic is not implemented there
27    cfg_select! {
28        all(not(miri), target_arch = "riscv64", target_feature = "zbkc") => {
29            // SAFETY: Compile-time checked for supported feature
30            unsafe { core::arch::riscv64::clmulh(a as usize, b as usize) as u64 }
31        }
32        _ => (clmul_internal(a, b) >> 64) as u64,
33    }
34}
35
36#[const_fn_specialization]
37#[inline(always)]
38#[doc(hidden)]
39#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
40pub fn clmulr(a: u64, b: u64) -> u64 {
41    // TODO: Miri is excluded because corresponding intrinsic is not implemented there
42    cfg_select! {
43        all(not(miri), target_arch = "riscv64", target_feature = "zbc") => {
44            // SAFETY: Compile-time checked for supported feature
45            unsafe { core::arch::riscv64::clmulr(a as usize, b as usize) as u64 }
46        }
47        _ => (clmul_internal(a, b) >> 63) as u64,
48    }
49}
50
51#[const_fn_specialization]
52#[inline(always)]
53#[doc(hidden)]
54#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
55pub const fn clmul(a: u64, b: u64) -> u64 {
56    clmul_internal_generic(a, b) as u64
57}
58
59#[const_fn_specialization]
60#[inline(always)]
61#[doc(hidden)]
62#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
63pub const fn clmulh(a: u64, b: u64) -> u64 {
64    (clmul_internal_generic(a, b) >> 64) as u64
65}
66
67#[const_fn_specialization]
68#[inline(always)]
69#[doc(hidden)]
70#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
71pub const fn clmulr(a: u64, b: u64) -> u64 {
72    (clmul_internal_generic(a, b) >> 63) as u64
73}
74
75/// Carryless multiplication helper
76#[cfg(any(miri, not(all(target_arch = "riscv64", target_feature = "zbc"))))]
77#[inline(always)]
78#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
79fn clmul_internal(a: u64, b: u64) -> u128 {
80    cfg_select! {
81        all(
82            target_arch = "aarch64",
83            target_feature = "neon",
84            target_feature = "aes"
85        ) => {
86            use core::arch::aarch64::vmull_p64;
87
88            // SAFETY: Compile-time checked for supported feature
89            unsafe { vmull_p64(a, b) }
90        }
91        all(target_arch = "x86_64", target_feature = "pclmulqdq") => {
92            use core::arch::x86_64::{__m128i, _mm_clmulepi64_si128, _mm_cvtsi64_si128};
93            use core::mem::transmute;
94
95            // SAFETY: Necessary target features enabled, `__m128i` and `u128` have the same memory
96            // layout
97            unsafe {
98                transmute::<__m128i, u128>(_mm_clmulepi64_si128(
99                    _mm_cvtsi64_si128(a.cast_signed()),
100                    _mm_cvtsi64_si128(b.cast_signed()),
101                    0,
102                ))
103            }
104        }
105        _ => clmul_internal_generic(a, b),
106    }
107}
108
109/// Generic carryless multiplication of two 64-bit values, result fits in 128 bits
110#[inline(always)]
111#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
112const fn clmul_internal_generic(a: u64, b: u64) -> u128 {
113    let mut result = 0u128;
114    let a = u128::from(a);
115    let mut b = b;
116    for i in ConstRange::new(0, u64::BITS) {
117        let bit = u128::from(b & 1);
118        result ^= a.wrapping_shl(i) & (0u128.wrapping_sub(bit));
119        b >>= 1u8;
120    }
121    result
122}
ab_riscv_interpreter/rv64/b/zbc/rv64_zbc_helpers.rs

ab_riscv_interpreter/rv64/b/zbc/
rv64_zbc_helpers.rs
