ab_riscv_interpreter/zvbc/
zvbc_helpers.rs

1//! Opaque helpers for Zvbc extension
2
3use crate::rv64::b::zbc::rv64_zbc_helpers;
4use crate::v::vector_registers::VectorRegistersExt;
5pub use crate::v::zvexx::arith::zvexx_arith_helpers::{OpSrc, check_vreg_group_alignment};
6use crate::v::zvexx::arith::zvexx_arith_helpers::{read_element_u64, sew_mask, write_element_u64};
7use crate::v::zvexx::load::zvexx_load_helpers::mask_bit;
8use ab_riscv_primitives::prelude::*;
9
10/// Lower SEW bits of the carry-less product of two SEW-wide values.
11///
12/// Both inputs are masked to SEW bits before the multiplication so that the VX form (where
13/// the scalar register may carry bits above the SEW boundary) behaves identically to the VV
14/// form (where `read_element_u64` already zero-extends elements to exactly SEW bits).
15#[inline(always)]
16#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
17fn vclmul_element(a: u64, b: u64, sew: Vsew) -> u64 {
18    let mask = sew_mask(sew);
19    let a = a & mask;
20    let b = b & mask;
21    rv64_zbc_helpers::clmul(a, b) & mask
22}
23
24/// Upper SEW bits of the carry-less product of two SEW-wide values.
25///
26/// Both inputs are masked to SEW bits (see [`vclmul_element()`] for rationale).
27///
28/// For SEW < 64, the product fits in 64 bits; the upper half lives at bits
29/// `[2*SEW-1 : SEW]` of `clmul(a, b)`. `clmulh` would return 0 for SEW-bit inputs
30/// since the product never reaches bit 64.
31/// For SEW = 64, `clmulh` directly returns the upper half of the 128-bit product.
32#[inline(always)]
33#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
34fn vclmulh_element(a: u64, b: u64, sew: Vsew) -> u64 {
35    let mask = sew_mask(sew);
36    let a = a & mask;
37    let b = b & mask;
38    if sew == Vsew::E64 {
39        rv64_zbc_helpers::clmulh(a, b)
40    } else {
41        // The 2*SEW-bit product fits in the 64-bit return value of clmul; extract
42        // bits [2*SEW-1 : SEW] and mask back to SEW bits.
43        (rv64_zbc_helpers::clmul(a, b) >> sew.bits_width()) & mask
44    }
45}
46
47/// Execute element-wise carry-less multiplication (lower half) over `vstart..vl`.
48///
49/// For each active element i: `vd[i] = lower_sew_bits(clmul(vs2[i], src[i]))`.
50///
51/// When `vm=true` all elements are active. When `vm=false` the mask register `v0` gates
52/// each element; masked-off elements are left undisturbed (undisturbed policy).
53///
54/// # Safety
55/// - `vd.to_bits() % group_regs == 0` and `vd.to_bits() + group_regs <= 32`
56/// - `vs2.to_bits() % group_regs == 0` and `vs2.to_bits() + group_regs <= 32`
57/// - `src` register (if `Vreg`) satisfies the same alignment as `vs2`
58/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`
59#[inline(always)]
60#[doc(hidden)]
61#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
62pub unsafe fn execute_vclmul<Reg, Env>(
63    env: &mut Env,
64    vd: VReg,
65    vs2: VReg,
66    src: OpSrc,
67    sew: Vsew,
68    vm: bool,
69) where
70    Reg: Register,
71    Env: VectorRegistersExt<Reg>,
72    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
73{
74    let vl = env.vl();
75    let vstart = env.vstart();
76    for i in vstart.range_to(vl) {
77        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
78            continue;
79        }
80        // SAFETY: `vs2 % group_regs == 0` and `vs2 + group_regs <= 32` (caller precondition);
81        // `i < vl <= group_regs * elems_per_reg`, so
82        // `vs2 + i / elems_per_reg < vs2 + group_regs <= 32`
83        let a = unsafe { read_element_u64(env.read_vregs(), vs2, i, sew) };
84        let b = match src {
85            OpSrc::Vreg(vs1_base) => {
86                // SAFETY: caller verified the vs1 register group satisfies the same alignment
87                // constraint as vs2; the index argument is identical, so the same bound holds
88                unsafe { read_element_u64(env.read_vregs(), vs1_base, i, sew) }
89            }
90            OpSrc::Scalar(val) => val,
91        };
92        let result = vclmul_element(a, b, sew);
93        // SAFETY: `vd % group_regs == 0` and `vd + group_regs <= 32` (caller precondition);
94        // `i < vl <= group_regs * elems_per_reg`, so
95        // `vd + i / elems_per_reg < vd + group_regs <= 32`
96        unsafe {
97            write_element_u64(env.write_vregs(), vd, i, sew, result);
98        }
99    }
100    env.mark_vs_dirty();
101    env.reset_vstart();
102}
103
104/// Execute element-wise carry-less multiplication (upper half) over `vstart..vl`.
105///
106/// For each active element i: `vd[i] = upper_sew_bits(clmul(vs2[i], src[i]))`.
107///
108/// When `vm=false`, masked-off elements are left undisturbed.
109///
110/// # Safety
111/// Same register-group constraints as [`execute_vclmul`].
112#[inline(always)]
113#[doc(hidden)]
114#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
115pub unsafe fn execute_vclmulh<Reg, Env>(
116    env: &mut Env,
117    vd: VReg,
118    vs2: VReg,
119    src: OpSrc,
120    sew: Vsew,
121    vm: bool,
122) where
123    Reg: Register,
124    Env: VectorRegistersExt<Reg>,
125    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
126{
127    let vl = env.vl();
128    let vstart = env.vstart();
129    for i in vstart.range_to(vl) {
130        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
131            continue;
132        }
133        // SAFETY: `vs2 % group_regs == 0` and `vs2 + group_regs <= 32`; `i < vl`
134        let a = unsafe { read_element_u64(env.read_vregs(), vs2, i, sew) };
135        let b = match src {
136            OpSrc::Vreg(vs1_base) => {
137                // SAFETY: same alignment constraint as vs2; same index bound
138                unsafe { read_element_u64(env.read_vregs(), vs1_base, i, sew) }
139            }
140            OpSrc::Scalar(val) => val,
141        };
142        let result = vclmulh_element(a, b, sew);
143        // SAFETY: `vd % group_regs == 0` and `vd + group_regs <= 32`; `i < vl`
144        unsafe {
145            write_element_u64(env.write_vregs(), vd, i, sew, result);
146        }
147    }
148    env.mark_vs_dirty();
149    env.reset_vstart();
150}
ab_riscv_interpreter/zvbc/zvbc_helpers.rs

ab_riscv_interpreter/zvbc/
zvbc_helpers.rs
