ab_riscv_interpreter/zvbc/
zvbc_helpers.rs

1//! Opaque helpers for Zvbc extension
2
3use crate::rv64::b::zbc::rv64_zbc_helpers;
4use crate::v::vector_registers::{VRegGroup, VectorRegistersExt};
5pub use crate::v::zvexx::arith::zvexx_arith_helpers::OpSrc;
6use crate::v::zvexx::arith::zvexx_arith_helpers::sew_mask;
7use crate::v::zvexx::load::zvexx_load_helpers::mask_bit;
8use ab_riscv_primitives::prelude::*;
9use core::hint::cold_path;
10
11/// Lower SEW bits of the carry-less product of two SEW-wide values.
12///
13/// Both inputs are masked to SEW bits before the multiplication so that the VX form (where
14/// the scalar register may carry bits above the SEW boundary) behaves identically to the VV
15/// form (where `VectorRegisterFile::read()` already zero-extends elements to exactly SEW bits).
16#[inline(always)]
17#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
18fn vclmul_element(a: u64, b: u64, sew: Vsew) -> u64 {
19    let mask = sew_mask(sew);
20    let a = a & mask;
21    let b = b & mask;
22    rv64_zbc_helpers::clmul(a, b) & mask
23}
24
25/// Upper SEW bits of the carry-less product of two SEW-wide values.
26///
27/// Both inputs are masked to SEW bits (see [`vclmul_element()`] for rationale).
28///
29/// For SEW < 64, the product fits in 64 bits; the upper half lives at bits
30/// `[2*SEW-1 : SEW]` of `clmul(a, b)`. `clmulh` would return 0 for SEW-bit inputs
31/// since the product never reaches bit 64.
32/// For SEW = 64, `clmulh` directly returns the upper half of the 128-bit product.
33#[inline(always)]
34#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
35fn vclmulh_element(a: u64, b: u64, sew: Vsew) -> u64 {
36    let mask = sew_mask(sew);
37    let a = a & mask;
38    let b = b & mask;
39    if sew == Vsew::E64 {
40        rv64_zbc_helpers::clmulh(a, b)
41    } else {
42        // The 2*SEW-bit product fits in the 64-bit return value of clmul; extract
43        // bits [2*SEW-1 : SEW] and mask back to SEW bits.
44        (rv64_zbc_helpers::clmul(a, b) >> sew.bits_width()) & mask
45    }
46}
47
48/// Execute element-wise carry-less multiplication (lower half) over `0..vl`.
49///
50/// For each active element i: `vd[i] = lower_sew_bits(clmul(vs2[i], src[i]))`.
51///
52/// When `vm=true` all elements are active. When `vm=false` the mask register `v0` gates
53/// each element; masked-off elements are left undisturbed (undisturbed policy).
54///
55/// All register groups must have the same `vl`, which holds for groups created from the same
56/// configuration, nothing is written otherwise.
57#[inline(always)]
58#[doc(hidden)]
59#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
60pub fn execute_vclmul<Reg, Env>(
61    env: &mut Env,
62    vd: VRegGroup<Env::Hart>,
63    vs2: VRegGroup<Env::Hart>,
64    src: OpSrc<VRegGroup<Env::Hart>>,
65    sew: Vsew,
66    vm: bool,
67) where
68    Reg: Register,
69    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
70{
71    let vl = vd.vl();
72    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
73        cold_path();
74        return;
75    };
76    for i in vl.indices() {
77        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
78            continue;
79        }
80        let a = env
81            .read_vregs()
82            .read(vs2, i)
83            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
84        let b = match src.vreg {
85            Some(vs1) => env
86                .read_vregs()
87                .read(vs1, i)
88                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
89            None => src.scalar,
90        };
91        let result = vclmul_element(a, b, sew);
92        env.write_vregs()
93            .write(vd, i, result)
94            .expect("`i < vl` of `vd`; qed");
95    }
96    env.mark_vs_dirty();
97}
98
99/// Execute element-wise carry-less multiplication (upper half) over `0..vl`.
100///
101/// For each active element i: `vd[i] = upper_sew_bits(clmul(vs2[i], src[i]))`.
102///
103/// When `vm=false`, masked-off elements are left undisturbed.
104///
105/// All register groups must have the same `vl`, which holds for groups created from the same
106/// configuration, nothing is written otherwise.
107#[inline(always)]
108#[doc(hidden)]
109#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
110pub fn execute_vclmulh<Reg, Env>(
111    env: &mut Env,
112    vd: VRegGroup<Env::Hart>,
113    vs2: VRegGroup<Env::Hart>,
114    src: OpSrc<VRegGroup<Env::Hart>>,
115    sew: Vsew,
116    vm: bool,
117) where
118    Reg: Register,
119    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
120{
121    let vl = vd.vl();
122    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
123        cold_path();
124        return;
125    };
126    for i in vl.indices() {
127        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
128            continue;
129        }
130        let a = env
131            .read_vregs()
132            .read(vs2, i)
133            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
134        let b = match src.vreg {
135            Some(vs1) => env
136                .read_vregs()
137                .read(vs1, i)
138                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
139            None => src.scalar,
140        };
141        let result = vclmulh_element(a, b, sew);
142        env.write_vregs()
143            .write(vd, i, result)
144            .expect("`i < vl` of `vd`; qed");
145    }
146    env.mark_vs_dirty();
147}
ab_riscv_interpreter/zvbc/zvbc_helpers.rs

ab_riscv_interpreter/zvbc/
zvbc_helpers.rs
