ab_riscv_interpreter/zvbc/zvbc_helpers.rs
1//! Opaque helpers for Zvbc extension
2
3use crate::rv64::b::zbc::rv64_zbc_helpers;
4use crate::v::vector_registers::VectorRegistersExt;
5pub use crate::v::zvexx::arith::zvexx_arith_helpers::{OpSrc, check_vreg_group_alignment};
6use crate::v::zvexx::arith::zvexx_arith_helpers::{read_element_u64, sew_mask, write_element_u64};
7use crate::v::zvexx::load::zvexx_load_helpers::mask_bit;
8use ab_riscv_primitives::prelude::*;
9use core::fmt;
10
11/// Lower SEW bits of the carry-less product of two SEW-wide values.
12///
13/// Both inputs are masked to SEW bits before the multiplication so that the VX form (where
14/// the scalar register may carry bits above the SEW boundary) behaves identically to the VV
15/// form (where `read_element_u64` already zero-extends elements to exactly SEW bits).
16#[inline(always)]
17#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
18fn vclmul_element(a: u64, b: u64, sew: Vsew) -> u64 {
19 let mask = sew_mask(sew);
20 let a = a & mask;
21 let b = b & mask;
22 rv64_zbc_helpers::clmul(a, b) & mask
23}
24
25/// Upper SEW bits of the carry-less product of two SEW-wide values.
26///
27/// Both inputs are masked to SEW bits (see [`vclmul_element()`] for rationale).
28///
29/// For SEW < 64, the product fits in 64 bits; the upper half lives at bits
30/// `[2*SEW-1 : SEW]` of `clmul(a, b)`. `clmulh` would return 0 for SEW-bit inputs
31/// since the product never reaches bit 64.
32/// For SEW = 64, `clmulh` directly returns the upper half of the 128-bit product.
33#[inline(always)]
34#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
35fn vclmulh_element(a: u64, b: u64, sew: Vsew) -> u64 {
36 let mask = sew_mask(sew);
37 let a = a & mask;
38 let b = b & mask;
39 if sew == Vsew::E64 {
40 rv64_zbc_helpers::clmulh(a, b)
41 } else {
42 // The 2*SEW-bit product fits in the 64-bit return value of clmul; extract
43 // bits [2*SEW-1 : SEW] and mask back to SEW bits.
44 (rv64_zbc_helpers::clmul(a, b) >> sew.bits_width()) & mask
45 }
46}
47
48/// Execute element-wise carry-less multiplication (lower half) over `vstart..vl`.
49///
50/// For each active element i: `vd[i] = lower_sew_bits(clmul(vs2[i], src[i]))`.
51///
52/// When `vm=true` all elements are active. When `vm=false` the mask register `v0` gates
53/// each element; masked-off elements are left undisturbed (undisturbed policy).
54///
55/// # Safety
56/// - `vd.to_bits() % group_regs == 0` and `vd.to_bits() + group_regs <= 32`
57/// - `vs2.to_bits() % group_regs == 0` and `vs2.to_bits() + group_regs <= 32`
58/// - `src` register (if `Vreg`) satisfies the same alignment as `vs2`
59/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`
60#[inline(always)]
61#[doc(hidden)]
62#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
63pub unsafe fn execute_vclmul<Reg, ExtState, CustomError>(
64 ext_state: &mut ExtState,
65 vd: VReg,
66 vs2: VReg,
67 src: OpSrc,
68 sew: Vsew,
69 vm: bool,
70) where
71 Reg: Register,
72 ExtState: VectorRegistersExt<Reg, CustomError>,
73 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
74 CustomError: fmt::Debug,
75{
76 let vl = ext_state.vl();
77 let vstart = ext_state.vstart();
78 for i in vstart.range_to(vl) {
79 if !vm && !mask_bit(ext_state.read_vregs().get(VReg::V0), i) {
80 continue;
81 }
82 // SAFETY: `vs2 % group_regs == 0` and `vs2 + group_regs <= 32` (caller precondition);
83 // `i < vl <= group_regs * elems_per_reg`, so
84 // `vs2 + i / elems_per_reg < vs2 + group_regs <= 32`
85 let a = unsafe { read_element_u64(ext_state.read_vregs(), vs2, i, sew) };
86 let b = match src {
87 OpSrc::Vreg(vs1_base) => {
88 // SAFETY: caller verified the vs1 register group satisfies the same alignment
89 // constraint as vs2; the index argument is identical, so the same bound holds
90 unsafe { read_element_u64(ext_state.read_vregs(), vs1_base, i, sew) }
91 }
92 OpSrc::Scalar(val) => val,
93 };
94 let result = vclmul_element(a, b, sew);
95 // SAFETY: `vd % group_regs == 0` and `vd + group_regs <= 32` (caller precondition);
96 // `i < vl <= group_regs * elems_per_reg`, so
97 // `vd + i / elems_per_reg < vd + group_regs <= 32`
98 unsafe {
99 write_element_u64(ext_state.write_vregs(), vd, i, sew, result);
100 }
101 }
102 ext_state.mark_vs_dirty();
103 ext_state.reset_vstart();
104}
105
106/// Execute element-wise carry-less multiplication (upper half) over `vstart..vl`.
107///
108/// For each active element i: `vd[i] = upper_sew_bits(clmul(vs2[i], src[i]))`.
109///
110/// When `vm=false`, masked-off elements are left undisturbed.
111///
112/// # Safety
113/// Same register-group constraints as [`execute_vclmul`].
114#[inline(always)]
115#[doc(hidden)]
116#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
117pub unsafe fn execute_vclmulh<Reg, ExtState, CustomError>(
118 ext_state: &mut ExtState,
119 vd: VReg,
120 vs2: VReg,
121 src: OpSrc,
122 sew: Vsew,
123 vm: bool,
124) where
125 Reg: Register,
126 ExtState: VectorRegistersExt<Reg, CustomError>,
127 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
128 CustomError: fmt::Debug,
129{
130 let vl = ext_state.vl();
131 let vstart = ext_state.vstart();
132 for i in vstart.range_to(vl) {
133 if !vm && !mask_bit(ext_state.read_vregs().get(VReg::V0), i) {
134 continue;
135 }
136 // SAFETY: `vs2 % group_regs == 0` and `vs2 + group_regs <= 32`; `i < vl`
137 let a = unsafe { read_element_u64(ext_state.read_vregs(), vs2, i, sew) };
138 let b = match src {
139 OpSrc::Vreg(vs1_base) => {
140 // SAFETY: same alignment constraint as vs2; same index bound
141 unsafe { read_element_u64(ext_state.read_vregs(), vs1_base, i, sew) }
142 }
143 OpSrc::Scalar(val) => val,
144 };
145 let result = vclmulh_element(a, b, sew);
146 // SAFETY: `vd % group_regs == 0` and `vd + group_regs <= 32`; `i < vl`
147 unsafe {
148 write_element_u64(ext_state.write_vregs(), vd, i, sew, result);
149 }
150 }
151 ext_state.mark_vs_dirty();
152 ext_state.reset_vstart();
153}