ab_riscv_interpreter/zvbb/
zvbb_helpers.rs

1//! Opaque helpers for Zvbb extension
2
3use crate::v::vector_registers::{VRegGroup, VectorRegistersExt};
4pub use crate::v::zvexx::arith::zvexx_arith_helpers::OpSrc;
5use crate::v::zvexx::load::zvexx_load_helpers::mask_bit;
6use crate::v::zvexx::zvexx_helpers::WideningSew;
7use ab_riscv_primitives::prelude::*;
8use core::hint::cold_path;
9
10/// Execute element-wise full bit-reversal over `0..vl`, writing SEW-wide results into `vd`.
11///
12/// For each active element i: all bits within `vs2[i]` are reversed end-to-end
13/// (bit 0 <-> bit SEW-1). This differs from `vbrev8`, which reverses bits within each byte while
14/// preserving byte order; `vbrev` also inverts the byte order as a side effect of reversing the
15/// whole element.
16///
17/// When `vm=false`, masked-off elements are left undisturbed.
18///
19/// All register groups must have the same `vl`, which holds for groups created from the same
20/// configuration, nothing is written otherwise.
21#[inline(always)]
22#[doc(hidden)]
23#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
24pub fn execute_vbrev<Reg, Env>(
25    env: &mut Env,
26    vd: VRegGroup<Env::Hart>,
27    vs2: VRegGroup<Env::Hart>,
28    sew: Vsew,
29    vm: bool,
30) where
31    Reg: Register,
32    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
33{
34    let vl = vd.vl();
35    let Some(vs2) = vs2.with_same_vl(vl) else {
36        cold_path();
37        return;
38    };
39    for i in vl.indices() {
40        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
41            continue;
42        }
43        let elem = env
44            .read_vregs()
45            .read(vs2, i)
46            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
47        // `elem` is zero-extended from SEW bits to u64; reverse_bits() on the primitive type
48        // of exactly SEW width naturally handles the upper zero bits from zero-extension
49        let result = match sew {
50            Vsew::E8 => u64::from((elem as u8).reverse_bits()),
51            Vsew::E16 => u64::from((elem as u16).reverse_bits()),
52            Vsew::E32 => u64::from((elem as u32).reverse_bits()),
53            Vsew::E64 => elem.reverse_bits(),
54        };
55        env.write_vregs()
56            .write(vd, i, result)
57            .expect("`i < vl` of `vd`; qed");
58    }
59    env.mark_vs_dirty();
60}
61
62/// Execute element-wise count-leading-zeros over `0..vl`, writing SEW-wide results into `vd`.
63///
64/// For each active element i: `vd[i] = clz(vs2[i])`, counting within the SEW-wide field. An
65/// all-zero element produces SEW, not 64.
66///
67/// When `vm=false`, masked-off elements are left undisturbed.
68///
69/// All register groups must have the same `vl`, which holds for groups created from the same
70/// configuration, nothing is written otherwise.
71#[inline(always)]
72#[doc(hidden)]
73#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
74pub fn execute_vclz<Reg, Env>(
75    env: &mut Env,
76    vd: VRegGroup<Env::Hart>,
77    vs2: VRegGroup<Env::Hart>,
78    sew: Vsew,
79    vm: bool,
80) where
81    Reg: Register,
82    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
83{
84    let vl = vd.vl();
85    let Some(vs2) = vs2.with_same_vl(vl) else {
86        cold_path();
87        return;
88    };
89    let sew_bits = u32::from(sew.bits_width());
90    for i in vl.indices() {
91        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
92            continue;
93        }
94        let elem = env
95            .read_vregs()
96            .read(vs2, i)
97            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
98        // `elem` is zero-extended from SEW bits to u64; `leading_zeros()` on a u64 therefore counts
99        // the extra (64 - SEW) upper zero bits introduced by zero-extension. Subtracting them gives
100        // the count within the SEW-wide field.
101        let clz = elem.leading_zeros() - (64 - sew_bits);
102        env.write_vregs()
103            .write(vd, i, u64::from(clz))
104            .expect("`i < vl` of `vd`; qed");
105    }
106    env.mark_vs_dirty();
107}
108
109/// Execute element-wise count-trailing-zeros over `0..vl`, writing SEW-wide results into `vd`.
110///
111/// For each active element i: `vd[i] = ctz(vs2[i])`, counting within the SEW-wide field. An
112/// all-zero element produces SEW, not 64.
113///
114/// When `vm=false`, masked-off elements are left undisturbed.
115///
116/// All register groups must have the same `vl`, which holds for groups created from the same
117/// configuration, nothing is written otherwise.
118#[inline(always)]
119#[doc(hidden)]
120#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
121pub fn execute_vctz<Reg, Env>(
122    env: &mut Env,
123    vd: VRegGroup<Env::Hart>,
124    vs2: VRegGroup<Env::Hart>,
125    sew: Vsew,
126    vm: bool,
127) where
128    Reg: Register,
129    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
130{
131    let vl = vd.vl();
132    let Some(vs2) = vs2.with_same_vl(vl) else {
133        cold_path();
134        return;
135    };
136    let sew_bits = u32::from(sew.bits_width());
137    for i in vl.indices() {
138        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
139            continue;
140        }
141        let elem = env
142            .read_vregs()
143            .read(vs2, i)
144            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
145        // For non-zero `elem`, `trailing_zeros()` on the zero-extended u64 value is correct: the
146        // upper zero bits do not affect the trailing count. For zero, `trailing_zeros()` returns
147        // 64, but the spec result is SEW; cap at `sew_bits` handles both cases.
148        let ctz = elem.trailing_zeros().min(sew_bits);
149        env.write_vregs()
150            .write(vd, i, u64::from(ctz))
151            .expect("`i < vl` of `vd`; qed");
152    }
153    env.mark_vs_dirty();
154}
155
156/// Execute element-wise population count over `0..vl`, writing SEW-wide results into `vd`.
157///
158/// For each active element i: `vd[i] = popcount(vs2[i])`, in range `[0, SEW]`.
159///
160/// When `vm=false`, masked-off elements are left undisturbed.
161///
162/// All register groups must have the same `vl`, which holds for groups created from the same
163/// configuration, nothing is written otherwise.
164#[inline(always)]
165#[doc(hidden)]
166#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
167pub fn execute_vcpop<Reg, Env>(
168    env: &mut Env,
169    vd: VRegGroup<Env::Hart>,
170    vs2: VRegGroup<Env::Hart>,
171    vm: bool,
172) where
173    Reg: Register,
174    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
175{
176    let vl = vd.vl();
177    let Some(vs2) = vs2.with_same_vl(vl) else {
178        cold_path();
179        return;
180    };
181    for i in vl.indices() {
182        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
183            continue;
184        }
185        let elem = env
186            .read_vregs()
187            .read(vs2, i)
188            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
189        // `elem` is zero-extended from SEW bits; upper bits are already zero, so `count_ones()`
190        // directly gives the population count within the SEW-wide field
191        let cpop = elem.count_ones();
192        env.write_vregs()
193            .write(vd, i, u64::from(cpop))
194            .expect("`i < vl` of `vd`; qed");
195    }
196    env.mark_vs_dirty();
197}
198
199/// Execute element-wise widening shift-left-logical over `0..vl`, writing 2*SEW-wide
200/// results into `vd`.
201///
202/// For each active element i: `vd[i] = zero_extend_to_2SEW(vs2[i]) << (src[i] % (2*SEW))`.
203/// The source operand width is SEW; the destination element width is `double_sew` (2*SEW).
204///
205/// The caller must ensure SEW <= E32 (i.e., `sew.double_width()` is `Some`); passing SEW=E64 is a
206/// programming error that would produce a result wider than u64.
207///
208/// When `vm=false`, masked-off destination elements are left undisturbed.
209///
210/// All register groups must have the same `vl`, which holds for groups created from the same
211/// configuration, nothing is written otherwise.
212#[inline(always)]
213#[doc(hidden)]
214#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
215pub fn execute_vwsll<Reg, Env>(
216    env: &mut Env,
217    vd: VRegGroup<Env::Hart>,
218    vs2: VRegGroup<Env::Hart>,
219    src: OpSrc<VRegGroup<Env::Hart>>,
220    sew: WideningSew<Env::Hart>,
221    vm: bool,
222) where
223    Reg: Register,
224    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
225{
226    let double_sew = sew.wide();
227    let vl = vd.vl();
228    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
229        cold_path();
230        return;
231    };
232    // `double_sew_bits` is always a power of two (16, 32, or 64); `& (bits - 1)` is equivalent to
233    // `% bits` and avoids a division
234    let double_sew_bits = u64::from(double_sew.bits_width());
235    for i in vl.indices() {
236        if !vm && !mask_bit(env.read_vregs().get(VReg::V0), i) {
237            continue;
238        }
239        let a = env
240            .read_vregs()
241            .read(vs2, i)
242            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
243        let amount = match src.vreg {
244            Some(vs1) => env
245                .read_vregs()
246                .read(vs1, i)
247                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
248            None => src.scalar,
249        };
250        let shift = (amount % double_sew_bits) as u32;
251        // `a` is zero-extended from SEW bits; `shift < double_sew_bits <= 64`, so this never shifts
252        // by >= 64.
253        let result = a << shift;
254        env.write_vregs()
255            .write(vd, i, result)
256            .expect("`i < vl` of `vd`; qed");
257    }
258    env.mark_vs_dirty();
259}
ab_riscv_interpreter/zvbb/zvbb_helpers.rs

ab_riscv_interpreter/zvbb/
zvbb_helpers.rs
