ab_riscv_interpreter/v/zvexx/muldiv/
zvexx_muldiv_helpers.rs

1//! Opaque helpers for ZveXx extension
2
3use crate::v::vector_registers::{VRegGroup, VectorRegistersExt};
4pub use crate::v::zvexx::arith::zvexx_arith_helpers::{OpSrc, sew_mask, sign_extend};
5use crate::v::zvexx::load::zvexx_load_helpers::{mask_bit, snapshot_mask};
6use crate::v::zvexx::zvexx_helpers::WideningSew;
7use ab_riscv_primitives::prelude::*;
8use core::hint::cold_path;
9
10/// Execute a single-width element-wise arithmetic operation over `0..vl`.
11///
12/// `op` receives `(vs2_elem: u64, src_elem: u64, sew: Vsew)` and returns the `u64` result.
13/// Only the low `sew.bytes()` of the result are written back.
14///
15/// All register groups must have the same `vl`, which holds for groups created from the same
16/// configuration, nothing is written otherwise.
17#[inline(always)]
18#[doc(hidden)]
19#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
20pub fn execute_arith_op<Reg, Env, F>(
21    env: &mut Env,
22    vd: VRegGroup<Env::Hart>,
23    vs2: VRegGroup<Env::Hart>,
24    src: OpSrc<VRegGroup<Env::Hart>>,
25    vm: bool,
26    sew: Vsew,
27    op: F,
28) where
29    Reg: Register,
30    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
31    F: Fn(u64, u64, Vsew) -> u64,
32{
33    let vl = vd.vl();
34    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
35        cold_path();
36        return;
37    };
38    let mask_buf = snapshot_mask(env.read_vregs(), vm);
39    for i in vl.indices() {
40        if !mask_bit(&mask_buf, i) {
41            continue;
42        }
43        let a = env
44            .read_vregs()
45            .read(vs2, i)
46            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
47        let b = match src.vreg {
48            Some(vs1) => env
49                .read_vregs()
50                .read(vs1, i)
51                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
52            None => src.scalar,
53        };
54        let result = op(a, b, sew);
55        env.write_vregs()
56            .write(vd, i, result)
57            .expect("`i < vl` of `vd`; qed");
58    }
59    env.mark_vs_dirty();
60}
61
62/// Execute a single-width widening operation over `0..vl`.
63///
64/// Reads SEW-wide elements from `vs2` and `src`, computes `op`, and writes a 2*SEW-wide result
65/// into `vd`.
66///
67/// All register groups must have the same `vl`, which holds for groups created from the same
68/// configuration, nothing is written otherwise.
69#[inline(always)]
70#[doc(hidden)]
71#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
72pub fn execute_widening_op<Reg, Env, F>(
73    env: &mut Env,
74    vd: VRegGroup<Env::Hart>,
75    vs2: VRegGroup<Env::Hart>,
76    src: OpSrc<VRegGroup<Env::Hart>>,
77    vm: bool,
78    sew: WideningSew<Env::Hart>,
79    op: F,
80) where
81    Reg: Register,
82    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
83    F: Fn(u64, u64, Vsew) -> u64,
84{
85    let sew = sew.narrow();
86    let vl = vd.vl();
87    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
88        cold_path();
89        return;
90    };
91    let mask_buf = snapshot_mask(env.read_vregs(), vm);
92    for i in vl.indices() {
93        if !mask_bit(&mask_buf, i) {
94            continue;
95        }
96        let a = env
97            .read_vregs()
98            .read(vs2, i)
99            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
100        let b = match src.vreg {
101            Some(vs1) => env
102                .read_vregs()
103                .read(vs1, i)
104                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
105            None => src.scalar,
106        };
107        let result = op(a, b, sew);
108        env.write_vregs()
109            .write(vd, i, result)
110            .expect("`i < vl` of `vd`; qed");
111    }
112    env.mark_vs_dirty();
113}
114
115/// Execute a single-width multiply-add where the first multiplier is a vector register group.
116///
117/// `op` receives `(acc: u64, a: u64, b: u64, sew: Vsew)` where `acc` is the current `vd[i]`,
118/// `a` is the element from `a_reg`, and `b` is the element from `src`. Returns the new `vd[i]`.
119///
120/// All register groups must have the same `vl`, which holds for groups created from the same
121/// configuration, nothing is written otherwise.
122#[inline(always)]
123#[doc(hidden)]
124#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
125pub fn execute_muladd_op<Reg, Env, F>(
126    env: &mut Env,
127    vd: VRegGroup<Env::Hart>,
128    a_reg: VRegGroup<Env::Hart>,
129    src: OpSrc<VRegGroup<Env::Hart>>,
130    vm: bool,
131    sew: Vsew,
132    op: F,
133) where
134    Reg: Register,
135    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
136    F: Fn(u64, u64, u64, Vsew) -> u64,
137{
138    let vl = vd.vl();
139    let (Some(a_reg), Some(src)) = (a_reg.with_same_vl(vl), src.with_same_vl(vl)) else {
140        cold_path();
141        return;
142    };
143    let mask_buf = snapshot_mask(env.read_vregs(), vm);
144    for i in vl.indices() {
145        if !mask_bit(&mask_buf, i) {
146            continue;
147        }
148        let acc = env.read_vregs().read(vd, i).expect("`i < vl` of `vd`; qed");
149        let a = env
150            .read_vregs()
151            .read(a_reg, i)
152            .expect("`a_reg` has the same `vl` as `vd`, checked above; qed");
153        let b = match src.vreg {
154            Some(b_reg) => env
155                .read_vregs()
156                .read(b_reg, i)
157                .expect("`b_reg` has the same `vl` as `vd`, checked above; qed"),
158            None => src.scalar,
159        };
160        let result = op(acc, a, b, sew);
161        env.write_vregs()
162            .write(vd, i, result)
163            .expect("`i < vl` of `vd`; qed");
164    }
165    env.mark_vs_dirty();
166}
167
168/// Execute a single-width multiply-add where the first multiplier is a scalar.
169///
170/// Analogous to [`execute_muladd_op`] but `a` is a fixed scalar instead of a register element.
171///
172/// All register groups must have the same `vl`, which holds for groups created from the same
173/// configuration, nothing is written otherwise.
174#[inline(always)]
175#[doc(hidden)]
176#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
177pub fn execute_muladd_scalar_op<Reg, Env, F>(
178    env: &mut Env,
179    vd: VRegGroup<Env::Hart>,
180    scalar: u64,
181    src: OpSrc<VRegGroup<Env::Hart>>,
182    vm: bool,
183    sew: Vsew,
184    op: F,
185) where
186    Reg: Register,
187    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
188    F: Fn(u64, u64, u64, Vsew) -> u64,
189{
190    let vl = vd.vl();
191    let Some(src) = src.with_same_vl(vl) else {
192        cold_path();
193        return;
194    };
195    let mask_buf = snapshot_mask(env.read_vregs(), vm);
196    for i in vl.indices() {
197        if !mask_bit(&mask_buf, i) {
198            continue;
199        }
200        let acc = env.read_vregs().read(vd, i).expect("`i < vl` of `vd`; qed");
201        let b = match src.vreg {
202            Some(b_reg) => env
203                .read_vregs()
204                .read(b_reg, i)
205                .expect("`b_reg` has the same `vl` as `vd`, checked above; qed"),
206            None => src.scalar,
207        };
208        let result = op(acc, scalar, b, sew);
209        env.write_vregs()
210            .write(vd, i, result)
211            .expect("`i < vl` of `vd`; qed");
212    }
213    env.mark_vs_dirty();
214}
215
216/// Execute a widening multiply-add where the first multiplier is a vector register group.
217///
218/// Reads SEW-wide `acc` from the widened `vd` group, SEW-wide `a` from `a_reg`, and SEW-wide
219/// `b` from `src`. Writes a 2*SEW-wide result back into `vd`.
220///
221/// `op` receives `(acc: u64, a: u64, b: u64, sew: Vsew)`.
222///
223/// All register groups must have the same `vl`, which holds for groups created from the same
224/// configuration, nothing is written otherwise.
225#[inline(always)]
226#[doc(hidden)]
227#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
228pub fn execute_widening_muladd_op<Reg, Env, F>(
229    env: &mut Env,
230    vd: VRegGroup<Env::Hart>,
231    a_reg: VRegGroup<Env::Hart>,
232    src: OpSrc<VRegGroup<Env::Hart>>,
233    vm: bool,
234    sew: WideningSew<Env::Hart>,
235    op: F,
236) where
237    Reg: Register,
238    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
239    F: Fn(u64, u64, u64, Vsew) -> u64,
240{
241    let sew = sew.narrow();
242    let vl = vd.vl();
243    let (Some(a_reg), Some(src)) = (a_reg.with_same_vl(vl), src.with_same_vl(vl)) else {
244        cold_path();
245        return;
246    };
247    let mask_buf = snapshot_mask(env.read_vregs(), vm);
248    for i in vl.indices() {
249        if !mask_bit(&mask_buf, i) {
250            continue;
251        }
252        // Read the existing 2*SEW accumulator from vd
253        let acc = env.read_vregs().read(vd, i).expect("`i < vl` of `vd`; qed");
254        let a = env
255            .read_vregs()
256            .read(a_reg, i)
257            .expect("`a_reg` has the same `vl` as `vd`, checked above; qed");
258        let b = match src.vreg {
259            Some(b_reg) => env
260                .read_vregs()
261                .read(b_reg, i)
262                .expect("`b_reg` has the same `vl` as `vd`, checked above; qed"),
263            None => src.scalar,
264        };
265        let result = op(acc, a, b, sew);
266        env.write_vregs()
267            .write(vd, i, result)
268            .expect("`i < vl` of `vd`; qed");
269    }
270    env.mark_vs_dirty();
271}
272
273/// Execute a widening multiply-add where the first multiplier is a scalar.
274///
275/// Analogous to [`execute_widening_muladd_op`] but `a` is a fixed scalar.
276///
277/// All register groups must have the same `vl`, which holds for groups created from the same
278/// configuration, nothing is written otherwise.
279#[inline(always)]
280#[doc(hidden)]
281#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
282pub fn execute_widening_muladd_scalar_op<Reg, Env, F>(
283    env: &mut Env,
284    vd: VRegGroup<Env::Hart>,
285    scalar: u64,
286    src: OpSrc<VRegGroup<Env::Hart>>,
287    vm: bool,
288    sew: WideningSew<Env::Hart>,
289    op: F,
290) where
291    Reg: Register,
292    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
293    F: Fn(u64, u64, u64, Vsew) -> u64,
294{
295    let sew = sew.narrow();
296    let vl = vd.vl();
297    let Some(src) = src.with_same_vl(vl) else {
298        cold_path();
299        return;
300    };
301    let mask_buf = snapshot_mask(env.read_vregs(), vm);
302    for i in vl.indices() {
303        if !mask_bit(&mask_buf, i) {
304            continue;
305        }
306        let acc = env.read_vregs().read(vd, i).expect("`i < vl` of `vd`; qed");
307        let b = match src.vreg {
308            Some(b_reg) => env
309                .read_vregs()
310                .read(b_reg, i)
311                .expect("`b_reg` has the same `vl` as `vd`, checked above; qed"),
312            None => src.scalar,
313        };
314        let result = op(acc, scalar, b, sew);
315        env.write_vregs()
316            .write(vd, i, result)
317            .expect("`i < vl` of `vd`; qed");
318    }
319    env.mark_vs_dirty();
320}
321
322/// Signed × signed high half.
323///
324/// Both operands are sign-extended to i64, multiplied as i128, and the upper SEW bits of the
325/// 2*SEW product are returned (zero-extended to u64 for writeback into a SEW-wide element slot).
326///
327/// Valid for every SEW including 64, because the intermediate product is formed in i128. Whether
328/// SEW=64 is reachable at all is an extension-level decision made by the caller (Zve64x excludes
329/// it, the full "V" extension does not).
330#[inline(always)]
331#[doc(hidden)]
332#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
333pub fn mulh_ss(a: u64, b: u64, sew: Vsew) -> u64 {
334    let sa = sign_extend(a, sew);
335    let sb = sign_extend(b, sew);
336    let product = sa.widening_mul(sb);
337    // Extract bits [2*SEW-1 : SEW] of the product
338    let high = (product >> sew.bits_width()).cast_unsigned() as u64;
339    high & sew_mask(sew)
340}
341
342/// Unsigned × unsigned high half.
343///
344/// Valid for every SEW including 64; see [`mulh_ss()`] for the extension-level caveat.
345#[inline(always)]
346#[doc(hidden)]
347#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
348pub fn mulhu_uu(a: u64, b: u64, sew: Vsew) -> u64 {
349    let ua = a & sew_mask(sew);
350    let ub = b & sew_mask(sew);
351    let product = ua.widening_mul(ub);
352    let high = (product >> sew.bits_width()) as u64;
353    high & sew_mask(sew)
354}
355
356/// Signed × unsigned high half.
357///
358/// `a` (vs2) is the signed operand; `b` (vs1/rs1) is the unsigned operand.
359///
360/// Valid for every SEW including 64; see [`mulh_ss()`] for the extension-level caveat. Note that
361/// the unsigned operand is widened to i128 rather than i64, so a full-width unsigned value at
362/// SEW=64 keeps its magnitude instead of being reinterpreted as negative.
363#[inline(always)]
364#[doc(hidden)]
365#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
366pub fn mulhsu_su(a: u64, b: u64, sew: Vsew) -> u64 {
367    let sa = i128::from(sign_extend(a, sew));
368    let ub = i128::from(b & sew_mask(sew));
369    let product = sa.wrapping_mul(ub);
370    let high = (product >> sew.bits_width()).cast_unsigned() as u64;
371    high & sew_mask(sew)
372}
373
374/// Signed divide with division-by-zero and signed-overflow semantics from the RISC-V V spec §12.11.
375///
376/// - Division by zero: result = all-ones (i.e., −1 as signed SEW-wide integer)
377/// - Signed overflow (MIN / −1): result = MIN (i.e., `1 << (SEW-1)`)
378#[inline(always)]
379#[doc(hidden)]
380#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
381pub fn sdiv(a: u64, b: u64, sew: Vsew) -> u64 {
382    let sa = sign_extend(a, sew);
383    let sb = sign_extend(b, sew);
384    // Division by zero: return all-ones in the SEW-wide slot (= −1 signed)
385    if sb == 0 {
386        return sew_mask(sew);
387    }
388    sa.wrapping_div(sb).cast_unsigned() & sew_mask(sew)
389}
390
391/// Signed remainder with division-by-zero and signed-overflow semantics from the RISC-V V spec
392/// §12.11.
393///
394/// - Division by zero: remainder = dividend
395/// - Signed overflow (MIN % −1): remainder = 0
396#[inline(always)]
397#[doc(hidden)]
398#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
399pub fn srem(a: u64, b: u64, sew: Vsew) -> u64 {
400    let sa = sign_extend(a, sew);
401    let sb = sign_extend(b, sew);
402    // Division by zero: remainder = dividend
403    if sb == 0 {
404        return a & sew_mask(sew);
405    }
406    sa.wrapping_rem(sb).cast_unsigned() & sew_mask(sew)
407}
ab_riscv_interpreter/v/zvexx/muldiv/zvexx_muldiv_helpers.rs

ab_riscv_interpreter/v/zvexx/muldiv/
zvexx_muldiv_helpers.rs
