ab_riscv_interpreter/v/zvexx/carry/
zvexx_carry_helpers.rs

1//! Opaque helpers for ZveXx extension
2
3use crate::v::vector_registers::{VRegGroup, VectorRegisterFile, VectorRegistersExt};
4pub use crate::v::zvexx::arith::zvexx_arith_helpers::{OpSrc, check_mask_dest_overlap};
5use crate::v::zvexx::arith::zvexx_arith_helpers::{sew_mask, write_mask_bit};
6use crate::v::zvexx::load::zvexx_load_helpers::mask_bit;
7use ab_riscv_primitives::prelude::*;
8use core::hint::cold_path;
9
10/// Read a single mask bit from vector register `v0` at element index `i`.
11///
12/// Used to retrieve the per-element carry-in or borrow-in for vadc/vsbc.
13#[inline(always)]
14#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
15pub(in super::super) fn carry_bit<Hart>(vregs: &VectorRegisterFile<Hart>, i: u16) -> u64
16where
17    Hart: VectorHartConfig,
18{
19    let v0 = vregs.get(VReg::V0);
20    u64::from(mask_bit(v0, i))
21}
22
23/// Execute an element-wise add-with-carry over `0..vl`, writing SEW-wide data results into
24/// `vd`.
25///
26/// Carry-in for each element is read from `v0[i]` when `WITH_CARRY` is true. All elements in
27/// `0..vl` are processed unconditionally (no execution mask).
28///
29/// All register groups must have the same `vl`, which holds for groups created from the same
30/// configuration, nothing is written otherwise.
31#[inline(always)]
32#[doc(hidden)]
33#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
34pub fn execute_carry_add<const WITH_CARRY: bool, Reg, Env>(
35    env: &mut Env,
36    vd: VRegGroup<Env::Hart>,
37    vs2: VRegGroup<Env::Hart>,
38    src: OpSrc<VRegGroup<Env::Hart>>,
39) where
40    Reg: Register,
41    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
42{
43    let vl = vd.vl();
44    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
45        cold_path();
46        return;
47    };
48    for i in vl.indices() {
49        let a = env
50            .read_vregs()
51            .read(vs2, i)
52            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
53        let b = match src.vreg {
54            Some(vs1) => env
55                .read_vregs()
56                .read(vs1, i)
57                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
58            None => src.scalar,
59        };
60        let c = if WITH_CARRY {
61            carry_bit(env.read_vregs(), i)
62        } else {
63            0
64        };
65
66        // Wrap naturally: `VectorRegisterFile::write()` writes only the low `SEW` bits
67        let result = a.wrapping_add(b).wrapping_add(c);
68        env.write_vregs()
69            .write(vd, i, result)
70            .expect("`i < vl` of `vd`; qed");
71    }
72
73    env.mark_vs_dirty();
74}
75
76/// Execute an element-wise subtract-with-borrow over `0..vl`, writing SEW-wide data results
77/// into `vd`.
78///
79/// Borrow-in for each element is read from `v0[i]` (always true for vsbc). All elements in
80/// `0..vl` are processed unconditionally.
81///
82/// All register groups must have the same `vl`, which holds for groups created from the same
83/// configuration, nothing is written otherwise.
84#[inline(always)]
85#[doc(hidden)]
86#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
87pub fn execute_carry_sub<Reg, Env>(
88    env: &mut Env,
89    vd: VRegGroup<Env::Hart>,
90    vs2: VRegGroup<Env::Hart>,
91    src: OpSrc<VRegGroup<Env::Hart>>,
92) where
93    Reg: Register,
94    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
95{
96    let vl = vd.vl();
97    let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
98        cold_path();
99        return;
100    };
101    for i in vl.indices() {
102        let a = env
103            .read_vregs()
104            .read(vs2, i)
105            .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
106        let b = match src.vreg {
107            Some(vs1) => env
108                .read_vregs()
109                .read(vs1, i)
110                .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
111            None => src.scalar,
112        };
113        let borrow = carry_bit(env.read_vregs(), i);
114
115        let result = a.wrapping_sub(b).wrapping_sub(borrow);
116        env.write_vregs()
117            .write(vd, i, result)
118            .expect("`i < vl` of `vd`; qed");
119    }
120
121    env.mark_vs_dirty();
122}
123
124/// Execute an element-wise add-with-carry over `0..vl`, writing the carry-out as a single mask
125/// bit per element into `vd`.
126///
127/// When `WITH_CARRY` is true, carry-in for element `i` is read from `v0[i]`. When false, carry-in
128/// is treated as zero.
129///
130/// All elements are processed unconditionally (no execution mask).
131///
132/// Tail mask bits (indices `>= vl`) are left undisturbed per spec §5.3.
133///
134/// All register groups must have the same `vl`, which holds for groups created from the same
135/// configuration, nothing is written otherwise.
136#[inline(always)]
137#[doc(hidden)]
138#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
139pub fn execute_carry_add_mask<const WITH_CARRY: bool, Reg, Env>(
140    env: &mut Env,
141    vd: VReg,
142    vs2: VRegGroup<Env::Hart>,
143    src: OpSrc<VRegGroup<Env::Hart>>,
144    sew: Vsew,
145) where
146    Reg: Register,
147    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
148{
149    let vl = vs2.vl();
150    let Some(src) = src.with_same_vl(vl) else {
151        cold_path();
152        return;
153    };
154    let mask = sew_mask(sew);
155
156    for i in vl.indices() {
157        let a = env
158            .read_vregs()
159            .read(vs2, i)
160            .expect("`i < vl` of `vs2`; qed");
161        let b = match src.vreg {
162            Some(vs1) => env
163                .read_vregs()
164                .read(vs1, i)
165                .expect("`vs1` has the same `vl` as `vs2`, checked above; qed"),
166            None => src.scalar,
167        };
168        let c = if WITH_CARRY {
169            carry_bit(env.read_vregs(), i)
170        } else {
171            0
172        };
173
174        // Use u128 to capture the carry-out bit beyond SEW
175        let sum = u128::from(a & mask) + u128::from(b & mask) + u128::from(c);
176        let carry_out = (sum >> sew.bits_width()) & 1 != 0;
177
178        write_mask_bit(env.write_vregs(), vd, i, carry_out)
179            .expect("`i < vl <= VLEN`, so the mask bit is within `vd`; qed");
180    }
181
182    env.mark_vs_dirty();
183}
184
185/// Execute an element-wise subtract-with-borrow over `0..vl`, writing the borrow-out as a
186/// single mask bit per element into `vd`.
187///
188/// When `WITH_BORROW` is true, borrow-in for element `i` is read from `v0[i]`. When false,
189/// borrow-in is treated as zero.
190///
191/// Borrow-out is 1 when the subtraction underflows unsigned:
192/// `borrow_out = (b + borrow_in) > a` (compared as SEW-wide unsigned values).
193///
194/// All register groups must have the same `vl`, which holds for groups created from the same
195/// configuration, nothing is written otherwise.
196#[inline(always)]
197#[doc(hidden)]
198#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
199pub fn execute_carry_sub_mask<const WITH_BORROW: bool, Reg, Env>(
200    env: &mut Env,
201    vd: VReg,
202    vs2: VRegGroup<Env::Hart>,
203    src: OpSrc<VRegGroup<Env::Hart>>,
204    sew: Vsew,
205) where
206    Reg: Register,
207    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
208{
209    let vl = vs2.vl();
210    let Some(src) = src.with_same_vl(vl) else {
211        cold_path();
212        return;
213    };
214    let mask = sew_mask(sew);
215
216    for i in vl.indices() {
217        let a = env
218            .read_vregs()
219            .read(vs2, i)
220            .expect("`i < vl` of `vs2`; qed");
221        let b = match src.vreg {
222            Some(vs1) => env
223                .read_vregs()
224                .read(vs1, i)
225                .expect("`vs1` has the same `vl` as `vs2`, checked above; qed"),
226            None => src.scalar,
227        };
228        let borrow_in = if WITH_BORROW {
229            carry_bit(env.read_vregs(), i)
230        } else {
231            0
232        };
233
234        let a_m = u128::from(a & mask);
235        let rhs = u128::from(b & mask) + u128::from(borrow_in);
236        let borrow_out = a_m < rhs;
237
238        write_mask_bit(env.write_vregs(), vd, i, borrow_out)
239            .expect("`i < vl <= VLEN`, so the mask bit is within `vd`; qed");
240    }
241
242    env.mark_vs_dirty();
243}
ab_riscv_interpreter/v/zvexx/carry/zvexx_carry_helpers.rs

ab_riscv_interpreter/v/zvexx/carry/
zvexx_carry_helpers.rs
