ab_riscv_interpreter/v/zvexx/store/
zvexx_store_helpers.rs

1//! Opaque helpers for ZveXx extension
2
3use crate::v::vector_registers::{VRegGroup, VRegSegmentGroup, VectorRegistersExt};
4use crate::v::zvexx::load::zvexx_load_helpers::{
5    access_wraps, bytes_before_wrap, effective_address, mask_bit, snapshot_mask,
6};
7use crate::{ExecutionError, VirtualMemory, VirtualMemoryError};
8use ab_riscv_primitives::prelude::*;
9use core::hint::cold_path;
10
11/// Write `eew`-sized data from `buf[..eew.bytes()]` to memory at `addr` (little-endian)
12#[inline(always)]
13#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
14fn write_mem_element<Reg>(
15    memory: &mut impl VirtualMemory,
16    addr: u64,
17    eew: Eew,
18    buf: [u8; const { usize::from(Eew::MAX_BYTES) }],
19) -> Result<(), VirtualMemoryError>
20where
21    Reg: Register,
22{
23    write_bytes::<Reg, _>(
24        memory,
25        addr,
26        buf.get(..usize::from(eew.bytes_width()))
27            .expect("Element width never exceeds `Eew::MAX_BYTES`; qed"),
28    )
29}
30
31/// Write `src` starting at effective address `address`, wrapping around at the end of the
32/// `XLEN`-bit address space
33#[inline(always)]
34#[doc(hidden)]
35#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
36pub fn write_bytes<Reg, Memory>(
37    memory: &mut Memory,
38    address: u64,
39    src: &[u8],
40) -> Result<(), VirtualMemoryError>
41where
42    Reg: Register,
43    Memory: VirtualMemory,
44{
45    let before_wrap = bytes_before_wrap::<Reg>(address, src.len());
46    let Some((head, tail)) = src.split_at_checked(before_wrap) else {
47        cold_path();
48        return Err(VirtualMemoryError::OutOfBoundsWrite { address });
49    };
50    memory.write_slice(address, head)?;
51    if !tail.is_empty() {
52        cold_path();
53        memory.write_slice(0, tail)?;
54    }
55    Ok(())
56}
57
58/// Execute a unit-stride or unit-stride segment store.
59///
60/// Segment stride between elements is `nf * eew.bytes()`. Field `f` for element `i` is at
61/// `base + i * nf * eew.bytes() + f * eew.bytes()`. When `nf == 1` this degenerates to a
62/// plain unit-stride store.
63///
64/// Elements `vstart..vl` of `vs3` are stored, where `vl` is the architectural one for regular
65/// stores.
66#[inline(always)]
67#[doc(hidden)]
68#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
69pub fn execute_unit_stride_store<Reg, Env, Memory>(
70    env: &mut Env,
71    memory: &mut Memory,
72    vs3: VRegSegmentGroup<Env::Hart>,
73    vm: bool,
74    base: u64,
75) -> Result<(), ExecutionError<Reg::Type>>
76where
77    Reg: Register,
78    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
79    Memory: VirtualMemory,
80{
81    let vl = vs3.first().vl();
82    let vstart = env.vstart();
83    let eew = vs3.first().eew();
84    let elem_bytes = eew.bytes_width();
85    let nf = vs3.nf();
86
87    // Unmasked non-segment store is a plain copy of the contiguous element range of the register
88    // group into a contiguous memory range, as long as the whole range is writable. If it is not,
89    // the element-wise path below is what determines the faulting element and writes everything
90    // before it, which is the same outcome a partially written copy would have.
91    if vm
92        && nf.fields_per_segment() == 1
93        && let Some(count) = u32::from(vl.get()).checked_sub(u32::from(u16::from(vstart)))
94        && count > 0
95    {
96        let first = u16::from(vstart);
97        let addr = effective_address::<Reg>(base, u64::from(first) * u64::from(elem_bytes));
98        // A range that wraps around the end of the address space is not contiguous in memory
99        if let Some(data) = env.read_vregs().elements_bytes(vs3.first(), first, count)
100            && !access_wraps::<Reg>(addr, data.len())
101            && memory.write_slice(addr, data).is_ok()
102        {
103            env.reset_vstart();
104            return Ok(());
105        }
106        cold_path();
107    }
108
109    let segment_stride = u64::from(nf.fields_per_segment() * elem_bytes);
110    let mask_buf = snapshot_mask(env.read_vregs(), vm);
111    for element in vs3.elements(u16::from(vstart)) {
112        let i = element.index();
113        if !vm && !mask_bit(&mask_buf, i) {
114            continue;
115        }
116        let elem_base = effective_address::<Reg>(base, u64::from(i) * segment_stride);
117        let fields = element.read_fields(env.read_vregs());
118        for (f, field) in (0..nf.fields_per_segment()).zip(fields) {
119            let addr = effective_address::<Reg>(elem_base, u64::from(f * elem_bytes));
120            let data = field.to_le_bytes();
121            // Record the current element index in `vstart` so that, on a memory fault, the failing
122            // element can be identified and the operation can be restarted
123            if let Err(error) = write_mem_element::<Reg>(memory, addr, eew, data) {
124                cold_path();
125                env.set_vstart(Vstart::from(i));
126                return Err(ExecutionError::from(error));
127            }
128        }
129    }
130    env.reset_vstart();
131    Ok(())
132}
133
134/// Execute a strided or strided-segment store.
135///
136///   `base + i * stride + f * eew.bytes()`, wrapping around modulo `2^XLEN`
137///   `effective_address::<Reg>(base, i.wrapping_mul(stride) as u64).wrapping_add(f * eew.bytes())`
138///
139/// `stride` is the raw XLEN register value reinterpreted as a signed integer, matching the RVV
140/// specification where the stride operand is a two's-complement signed offset.
141#[inline(always)]
142#[doc(hidden)]
143#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
144pub fn execute_strided_store<Reg, Env, Memory>(
145    env: &mut Env,
146    memory: &mut Memory,
147    vs3: VRegSegmentGroup<Env::Hart>,
148    vm: bool,
149    base: u64,
150    stride: i64,
151) -> Result<(), ExecutionError<Reg::Type>>
152where
153    Reg: Register,
154    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
155    Memory: VirtualMemory,
156{
157    let vstart = env.vstart();
158    let eew = vs3.first().eew();
159    let elem_bytes = eew.bytes_width();
160    let mask_buf = snapshot_mask(env.read_vregs(), vm);
161    for element in vs3.elements(u16::from(vstart)) {
162        let i = element.index();
163        if !vm && !mask_bit(&mask_buf, i) {
164            continue;
165        }
166        let elem_base =
167            effective_address::<Reg>(base, i64::from(i).wrapping_mul(stride).cast_unsigned());
168        let fields = element.read_fields(env.read_vregs());
169        for (f, field) in (0..vs3.nf().fields_per_segment()).zip(fields) {
170            let addr = effective_address::<Reg>(elem_base, u64::from(f * elem_bytes));
171            let data = field.to_le_bytes();
172            // Record the current element index in `vstart` so that, on a memory fault, the failing
173            // element can be identified and the operation can be restarted
174            if let Err(error) = write_mem_element::<Reg>(memory, addr, eew, data) {
175                cold_path();
176                env.set_vstart(Vstart::from(i));
177                return Err(ExecutionError::from(error));
178            }
179        }
180    }
181    env.reset_vstart();
182    Ok(())
183}
184
185/// Execute an indexed (unordered or ordered) store or indexed-segment store.
186///
187/// The effective address of element `i`, field `f` is:
188///   `base + index[i] + f * eew.bytes()`
189/// where `index[i]` is element `i` of the index register group `vs2`, interpreted as an
190/// unsigned integer of its element width.
191///
192/// `vs2` must have the same `vl` as `vs3`, which holds for groups created from the same
193/// configuration, nothing is stored otherwise.
194#[inline(always)]
195#[doc(hidden)]
196#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
197pub fn execute_indexed_store<Reg, Env, Memory>(
198    env: &mut Env,
199    memory: &mut Memory,
200    vs3: VRegSegmentGroup<Env::Hart>,
201    vs2: VRegGroup<Env::Hart>,
202    vm: bool,
203    base: u64,
204) -> Result<(), ExecutionError<Reg::Type>>
205where
206    Reg: Register,
207    Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
208    Memory: VirtualMemory,
209{
210    let Some(indexed) = vs3.with_index(vs2) else {
211        cold_path();
212        return Ok(());
213    };
214    let vstart = env.vstart();
215    let data_eew = vs3.first().eew();
216    let data_elem_bytes = data_eew.bytes_width();
217    let mask_buf = snapshot_mask(env.read_vregs(), vm);
218    for indexed in indexed.elements(u16::from(vstart)) {
219        let element = indexed.element();
220        let i = element.index();
221        if !vm && !mask_bit(&mask_buf, i) {
222            continue;
223        }
224        let offset = indexed.read_index(env.read_vregs());
225        let elem_base = effective_address::<Reg>(base, offset);
226        let fields = element.read_fields(env.read_vregs());
227        for (f, field) in (0..vs3.nf().fields_per_segment()).zip(fields) {
228            let addr =
229                effective_address::<Reg>(elem_base, u64::from(f) * u64::from(data_elem_bytes));
230            let data = field.to_le_bytes();
231            // Record the current element index in `vstart` so that, on a memory fault, the failing
232            // element can be identified and the operation can be restarted
233            if let Err(error) = write_mem_element::<Reg>(memory, addr, data_eew, data) {
234                cold_path();
235                env.set_vstart(Vstart::from(i));
236                return Err(ExecutionError::from(error));
237            }
238        }
239    }
240    env.reset_vstart();
241    Ok(())
242}
ab_riscv_interpreter/v/zvexx/store/zvexx_store_helpers.rs

ab_riscv_interpreter/v/zvexx/store/
zvexx_store_helpers.rs
