ab_riscv_interpreter/v/zvexx/perm/
zvexx_perm_helpers.rs

1//! Opaque helpers for ZveXx extension
2
3use crate::v::vector_registers::{VLENB_USIZE, VectorRegisterFile, VectorRegistersExt};
4pub use crate::v::zvexx::arith::zvexx_arith_helpers::check_vreg_group_alignment;
5use crate::v::zvexx::load::zvexx_load_helpers::{mask_bit, snapshot_mask};
6use crate::v::zvexx::zvexx_helpers::INSTRUCTION_SIZE;
7use crate::{ExecutionError, PackedAddress, ProgramCounter};
8use ab_riscv_primitives::prelude::*;
9use core::hint::cold_path;
10use core::ptr;
11
12/// Check that register groups `[a, a+count)` and `[b, b+count)` do not overlap.
13///
14/// Both groups must have the same size `count`. For groups of different sizes use
15/// [`check_no_overlap_asymmetric`].
16#[inline(always)]
17#[doc(hidden)]
18#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
19pub fn check_no_overlap<Reg, Memory, PC>(
20    program_counter: &PC,
21    a: VReg,
22    b: VReg,
23    count: VRegGroupSize,
24) -> Result<(), ExecutionError<Reg::Type>>
25where
26    Reg: Register,
27    PC: ProgramCounter<Reg::Type, Memory>,
28{
29    let a_start = u16::from(a.to_bits());
30    let b_start = u16::from(b.to_bits());
31    let count = u16::from(count.get());
32    // Intervals [a_start, a_start+count) and [b_start, b_start+count) overlap iff
33    // each starts before the other ends. Arithmetic is widened to u16 to avoid u8 overflow
34    // (e.g., b_start=30 + count=8 = 38, which overflows u8).
35    if a_start < b_start + count && b_start < a_start + count {
36        cold_path();
37        return Err(ExecutionError::IllegalInstruction {
38            address: PackedAddress::new(program_counter.old_pc(INSTRUCTION_SIZE)),
39        });
40    }
41    Ok(())
42}
43
44/// Check that register group `[a, a+a_count)` does not overlap `[b, b+b_count)`.
45///
46/// Unlike [`check_no_overlap`], the two groups are allowed to have different sizes.
47/// Used for `vrgatherei16.vv` where vd/vs2 use LMUL-derived `group_regs` and vs1
48/// uses EEW=16-derived `index_group_regs`.
49#[inline(always)]
50#[doc(hidden)]
51#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
52pub fn check_no_overlap_asymmetric<Reg, Memory, PC>(
53    program_counter: &PC,
54    a: VReg,
55    a_count: VRegGroupSize,
56    b: VReg,
57    b_count: VRegGroupSize,
58) -> Result<(), ExecutionError<Reg::Type>>
59where
60    Reg: Register,
61    PC: ProgramCounter<Reg::Type, Memory>,
62{
63    let a_start = u16::from(a.to_bits());
64    let b_start = u16::from(b.to_bits());
65    let a_count = u16::from(a_count.get());
66    let b_count = u16::from(b_count.get());
67    // Intervals [a_start, a_start+a_count) and [b_start, b_start+b_count) overlap iff
68    // each starts before the other ends.
69    if a_start < b_start + b_count && b_start < a_start + a_count {
70        cold_path();
71        return Err(ExecutionError::IllegalInstruction {
72            address: PackedAddress::new(program_counter.old_pc(INSTRUCTION_SIZE)),
73        });
74    }
75    Ok(())
76}
77
78/// Sign-extend the low `sew.bits_width()` of `val` to the register type width.
79///
80/// The arithmetic is performed entirely in 64-bit signed integer space: we shift the SEW-wide
81/// value left to place its sign bit at bit 63, then arithmetic-right-shift back to propagate it.
82/// The resulting `u64` is then narrowed to `Reg::Type` (32 or 64 bits) by combining via
83/// `From<u32>` - the only integer conversion in the `Register::Type` trait bounds.
84///
85/// For RV32 (`Reg::XLEN == 32`) the low 32 bits are already the correct sign-extended result
86/// because the arithmetic shift propagates the sign across all 64 bits and then we discard the
87/// upper half.
88///
89/// For RV64 (`Reg::XLEN == 64`) we must preserve all 64 bits. Since `Reg::Type: From<u32>` and
90/// `Reg::Type: Shl<u8>`, we reconstruct the 64-bit value by OR-ing two 32-bit halves shifted
91/// into position.
92#[inline(always)]
93#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
94pub fn sign_extend_to_reg<Reg>(val: u64, sew: Vsew) -> Reg::Type
95where
96    Reg: Register,
97{
98    let sew_bits = u32::from(sew.bits_width());
99    // `shift` is in [0, 64). When sew_bits == 64, shift == 0 and the value is unchanged.
100    let shift = u64::BITS - sew_bits;
101    // Cast to i64 so the right-shift is arithmetic (sign-extending).
102    let sign_extended = (val.cast_signed() << shift) >> shift;
103    let raw = sign_extended.cast_unsigned();
104    if Reg::XLEN == u64::BITS as u8 {
105        // RV64: preserve all 64 bits by splitting into two u32 halves.
106        let lo = Reg::Type::from(raw as u32);
107        let hi = Reg::Type::from((raw >> u32::BITS) as u32);
108        lo | (hi << 32u8)
109    } else {
110        // RV32: the low 32 bits are the correctly truncated result.
111        Reg::Type::from(raw as u32)
112    }
113}
114
115/// Execute a vslideup operation.
116///
117/// Elements `vstart..min(offset, vl)` in vd are unchanged.
118/// Elements `max(vstart, offset)..vl` where mask is active get vs2[i - offset].
119///
120/// # Safety
121/// - `vd` and `vs2` are validly aligned and non-overlapping (verified by caller).
122/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`.
123/// - When `vm=false`: `vd.to_bits() != 0`.
124#[inline(always)]
125#[doc(hidden)]
126#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
127pub unsafe fn execute_slideup<Reg, Env>(
128    env: &mut Env,
129    vd: VReg,
130    vs2: VReg,
131    vm: bool,
132    sew: Vsew,
133    offset: u64,
134) where
135    Reg: Register,
136    Env: VectorRegistersExt<Reg>,
137    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
138{
139    let vl = env.vl();
140    let vstart = env.vstart();
141    // Per spec §16.3.1: elements 0..offset are never written (vd keeps its value).
142    // The active range starts at max(vstart, offset).
143    let first = vstart.max(Vstart::from(offset.saturating_truncate::<u16>()));
144    let range = first.range_to(vl);
145
146    // Unmasked slide is a copy of a contiguous element range between two contiguous register
147    // groups
148    if vm && !range.is_empty() {
149        let len = range.len() * usize::from(sew.bytes_width());
150        // `first >= offset`, hence the truncation is lossless
151        let src_first = *range.start() - offset.saturating_truncate::<u16>();
152        let src = VectorRegisterFile::<{ Env::VLEN }>::element_offset(vs2, src_first, sew);
153        let dst = VectorRegisterFile::<{ Env::VLEN }>::element_offset(vd, *range.start(), sew);
154        let bytes = env.write_vregs().as_bytes_mut().as_flattened_mut();
155        // SAFETY: Elements `first..vl` of `vd` and `first - offset..vl - offset` of `vs2` lie
156        // within their register groups, which end within the register file (precondition), so
157        // both ranges are within `bytes`. Overlapping ranges are fine for `ptr::copy()`. Both
158        // pointers derive from the same mutable one, a separate shared one would be invalidated
159        // by it.
160        unsafe {
161            let bytes = bytes.as_mut_ptr();
162            ptr::copy(bytes.byte_add(src), bytes.byte_add(dst), len);
163        }
164        env.mark_vs_dirty();
165        env.reset_vstart();
166        return;
167    }
168
169    // SAFETY: `vl <= VLEN`
170    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
171    for i in range {
172        if !mask_bit(&mask_buf, i) {
173            continue;
174        }
175        let src_idx = i - offset.saturating_truncate::<u16>();
176        // SAFETY: src_idx < vl <= group_regs * elems_per_reg, so source element is in range
177        let val = unsafe { env.read_vregs().read_element(vs2, src_idx, sew) };
178        // SAFETY: i < vl <= group_regs * elems_per_reg, so dest element is in range
179        unsafe {
180            env.write_vregs().write_element(vd, i, sew, val);
181        }
182    }
183    env.mark_vs_dirty();
184    env.reset_vstart();
185}
186
187/// Execute a vslidedown operation.
188///
189/// Element `vd[i] = vs2[i + offset]` if `i + offset < vlmax`, else `0`.
190///
191/// # Safety
192/// - `vd` and `vs2` are validly aligned (verified by caller); overlap is permitted.
193/// - `vl <= vlmax`.
194/// - When `vm=false`: `vd.to_bits() != 0`.
195#[inline(always)]
196#[doc(hidden)]
197#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
198pub unsafe fn execute_slidedown<Reg, Env>(
199    env: &mut Env,
200    vd: VReg,
201    vs2: VReg,
202    vm: bool,
203    sew: Vsew,
204    vlmax: Vl,
205    offset: u64,
206) where
207    Reg: Register,
208    Env: VectorRegistersExt<Reg>,
209    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
210{
211    let vl = env.vl();
212    let vstart = env.vstart();
213
214    let range = vstart.range_to(vl);
215
216    // Unmasked slide is a copy of a contiguous element range between two contiguous register
217    // groups (or within one), followed by zeroing of the elements whose source is beyond `vlmax`
218    if vm && !range.is_empty() {
219        let elem_bytes = usize::from(sew.bytes_width());
220        let first = *range.start();
221        // Elements `i` with `i + offset < vlmax` have a source, the rest are zeroed
222        let with_source = u64::from(vlmax)
223            .saturating_sub(offset)
224            .saturating_sub(u64::from(first));
225        let copied = usize::try_from(with_source)
226            .map_or(range.len(), |with_source| with_source.min(range.len()));
227        let copied_len = copied * elem_bytes;
228        let len = range.len() * elem_bytes;
229        let dst = VectorRegisterFile::<{ Env::VLEN }>::element_offset(vd, first, sew);
230        let bytes = env.write_vregs().as_bytes_mut().as_flattened_mut();
231        // SAFETY: Elements `first..vl` of `vd` lie within its register group, which ends within
232        // the register file (precondition), so `dst..dst + len` is within `bytes`. So do elements
233        // `first + offset..vlmax` of `vs2` when there are any to copy: `copied > 0` implies
234        // `first + offset < vlmax`, hence the truncation of `offset` is lossless and
235        // `src..src + copied_len` is within the group. Overlapping ranges are fine for
236        // `ptr::copy()`. Both pointers derive from the same mutable one, a separate shared one
237        // would be invalidated by it.
238        unsafe {
239            if copied > 0 {
240                let src_first = first + offset.saturating_truncate::<u16>();
241                let src = VectorRegisterFile::<{ Env::VLEN }>::element_offset(vs2, src_first, sew);
242                let bytes = bytes.as_mut_ptr();
243                ptr::copy(bytes.byte_add(src), bytes.byte_add(dst), copied_len);
244            }
245            bytes.get_unchecked_mut(dst + copied_len..dst + len).fill(0);
246        }
247        env.mark_vs_dirty();
248        env.reset_vstart();
249        return;
250    }
251
252    // SAFETY: `vl <= VLEN`
253    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
254    for i in range {
255        if !mask_bit(&mask_buf, i) {
256            continue;
257        }
258        // Use checked_add to guard against offset being so large that i + offset overflows u64.
259        // Any value that wraps past u64::MAX is trivially >= vlmax, so the spec requires vd[i]=0.
260        let val = if let Some(src_idx) = u64::from(i).checked_add(offset)
261            && src_idx < u64::from(vlmax)
262        {
263            // SAFETY: src_idx < vlmax <= group_regs * elems_per_reg, so element is in range
264            unsafe { env.read_vregs().read_element(vs2, src_idx as u16, sew) }
265        } else {
266            0
267        };
268        // SAFETY: i < vl <= vlmax <= group_regs * elems_per_reg
269        unsafe {
270            env.write_vregs().write_element(vd, i, sew, val);
271        }
272    }
273    env.mark_vs_dirty();
274    env.reset_vstart();
275}
276
277/// Execute a vslide1up operation.
278///
279/// Element 0 of vd gets `scalar` (when active and vl > 0).
280/// Element `i` for `1 <= i < vl` gets `vs2[i - 1]`.
281/// vd must not overlap vs2.
282///
283/// # Safety
284/// - `vd` and `vs2` are validly aligned and non-overlapping (verified by caller).
285/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`.
286/// - When `vm=false`: `vd.to_bits() != 0`.
287#[inline(always)]
288#[doc(hidden)]
289#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
290pub unsafe fn execute_slide1up<Reg, Env>(
291    env: &mut Env,
292    vd: VReg,
293    vs2: VReg,
294    vm: bool,
295    sew: Vsew,
296    scalar: u64,
297) where
298    Reg: Register,
299    Env: VectorRegistersExt<Reg>,
300    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
301{
302    let vl = env.vl();
303    let vstart = env.vstart();
304    // SAFETY: `vl <= VLEN`
305    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
306    for i in vstart.range_to(vl) {
307        if !mask_bit(&mask_buf, i) {
308            continue;
309        }
310        let val = if i == 0 {
311            scalar
312        } else {
313            // SAFETY: i - 1 < vl <= group_regs * elems_per_reg
314            unsafe { env.read_vregs().read_element(vs2, i - 1, sew) }
315        };
316        // SAFETY: i < vl <= group_regs * elems_per_reg
317        unsafe {
318            env.write_vregs().write_element(vd, i, sew, val);
319        }
320    }
321    env.mark_vs_dirty();
322    env.reset_vstart();
323}
324
325/// Execute a vslide1down operation.
326///
327/// Element `vd[i] = vs2[i + 1]` for `i < vl - 1`; element `vd[vl - 1]` gets `scalar`.
328///
329/// Overlap between `vd` and `vs2` is permitted by the spec. When they share the same register
330/// group base (exact overlap), ascending iteration is still correct: each write goes to byte range
331/// `[i*sew, (i+1)*sew)` while the subsequent read comes from `[(i+1)*sew, (i+2)*sew)`. These
332/// ranges are adjacent and non-overlapping, so writing element `i` never corrupts the source bytes
333/// of element `i+1`.
334///
335/// # Safety
336/// - `vd` and `vs2` are validly aligned (verified by caller); overlap is permitted.
337/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`.
338/// - When `vm=false`: `vd.to_bits() != 0`.
339#[inline(always)]
340#[doc(hidden)]
341#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
342pub unsafe fn execute_slide1down<Reg, Env>(
343    env: &mut Env,
344    vd: VReg,
345    vs2: VReg,
346    vm: bool,
347    sew: Vsew,
348    scalar: u64,
349) where
350    Reg: Register,
351    Env: VectorRegistersExt<Reg>,
352    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
353{
354    let vl = env.vl();
355    let vstart = env.vstart();
356    // SAFETY: `vl <= VLEN`
357    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
358    let range = vstart.range_to(vl);
359    for i in range.clone() {
360        if !mask_bit(&mask_buf, i) {
361            continue;
362        }
363        let val = if i < *range.end() {
364            // SAFETY: i + 1 < vl <= group_regs * elems_per_reg
365            unsafe { env.read_vregs().read_element(vs2, i + 1, sew) }
366        } else {
367            scalar
368        };
369        // SAFETY: i < vl <= group_regs * elems_per_reg
370        unsafe {
371            env.write_vregs().write_element(vd, i, sew, val);
372        }
373    }
374    env.mark_vs_dirty();
375    env.reset_vstart();
376}
377
378/// Execute vrgather.vv: `vd[i] = (vs1[i] < vlmax) ? vs2[vs1[i]] : 0`.
379///
380/// # Safety
381/// - `vd`, `vs2`, and `vs1` are validly aligned and mutually non-overlapping (verified by caller).
382/// - `vl <= vlmax`.
383/// - When `vm=false`: `vd.to_bits() != 0`.
384#[inline(always)]
385#[doc(hidden)]
386#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
387pub unsafe fn execute_rgather_vv<Reg, Env>(
388    env: &mut Env,
389    vd: VReg,
390    vs2: VReg,
391    vs1: VReg,
392    vm: bool,
393    sew: Vsew,
394    vlmax: Vl,
395) where
396    Reg: Register,
397    Env: VectorRegistersExt<Reg>,
398    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
399{
400    let vl = env.vl();
401    let vstart = env.vstart();
402    // SAFETY: `vl <= VLEN`
403    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
404    for i in vstart.range_to(vl) {
405        if !mask_bit(&mask_buf, i) {
406            continue;
407        }
408        // SAFETY: i < vl <= group_regs * elems_per_reg for vs1
409        let index = unsafe { env.read_vregs().read_element(vs1, i, sew) };
410        let val = if index < u64::from(vlmax) {
411            // SAFETY: index < vlmax <= group_regs * elems_per_reg for vs2
412            unsafe { env.read_vregs().read_element(vs2, index as u16, sew) }
413        } else {
414            0u64
415        };
416        // SAFETY: i < vl <= group_regs * elems_per_reg for vd
417        unsafe {
418            env.write_vregs().write_element(vd, i, sew, val);
419        }
420    }
421    env.mark_vs_dirty();
422    env.reset_vstart();
423}
424
425/// Execute vrgather.vx / vrgather.vi: all active elements get `vs2[index]` or `0`.
426///
427/// # Safety
428/// - `vd` and `vs2` are validly aligned and non-overlapping (verified by caller).
429/// - `vl <= vlmax`.
430/// - When `vm=false`: `vd.to_bits() != 0`.
431#[inline(always)]
432#[doc(hidden)]
433#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
434pub unsafe fn execute_rgather_scalar<Reg, Env>(
435    env: &mut Env,
436    vd: VReg,
437    vs2: VReg,
438    vm: bool,
439    sew: Vsew,
440    vlmax: Vl,
441    index: u64,
442) where
443    Reg: Register,
444    Env: VectorRegistersExt<Reg>,
445    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
446{
447    let vl = env.vl();
448    let vstart = env.vstart();
449    // SAFETY: `vl <= VLEN`
450    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
451    // Pre-compute the gathered value; it's the same for all elements.
452    let val = if index < u64::from(vlmax) {
453        // SAFETY: index < vlmax <= group_regs * elems_per_reg for vs2
454        unsafe { env.read_vregs().read_element(vs2, index as u16, sew) }
455    } else {
456        0u64
457    };
458    for i in vstart.range_to(vl) {
459        if !mask_bit(&mask_buf, i) {
460            continue;
461        }
462        // SAFETY: i < vl <= group_regs * elems_per_reg for vd
463        unsafe {
464            env.write_vregs().write_element(vd, i, sew, val);
465        }
466    }
467    env.mark_vs_dirty();
468    env.reset_vstart();
469}
470
471/// Execute vrgatherei16.vv: `vd[i] = (vs1_16[i] < vlmax) ? vs2[vs1_16[i]] : 0`.
472///
473/// `vs1` always uses EEW=16 regardless of SEW. `vl` must not exceed the index register group
474/// capacity, i.e. `vl <= index_group_regs * VLEN.bytes() / 2` (VLEN.bytes() / 2 = elems per
475/// register at EEW=16).
476///
477/// # Safety
478/// - `vd`, `vs2`, and `vs1` are validly aligned and mutually non-overlapping (verified by caller).
479/// - `vl <= vlmax` (for the data register group) AND `vl <= index_group_regs * VLEN.bytes() / 2`
480///   (for the index register group).
481/// - When `vm=false`: `vd.to_bits() != 0`.
482#[inline(always)]
483#[expect(clippy::too_many_arguments, reason = "Internal API")]
484#[doc(hidden)]
485#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
486pub unsafe fn execute_rgatherei16<Reg, Env>(
487    env: &mut Env,
488    vd: VReg,
489    vs2: VReg,
490    vs1: VReg,
491    vm: bool,
492    sew: Vsew,
493    vlmax: Vl,
494    index_group_regs: VRegGroupSize,
495) where
496    Reg: Register,
497    Env: VectorRegistersExt<Reg>,
498    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
499{
500    let index_group_regs = index_group_regs.get();
501    let vl = env.vl();
502    let vstart = env.vstart();
503    // Maximum number of EEW=16 elements the index register group can hold.
504    // Each register holds VLEN.bytes() / 2 elements at EEW=16.
505    let index_capacity = u32::from(index_group_regs) * (Env::VLEN.bytes() / 2);
506    // `vl` must not exceed either the data VLMAX or the index register group capacity.
507    // Both bounds are guaranteed by the caller; this debug assertion catches misuse early.
508    debug_assert!(
509        vl <= vlmax && u32::from(vl) <= index_capacity,
510        "vl={vl} exceeds vlmax={vlmax} or index_capacity={index_capacity}"
511    );
512    // SAFETY: `vl <= VLEN`
513    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
514    for i in vstart.range_to(vl) {
515        if !mask_bit(&mask_buf, i) {
516            continue;
517        }
518        // Read 16-bit index from vs1; EEW=16 always.
519        // SAFETY: i < vl <= index_capacity = index_group_regs * (VLEN.bytes() / 2), so element i
520        // fits within the index register group.
521        let index = unsafe { env.read_vregs().read_element(vs1, i, Vsew::E16) };
522        let val = if index < u64::from(vlmax) {
523            // SAFETY: index < vlmax <= group_regs * elems_per_reg for vs2
524            unsafe { env.read_vregs().read_element(vs2, index as u16, sew) }
525        } else {
526            0u64
527        };
528        // SAFETY: i < vl <= group_regs * elems_per_reg for vd
529        unsafe {
530            env.write_vregs().write_element(vd, i, sew, val);
531        }
532    }
533    env.mark_vs_dirty();
534    env.reset_vstart();
535}
536
537/// Execute vmerge.vvm / vmv.v.v.
538///
539/// When `vm=true` (vmv.v.v): all active elements `vstart..vl` get `vs1[i]`; vs2 unused.
540/// When `vm=false` (vmerge.vvm): active elements where `v0[i]=1` get `vs1[i]`,
541/// inactive elements get `vs2[i]`.
542///
543/// # Safety
544/// - `vd` and `vs1` are validly aligned (verified by caller).
545/// - When `vm=false`: `vs2` is validly aligned and `vd` does not overlap v0 (verified by caller).
546/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`.
547#[inline(always)]
548#[doc(hidden)]
549#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
550pub unsafe fn execute_merge_vv<Reg, Env>(
551    env: &mut Env,
552    vd: VReg,
553    vs2: VReg,
554    vs1: VReg,
555    vm: bool,
556    sew: Vsew,
557) where
558    Reg: Register,
559    Env: VectorRegistersExt<Reg>,
560    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
561{
562    let vl = env.vl();
563    let vstart = env.vstart();
564    // SAFETY: `vl <= VLEN`
565    // For vmv.v.v (vm=true) the mask is all-ones so snapshot_mask is still valid.
566    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
567    for i in vstart.range_to(vl) {
568        let mask_set = mask_bit(&mask_buf, i);
569        let val = if mask_set {
570            // SAFETY: i < vl <= group_regs * elems_per_reg for vs1
571            unsafe { env.read_vregs().read_element(vs1, i, sew) }
572        } else {
573            // mask_set=false only reachable when vm=false (vmerge path).
574            // SAFETY: i < vl <= group_regs * elems_per_reg for vs2
575            unsafe { env.read_vregs().read_element(vs2, i, sew) }
576        };
577        // SAFETY: i < vl <= group_regs * elems_per_reg for vd
578        unsafe {
579            env.write_vregs().write_element(vd, i, sew, val);
580        }
581    }
582    env.mark_vs_dirty();
583    env.reset_vstart();
584}
585
586/// Execute vmerge.vxm / vmerge.vim / vmv.v.x / vmv.v.i.
587///
588/// When `vm=true`: all active elements `vstart..vl` get `scalar`; vs2 unused.
589/// When `vm=false`: active elements where `v0[i]=1` get `scalar`,
590/// inactive elements get `vs2[i]`.
591///
592/// # Safety
593/// - `vd` is validly aligned (verified by caller).
594/// - When `vm=false`: `vs2` is validly aligned and `vd` does not overlap v0 (verified by caller).
595/// - `vl <= group_regs * VLEN.bytes() / sew_bytes`.
596#[inline(always)]
597#[doc(hidden)]
598#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
599pub unsafe fn execute_merge_scalar<Reg, Env>(
600    env: &mut Env,
601    vd: VReg,
602    vs2: VReg,
603    vm: bool,
604    sew: Vsew,
605    scalar: u64,
606) where
607    Reg: Register,
608    Env: VectorRegistersExt<Reg>,
609    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
610{
611    let vl = env.vl();
612    let vstart = env.vstart();
613    // SAFETY: `vl <= VLEN`
614    let mask_buf = unsafe { snapshot_mask(env.read_vregs(), vm, vl) };
615
616    for i in vstart.range_to(vl) {
617        let val = if mask_bit(&mask_buf, i) {
618            scalar
619        } else {
620            // SAFETY: i < vl <= group_regs * elems_per_reg for vs2
621            unsafe { env.read_vregs().read_element(vs2, i, sew) }
622        };
623        // SAFETY: i < vl <= group_regs * elems_per_reg for vd
624        unsafe {
625            env.write_vregs().write_element(vd, i, sew, val);
626        }
627    }
628    env.mark_vs_dirty();
629    env.reset_vstart();
630}
631
632/// Execute vcompress.vm: pack active elements of vs2 (under vs1 mask) sequentially into vd.
633///
634/// `vs1` is treated as an explicit mask register (single register, not LMUL-grouped).
635/// The output write index increments only for elements where `vs1[i]` is set.
636/// vd must not overlap vs1 or vs2.
637///
638/// # Safety
639/// - `vd`, `vs2` are validly aligned and non-overlapping (verified by caller).
640/// - `vs1` does not overlap `vd` (verified by caller).
641/// - `vl <= VLMAX`.
642#[inline(always)]
643#[doc(hidden)]
644#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
645pub unsafe fn execute_compress<Reg, Env>(
646    env: &mut Env,
647    vd: VReg,
648    vs2: VReg,
649    vs1: VReg,
650    vl: Vl,
651    sew: Vsew,
652) where
653    Reg: Register,
654    Env: VectorRegistersExt<Reg>,
655    [(); SUPPORTED_ELEN_VLEN::<{ Env::ELEN }, { Env::VLEN }>]:,
656{
657    let mask_bytes = usize::from(vl.bytes());
658    let vreg = env.read_vregs();
659    let mut vs1_buf = [0u8; VLENB_USIZE::<{ Env::VLEN }>];
660    // SAFETY: mask_bytes <= VLEN.bytes() since vl <= VLEN; vs1_base < 32
661    unsafe {
662        vs1_buf
663            .get_unchecked_mut(..mask_bytes)
664            .copy_from_slice(vreg.get(vs1).get_unchecked(..mask_bytes));
665    }
666    let mut out_idx = 0;
667    for i in Vstart::ZERO.range_to(vl) {
668        if !mask_bit(&vs1_buf, i) {
669            continue;
670        }
671        // SAFETY: i < vl <= group_regs * elems_per_reg
672        let val = unsafe { env.read_vregs().read_element(vs2, i, sew) };
673        // SAFETY: out_idx <= popcount(vs1[0..vl)) <= vl
674        unsafe {
675            env.write_vregs().write_element(vd, out_idx, sew, val);
676        }
677        out_idx += 1;
678    }
679    env.mark_vs_dirty();
680    env.reset_vstart();
681}
682
683/// Copy `COUNT` whole vector registers from `src_base` to `dst_base`.
684///
685/// No masking, no vtype dependency. Uses snapshot semantics: all source registers are read into
686/// a stack buffer before any destination registers are written, giving correct memmove-style
687/// behaviour for all overlap patterns (including partial overlap such as src=V0, dst=V1, count=2).
688///
689/// # Safety
690/// - `dst_base + COUNT <= 32` and `src_base + COUNT <= 32` (verified by caller via alignment
691///   checks).
692/// - `dst_base % COUNT == 0` and `src_base % COUNT == 0` (verified by caller).
693#[inline(always)]
694#[doc(hidden)]
695#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
696pub unsafe fn execute_whole_reg_move<const COUNT: usize, const VLEN: Vlen>(
697    vregs: &mut VectorRegisterFile<VLEN>,
698    dst_base: VReg,
699    src_base: VReg,
700) {
701    // Snapshot all source registers before writing any destination registers.
702    // This is correct for all overlap patterns without direction-dependent logic.
703    let mut tmp = [[0u8; _]; COUNT];
704    for (k, item) in tmp.iter_mut().enumerate() {
705        // SAFETY: Guaranteed by function contract
706        let src = unsafe { VReg::from_bits(src_base.to_bits() + k as u8).unwrap_unchecked() };
707        *item = *vregs.get(src);
708    }
709    for (k, item) in tmp.iter().enumerate() {
710        // SAFETY: Guaranteed by function contract
711        let dst = unsafe { VReg::from_bits(dst_base.to_bits() + k as u8).unwrap_unchecked() };
712        *vregs.get_mut(dst) = *item;
713    }
714}
ab_riscv_interpreter/v/zvexx/perm/zvexx_perm_helpers.rs

ab_riscv_interpreter/v/zvexx/perm/
zvexx_perm_helpers.rs
