ab_riscv_interpreter/v/zvexx/

load.rs

//! ZveXx vector load instructions

#[cfg(test)]
mod tests;
pub mod zvexx_load_helpers;

use crate::v::vector_registers::VectorRegistersExt;
use crate::v::zvexx::zvexx_helpers;
use crate::{
    ExecutableInstruction, ExecutableInstructionCsr, ExecutableInstructionOperands, ExecutionError,
    ProgramCounter, RegisterFile, Rs1Rs2OperandValues, Rs1Rs2Operands, VirtualMemory,
};
use ab_riscv_macros::instruction_execution;
use ab_riscv_primitives::prelude::*;
use core::fmt;
use core::ops::ControlFlow;

#[instruction_execution]
impl<Reg> ExecutableInstructionOperands for ZveXxLoadInstruction<Reg> where Reg: Register {}

#[instruction_execution]
impl<Reg, ExtState, CustomError> ExecutableInstructionCsr<ExtState, CustomError>
    for ZveXxLoadInstruction<Reg>
where
    Reg: Register,
{
}

#[instruction_execution]
impl<Reg, Regs, ExtState, Memory, PC, InstructionHandler, CustomError>
    ExecutableInstruction<Regs, ExtState, Memory, PC, InstructionHandler, CustomError>
    for ZveXxLoadInstruction<Reg>
where
    Reg: Register,
    Regs: RegisterFile<Reg>,
    ExtState: VectorRegistersExt<Reg, CustomError>,
    [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
    Memory: VirtualMemory,
    PC: ProgramCounter<Reg::Type, Memory, CustomError>,
    CustomError: fmt::Debug,
{
    #[inline(always)]
    fn execute(
        self,
        Rs1Rs2OperandValues {
            rs1_value,
            rs2_value,
        }: Rs1Rs2OperandValues<<Self::Reg as Register>::Type>,
        _regs: &mut Regs,
        ext_state: &mut ExtState,
        memory: &mut Memory,
        program_counter: &mut PC,
        _system_instruction_handler: &mut InstructionHandler,
    ) -> Result<
        ControlFlow<(), (Self::Reg, <Self::Reg as Register>::Type)>,
        ExecutionError<Reg::Type, CustomError>,
    > {
        match self {
            // Whole-register load: loads `nreg` consecutive registers starting at `vd` directly
            // from memory. `vd` must be aligned to `nreg`. Ignores vtype, vl, vstart, masking.
            Self::Vlr {
                vd,
                rs1: _,
                nreg,
                eew: _,
            } => {
                let nreg = nreg.num_registers();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                if vd.to_bits() % nreg != 0 {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let base = rs1_value.as_u64();
                for reg_off in 0..nreg {
                    // SAFETY: the decoder guarantees nreg in {1,2,4,8} and vd is nreg-aligned
                    // (checked above), so vd.to_bits() + nreg - 1 <= 31.
                    let reg = unsafe { VReg::from_bits(vd.to_bits() + reg_off).unwrap_unchecked() };
                    let bytes = memory
                        .read_slice(
                            base + u64::from(reg_off) * u64::from(ExtState::VLEN.bytes()),
                            ExtState::VLEN.bytes(),
                        )
                        .inspect_err(|_error| {
                            if reg_off > 0 {
                                ext_state.mark_vs_dirty();
                                ext_state.reset_vstart();
                            }
                        })?;
                    ext_state.write_vregs().get_mut(reg).copy_from_slice(bytes);
                }
                ext_state.mark_vs_dirty();
                ext_state.reset_vstart();
            }

            // Mask load: loads ceil(vl / 8) bytes from base into vd with no masking applied.
            // Does not require a valid vtype: when vill is set vl is 0, so zero bytes are read.
            Self::Vlm { vd, rs1: _ } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let vl = ext_state.vl();
                let byte_count = vl.bytes();
                if byte_count > 0 {
                    let base = rs1_value.as_u64();
                    let bytes = memory.read_slice(base, u32::from(byte_count))?;
                    // SAFETY: `bytes.len() == byte_count = vl.div_ceil(8) <= VLEN / 8 =
                    // VLEN.bytes()` because `vl <= VLMAX <= VLEN`, so
                    // `..bytes.len()` is in bounds within the
                    // `VLEN.bytes()`-byte destination register.
                    unsafe {
                        ext_state
                            .write_vregs()
                            .get_mut(vd)
                            .get_unchecked_mut(..bytes.len())
                            .copy_from_slice(bytes);
                    }
                }
                ext_state.mark_vs_dirty();
                ext_state.reset_vstart();
            }

            // Unit-stride load.
            //
            // Destination EMUL = EEW/SEW * LMUL, computed via `index_register_count`. This
            // gives `group_regs` such that `VLMAX = group_regs * VLEN.bytes() / eew.bytes()`
            // matches the architectural `vl`.
            Self::Vle {
                vd,
                rs1: _,
                vm,
                eew,
            } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    group_regs,
                )?;
                if !vm
                    && zvexx_load_helpers::groups_overlap(
                        vd,
                        group_regs,
                        VReg::V0,
                        ::core::num::NonZeroU8::new(1).expect("Not zero; qed"),
                    )
                {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                // SAFETY:
                // - alignment: `check_register_group_alignment` verified `vd % group_regs == 0` and
                //   `vd + group_regs <= 32`, satisfying both the alignment and nf=1 bounds
                //   preconditions
                // - `vl <= group_regs * VLEN.bytes() / eew.bytes()`: `group_regs` is the EMUL
                //   computed for this `eew` and `vtype`, so this VLMAX equals the architectural
                //   VLMAX that bounds `vl`
                // - mask overlap: checked above via `groups_overlap`
                unsafe {
                    zvexx_load_helpers::execute_unit_stride_load::<false, _, _, _, _>(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        eew,
                        group_regs,
                        Nf::N1,
                    )?;
                }
            }

            // Fault-only-first unit-stride load. Preconditions identical to `Vle`.
            Self::Vleff {
                vd,
                rs1: _,
                vm,
                eew,
            } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    group_regs,
                )?;
                if !vm
                    && zvexx_load_helpers::groups_overlap(
                        vd,
                        group_regs,
                        VReg::V0,
                        ::core::num::NonZeroU8::new(1).expect("Not zero; qed"),
                    )
                {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                // SAFETY: preconditions identical to `Vle`; see that arm for the full argument.
                unsafe {
                    zvexx_load_helpers::execute_unit_stride_load::<true, _, _, _, _>(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        eew,
                        group_regs,
                        Nf::N1,
                    )?;
                }
            }

            // Strided load. Destination EMUL = EEW/SEW * LMUL as for unit-stride.
            Self::Vlse {
                vd,
                rs1: _,
                rs2: _,
                vm,
                eew,
            } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    group_regs,
                )?;
                if !vm
                    && zvexx_load_helpers::groups_overlap(
                        vd,
                        group_regs,
                        VReg::V0,
                        ::core::num::NonZeroU8::new(1).expect("Not zero; qed"),
                    )
                {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                // rs2 holds a signed stride; reinterpret the register value as signed
                let stride = rs2_value.as_i64();
                // SAFETY:
                // - alignment and nf=1 bounds: `check_register_group_alignment` verified `vd %
                //   group_regs == 0` and `vd + group_regs <= 32`
                // - `vl <= group_regs * VLEN.bytes() / eew.bytes()`: `group_regs` is the EMUL for
                //   this `eew` and `vtype`, so this VLMAX equals the architectural VLMAX bounding
                //   `vl`
                // - mask overlap: checked above via `groups_overlap`
                unsafe {
                    zvexx_load_helpers::execute_strided_load(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        stride,
                        eew,
                        group_regs,
                        Nf::N1,
                    )?;
                }
            }

            // Indexed-unordered load: eew is the index EEW; data EEW comes from vtype.vsew().
            // The data destination uses the base LMUL (data EEW = SEW for indexed loads).
            Self::Vluxei {
                vd,
                rs1: _,
                vs2,
                vm,
                eew: index_eew,
            } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let data_group_regs = vtype.vlmul().register_count();
                let index_group_regs = vtype
                    .vlmul()
                    .index_register_count(index_eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    data_group_regs,
                )?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vs2,
                    index_group_regs,
                )?;
                // Non-segment indexed loads permit `vd`/`vs2` overlap under the general
                // EEW-relative overlap rule (e.g. when the data and index EEW match); only
                // disallowed overlaps are reserved.
                if !zvexx_load_helpers::indexed_load_overlap_allowed(
                    vd,
                    data_group_regs,
                    vs2,
                    index_group_regs,
                    index_eew,
                    vtype.vsew(),
                    vtype.vlmul(),
                ) {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                if !vm
                    && zvexx_load_helpers::groups_overlap(
                        vd,
                        data_group_regs,
                        VReg::V0,
                        ::core::num::NonZeroU8::new(1).expect("Not zero; qed"),
                    )
                {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                // SAFETY:
                // - data alignment/nf=1 bounds: `check_register_group_alignment` on `vd`
                // - index alignment/bounds: `check_register_group_alignment` on `vs2`
                // - `vl <= data_group_regs * VLEN.bytes() / data_eew.bytes()`: data EEW = SEW and
                //   `data_group_regs = LMUL`, so VLMAX = LMUL * VLEN / SEW, which bounds `vl`
                // - `vl <= index_group_regs * VLEN.bytes() / index_eew.bytes()`: `index_group_regs`
                //   is EMUL_index defined so this VLMAX_index equals the architectural VLMAX
                // - `vd`/`vs2` overlap (if any) satisfies the general EEW overlap rule, checked
                //   above; the in-order element loop reads index element `i` before writing data
                //   element `i`, and that rule guarantees a data write never clobbers an index
                //   element that has not yet been consumed
                // - mask overlap: checked above via `groups_overlap`
                unsafe {
                    zvexx_load_helpers::execute_indexed_load(
                        ext_state,
                        memory,
                        vd,
                        vs2,
                        vm,
                        rs1_value.as_u64(),
                        vtype.vsew().as_eew(),
                        index_eew,
                        data_group_regs,
                        Nf::N1,
                    )?;
                }
            }

            // Indexed-ordered load: functionally identical to `Vluxei` for a software
            // interpreter; memory access ordering has no observable effect here.
            Self::Vloxei {
                vd,
                rs1: _,
                vs2,
                vm,
                eew: index_eew,
            } => {
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let data_group_regs = vtype.vlmul().register_count();
                let index_group_regs = vtype
                    .vlmul()
                    .index_register_count(index_eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    data_group_regs,
                )?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vs2,
                    index_group_regs,
                )?;
                // Non-segment indexed loads permit `vd`/`vs2` overlap under the general
                // EEW-relative overlap rule; see the `Vluxei` arm for details.
                if !zvexx_load_helpers::indexed_load_overlap_allowed(
                    vd,
                    data_group_regs,
                    vs2,
                    index_group_regs,
                    index_eew,
                    vtype.vsew(),
                    vtype.vlmul(),
                ) {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                if !vm
                    && zvexx_load_helpers::groups_overlap(
                        vd,
                        data_group_regs,
                        VReg::V0,
                        ::core::num::NonZeroU8::new(1).expect("Not zero; qed"),
                    )
                {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                // SAFETY: preconditions identical to `Vluxei`; see that arm for the full
                // argument.
                unsafe {
                    zvexx_load_helpers::execute_indexed_load(
                        ext_state,
                        memory,
                        vd,
                        vs2,
                        vm,
                        rs1_value.as_u64(),
                        vtype.vsew().as_eew(),
                        index_eew,
                        data_group_regs,
                        Nf::N1,
                    )?;
                }
            }

            // Unit-stride segment load. EMUL = EEW/SEW * LMUL per field group.
            Self::Vlseg {
                vd,
                rs1: _,
                eew,
                vm_nf,
            } => {
                let vm = vm_nf.vm();
                let nf = vm_nf.nf();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::validate_segment_registers::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    vm,
                    group_regs,
                    nf,
                )?;
                // SAFETY:
                // - alignment and nf-group bounds: `validate_segment_registers` verified `vd %
                //   group_regs == 0` and `vd + nf * group_regs <= 32`
                // - `vl <= group_regs * VLEN.bytes() / eew.bytes()`: `group_regs` is the EMUL for
                //   this `eew` and `vtype`, so this VLMAX equals the architectural VLMAX bounding
                //   `vl`
                // - mask overlap with v0: `validate_segment_registers` checked `vd.to_bits() != 0`
                //   when `vm=false`, ensuring no field group contains v0
                unsafe {
                    zvexx_load_helpers::execute_unit_stride_load::<false, _, _, _, _>(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        eew,
                        group_regs,
                        nf,
                    )?;
                }
            }

            // Fault-only-first segment load. Preconditions identical to `Vlseg`.
            Self::Vlsegff {
                vd,
                rs1: _,
                eew,
                vm_nf,
            } => {
                let vm = vm_nf.vm();
                let nf = vm_nf.nf();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::validate_segment_registers::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    vm,
                    group_regs,
                    nf,
                )?;
                // SAFETY: preconditions identical to `Vlseg`; see that arm for the full argument.
                unsafe {
                    zvexx_load_helpers::execute_unit_stride_load::<true, _, _, _, _>(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        eew,
                        group_regs,
                        nf,
                    )?;
                }
            }

            // Strided segment load. EMUL = EEW/SEW * LMUL as for `Vlse`.
            Self::Vlsseg {
                vd,
                rs1: _,
                rs2: _,
                eew,
                vm_nf,
            } => {
                let vm = vm_nf.vm();
                let nf = vm_nf.nf();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let group_regs = vtype
                    .vlmul()
                    .index_register_count(eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::validate_segment_registers::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    vm,
                    group_regs,
                    nf,
                )?;
                let stride = rs2_value.as_i64();
                // SAFETY:
                // - alignment and nf-group bounds: `validate_segment_registers` verified `vd %
                //   group_regs == 0` and `vd + nf * group_regs <= 32`
                // - `vl <= group_regs * VLEN.bytes() / eew.bytes()`: `group_regs` is EMUL for this
                //   `eew` and `vtype`
                // - mask overlap: `validate_segment_registers` checked `vd.to_bits() != 0` when
                //   `vm=false`
                unsafe {
                    zvexx_load_helpers::execute_strided_load(
                        ext_state,
                        memory,
                        vd,
                        vm,
                        rs1_value.as_u64(),
                        stride,
                        eew,
                        group_regs,
                        nf,
                    )?;
                }
            }

            // Indexed-unordered segment load
            Self::Vluxseg {
                vd,
                rs1: _,
                vs2,
                eew: index_eew,
                vm_nf,
            } => {
                let vm = vm_nf.vm();
                let nf = vm_nf.nf();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let data_group_regs = vtype.vlmul().register_count();
                let index_group_regs = vtype
                    .vlmul()
                    .index_register_count(index_eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                // `validate_segment_registers` is called before the per-field overlap loop so
                // that `vd.to_bits() + f * data_group_regs < 32` is established for all `f < nf`,
                // which is required by the `VReg::from_bits` call inside the loop.
                zvexx_load_helpers::validate_segment_registers::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    vm,
                    data_group_regs,
                    nf,
                )?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vs2,
                    index_group_regs,
                )?;
                for f in 0..nf.fields_per_segment() {
                    // SAFETY: `vd.to_bits() + f * data_group_regs < 32` because
                    // `validate_segment_registers` established `vd.to_bits() + nf * data_group_regs
                    // <= 32` and `f < nf`. The value is in [0, 31], so it is a valid `VReg`
                    // encoding.
                    let field_vd = unsafe {
                        VReg::from_bits(vd.to_bits() + f * data_group_regs.get()).unwrap_unchecked()
                    };
                    if zvexx_load_helpers::groups_overlap(
                        field_vd,
                        data_group_regs,
                        vs2,
                        index_group_regs,
                    ) {
                        ::core::hint::cold_path();
                        return Err(ExecutionError::IllegalInstruction {
                            address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                        });
                    }
                }
                // SAFETY:
                // - data alignment/nf-group bounds: `validate_segment_registers` verified `vd %
                //   data_group_regs == 0` and `vd + nf * data_group_regs <= 32`
                // - index alignment/bounds: `check_register_group_alignment` verified `vs2 %
                //   EMUL_index == 0` and `vs2 + EMUL_index <= 32`
                // - no field/index group overlap: verified by the loop above
                // - `vl <= data_group_regs * VLEN.bytes() / data_eew.bytes()`: data EEW = SEW and
                //   `data_group_regs = LMUL`, so VLMAX = LMUL * VLEN / SEW bounds `vl`
                // - `vl <= EMUL_index * VLEN.bytes() / index_eew.bytes()`: `index_group_regs`
                //   (EMUL_index) is defined so this VLMAX_index equals the architectural VLMAX
                // - mask overlap: `validate_segment_registers` checked `vd.to_bits() != 0` when
                //   `vm=false`, and no field group starts at 0 since groups are contiguous from
                //   `vd` which is nonzero
                unsafe {
                    zvexx_load_helpers::execute_indexed_load(
                        ext_state,
                        memory,
                        vd,
                        vs2,
                        vm,
                        rs1_value.as_u64(),
                        vtype.vsew().as_eew(),
                        index_eew,
                        data_group_regs,
                        nf,
                    )?;
                }
            }

            // Indexed-ordered segment load: functionally identical to `Vluxseg` for a software
            // interpreter
            Self::Vloxseg {
                vd,
                rs1: _,
                vs2,
                eew: index_eew,
                vm_nf,
            } => {
                let vm = vm_nf.vm();
                let nf = vm_nf.nf();
                if !ext_state.vector_instructions_allowed() {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                }
                let Some(vtype) = ext_state.vtype() else {
                    ::core::hint::cold_path();
                    return Err(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    });
                };
                let data_group_regs = vtype.vlmul().register_count();
                let index_group_regs = vtype
                    .vlmul()
                    .index_register_count(index_eew, vtype.vsew())
                    .ok_or(ExecutionError::IllegalInstruction {
                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                    })?;
                zvexx_load_helpers::validate_segment_registers::<Reg, _, _, _>(
                    program_counter,
                    vd,
                    vm,
                    data_group_regs,
                    nf,
                )?;
                zvexx_load_helpers::check_register_group_alignment::<Reg, _, _, _>(
                    program_counter,
                    vs2,
                    index_group_regs,
                )?;
                for f in 0..nf.fields_per_segment() {
                    // SAFETY: `vd.to_bits() + f * data_group_regs < 32` because
                    // `validate_segment_registers` established `vd.to_bits() + nf * data_group_regs
                    // <= 32` and `f < nf`. The value is in [0, 31], so it is a valid `VReg`
                    // encoding.
                    let field_vd = unsafe {
                        VReg::from_bits(vd.to_bits() + f * data_group_regs.get()).unwrap_unchecked()
                    };
                    if zvexx_load_helpers::groups_overlap(
                        field_vd,
                        data_group_regs,
                        vs2,
                        index_group_regs,
                    ) {
                        ::core::hint::cold_path();
                        return Err(ExecutionError::IllegalInstruction {
                            address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
                        });
                    }
                }
                // SAFETY: preconditions identical to `Vluxseg`; see that arm for the full
                // argument
                unsafe {
                    zvexx_load_helpers::execute_indexed_load(
                        ext_state,
                        memory,
                        vd,
                        vs2,
                        vm,
                        rs1_value.as_u64(),
                        vtype.vsew().as_eew(),
                        index_eew,
                        data_group_regs,
                        nf,
                    )?;
                }
            }
        }

        Ok(ControlFlow::Continue(Default::default()))
    }
}