Skip to main content

ab_riscv_interpreter/v/zvexx/
muldiv.rs

1//! ZveXx multiply and divide instructions
2
3#[cfg(test)]
4mod tests;
5pub mod zvexx_muldiv_helpers;
6
7use crate::v::vector_registers::VectorRegistersExt;
8use crate::v::zvexx::zvexx_helpers;
9use crate::{
10    ExecutableInstruction, ExecutableInstructionCsr, ExecutableInstructionOperands, ExecutionError,
11    ProgramCounter, RegisterFile, Rs1Rs2OperandValues, Rs1Rs2Operands, VirtualMemory,
12};
13use ab_riscv_macros::instruction_execution;
14use ab_riscv_primitives::prelude::*;
15use core::fmt;
16use core::ops::ControlFlow;
17
18#[instruction_execution]
19impl<Reg> ExecutableInstructionOperands for ZveXxMulDivInstruction<Reg> where Reg: Register {}
20
21#[instruction_execution]
22impl<Reg, ExtState, CustomError> ExecutableInstructionCsr<ExtState, CustomError>
23    for ZveXxMulDivInstruction<Reg>
24where
25    Reg: Register,
26{
27}
28
29#[instruction_execution]
30impl<Reg, Regs, ExtState, Memory, PC, InstructionHandler, CustomError>
31    ExecutableInstruction<Regs, ExtState, Memory, PC, InstructionHandler, CustomError>
32    for ZveXxMulDivInstruction<Reg>
33where
34    Reg: Register,
35    Regs: RegisterFile<Reg>,
36    ExtState: VectorRegistersExt<Reg, CustomError>,
37    [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
38    Memory: VirtualMemory,
39    PC: ProgramCounter<Reg::Type, Memory, CustomError>,
40    CustomError: fmt::Debug,
41{
42    #[inline(always)]
43    fn execute(
44        self,
45        Rs1Rs2OperandValues {
46            rs1_value,
47            rs2_value: _,
48        }: Rs1Rs2OperandValues<<Self::Reg as Register>::Type>,
49        _regs: &mut Regs,
50        ext_state: &mut ExtState,
51        _memory: &mut Memory,
52        program_counter: &mut PC,
53        _system_instruction_handler: &mut InstructionHandler,
54    ) -> Result<
55        ControlFlow<(), (Self::Reg, <Self::Reg as Register>::Type)>,
56        ExecutionError<Reg::Type, CustomError>,
57    > {
58        match self {
59            // vmul.vv / vmul.vx - signed multiply, low half
60            Self::VmulVv { vd, vs2, vs1, vm } => {
61                if !ext_state.vector_instructions_allowed() {
62                    ::core::hint::cold_path();
63                    return Err(ExecutionError::IllegalInstruction {
64                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
65                    });
66                }
67                let Some(vtype) = ext_state.vtype() else {
68                    ::core::hint::cold_path();
69                    return Err(ExecutionError::IllegalInstruction {
70                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
71                    });
72                };
73                let group_regs = vtype.vlmul().register_count();
74                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
75                    program_counter,
76                    vd,
77                    group_regs,
78                )?;
79                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
80                    program_counter,
81                    vs2,
82                    group_regs,
83                )?;
84                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
85                    program_counter,
86                    vs1,
87                    group_regs,
88                )?;
89                if !vm && vd == VReg::V0 {
90                    ::core::hint::cold_path();
91                    return Err(ExecutionError::IllegalInstruction {
92                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
93                    });
94                }
95                let sew = vtype.vsew();
96                // SAFETY: alignment checked above
97                unsafe {
98                    zvexx_muldiv_helpers::execute_arith_op(
99                        ext_state,
100                        vd,
101                        vs2,
102                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
103                        vm,
104                        sew,
105                        |a, b, _| a.wrapping_mul(b),
106                    );
107                }
108            }
109            Self::VmulVx {
110                vd,
111                vs2,
112                rs1: _,
113                vm,
114            } => {
115                if !ext_state.vector_instructions_allowed() {
116                    ::core::hint::cold_path();
117                    return Err(ExecutionError::IllegalInstruction {
118                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
119                    });
120                }
121                let Some(vtype) = ext_state.vtype() else {
122                    ::core::hint::cold_path();
123                    return Err(ExecutionError::IllegalInstruction {
124                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
125                    });
126                };
127                let group_regs = vtype.vlmul().register_count();
128                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
129                    program_counter,
130                    vd,
131                    group_regs,
132                )?;
133                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
134                    program_counter,
135                    vs2,
136                    group_regs,
137                )?;
138                if !vm && vd == VReg::V0 {
139                    ::core::hint::cold_path();
140                    return Err(ExecutionError::IllegalInstruction {
141                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
142                    });
143                }
144                let sew = vtype.vsew();
145                let scalar = rs1_value.as_i64().cast_unsigned();
146                // SAFETY: alignment checked above
147                unsafe {
148                    zvexx_muldiv_helpers::execute_arith_op(
149                        ext_state,
150                        vd,
151                        vs2,
152                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
153                        vm,
154                        sew,
155                        |a, b, _| a.wrapping_mul(b),
156                    );
157                }
158            }
159            // vmulh.vv / vmulh.vx - signed×signed multiply, high half; illegal for SEW=64
160            Self::VmulhVv { vd, vs2, vs1, vm } => {
161                if !ext_state.vector_instructions_allowed() {
162                    ::core::hint::cold_path();
163                    return Err(ExecutionError::IllegalInstruction {
164                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
165                    });
166                }
167                let Some(vtype) = ext_state.vtype() else {
168                    ::core::hint::cold_path();
169                    return Err(ExecutionError::IllegalInstruction {
170                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
171                    });
172                };
173                // vmulh is not supported for SEW=64 in Zve64x (would need 128-bit result)
174                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
175                    ::core::hint::cold_path();
176                    return Err(ExecutionError::IllegalInstruction {
177                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
178                    });
179                }
180                let group_regs = vtype.vlmul().register_count();
181                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
182                    program_counter,
183                    vd,
184                    group_regs,
185                )?;
186                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
187                    program_counter,
188                    vs2,
189                    group_regs,
190                )?;
191                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
192                    program_counter,
193                    vs1,
194                    group_regs,
195                )?;
196                if !vm && vd == VReg::V0 {
197                    ::core::hint::cold_path();
198                    return Err(ExecutionError::IllegalInstruction {
199                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
200                    });
201                }
202                let sew = vtype.vsew();
203                // SAFETY: alignment checked above; SEW < 64 checked above
204                unsafe {
205                    zvexx_muldiv_helpers::execute_arith_op(
206                        ext_state,
207                        vd,
208                        vs2,
209                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
210                        vm,
211                        sew,
212                        zvexx_muldiv_helpers::mulh_ss,
213                    );
214                }
215            }
216            Self::VmulhVx {
217                vd,
218                vs2,
219                rs1: _,
220                vm,
221            } => {
222                if !ext_state.vector_instructions_allowed() {
223                    ::core::hint::cold_path();
224                    return Err(ExecutionError::IllegalInstruction {
225                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
226                    });
227                }
228                let Some(vtype) = ext_state.vtype() else {
229                    ::core::hint::cold_path();
230                    return Err(ExecutionError::IllegalInstruction {
231                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
232                    });
233                };
234                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
235                    ::core::hint::cold_path();
236                    return Err(ExecutionError::IllegalInstruction {
237                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
238                    });
239                }
240                let group_regs = vtype.vlmul().register_count();
241                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
242                    program_counter,
243                    vd,
244                    group_regs,
245                )?;
246                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
247                    program_counter,
248                    vs2,
249                    group_regs,
250                )?;
251                if !vm && vd == VReg::V0 {
252                    ::core::hint::cold_path();
253                    return Err(ExecutionError::IllegalInstruction {
254                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
255                    });
256                }
257                let sew = vtype.vsew();
258                let scalar = rs1_value.as_u64();
259                // SAFETY: alignment checked above; SEW < 64 checked above
260                unsafe {
261                    zvexx_muldiv_helpers::execute_arith_op(
262                        ext_state,
263                        vd,
264                        vs2,
265                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
266                        vm,
267                        sew,
268                        zvexx_muldiv_helpers::mulh_ss,
269                    );
270                }
271            }
272            // vmulhu.vv / vmulhu.vx - unsigned×unsigned multiply, high half; illegal for SEW=64
273            Self::VmulhuVv { vd, vs2, vs1, vm } => {
274                if !ext_state.vector_instructions_allowed() {
275                    ::core::hint::cold_path();
276                    return Err(ExecutionError::IllegalInstruction {
277                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
278                    });
279                }
280                let Some(vtype) = ext_state.vtype() else {
281                    ::core::hint::cold_path();
282                    return Err(ExecutionError::IllegalInstruction {
283                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
284                    });
285                };
286                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
287                    ::core::hint::cold_path();
288                    return Err(ExecutionError::IllegalInstruction {
289                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
290                    });
291                }
292                let group_regs = vtype.vlmul().register_count();
293                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
294                    program_counter,
295                    vd,
296                    group_regs,
297                )?;
298                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
299                    program_counter,
300                    vs2,
301                    group_regs,
302                )?;
303                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
304                    program_counter,
305                    vs1,
306                    group_regs,
307                )?;
308                if !vm && vd == VReg::V0 {
309                    ::core::hint::cold_path();
310                    return Err(ExecutionError::IllegalInstruction {
311                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
312                    });
313                }
314                let sew = vtype.vsew();
315                // SAFETY: alignment checked above; SEW < 64 checked above
316                unsafe {
317                    zvexx_muldiv_helpers::execute_arith_op(
318                        ext_state,
319                        vd,
320                        vs2,
321                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
322                        vm,
323                        sew,
324                        zvexx_muldiv_helpers::mulhu_uu,
325                    );
326                }
327            }
328            Self::VmulhuVx {
329                vd,
330                vs2,
331                rs1: _,
332                vm,
333            } => {
334                if !ext_state.vector_instructions_allowed() {
335                    ::core::hint::cold_path();
336                    return Err(ExecutionError::IllegalInstruction {
337                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
338                    });
339                }
340                let Some(vtype) = ext_state.vtype() else {
341                    ::core::hint::cold_path();
342                    return Err(ExecutionError::IllegalInstruction {
343                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
344                    });
345                };
346                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
347                    ::core::hint::cold_path();
348                    return Err(ExecutionError::IllegalInstruction {
349                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
350                    });
351                }
352                let group_regs = vtype.vlmul().register_count();
353                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
354                    program_counter,
355                    vd,
356                    group_regs,
357                )?;
358                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
359                    program_counter,
360                    vs2,
361                    group_regs,
362                )?;
363                if !vm && vd == VReg::V0 {
364                    ::core::hint::cold_path();
365                    return Err(ExecutionError::IllegalInstruction {
366                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
367                    });
368                }
369                let sew = vtype.vsew();
370                let scalar = rs1_value.as_u64();
371                // SAFETY: alignment checked above; SEW < 64 checked above
372                unsafe {
373                    zvexx_muldiv_helpers::execute_arith_op(
374                        ext_state,
375                        vd,
376                        vs2,
377                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
378                        vm,
379                        sew,
380                        zvexx_muldiv_helpers::mulhu_uu,
381                    );
382                }
383            }
384            // vmulhsu.vv / vmulhsu.vx - signed×unsigned multiply, high half; illegal for SEW=64
385            Self::VmulhsuVv { vd, vs2, vs1, vm } => {
386                if !ext_state.vector_instructions_allowed() {
387                    ::core::hint::cold_path();
388                    return Err(ExecutionError::IllegalInstruction {
389                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
390                    });
391                }
392                let Some(vtype) = ext_state.vtype() else {
393                    ::core::hint::cold_path();
394                    return Err(ExecutionError::IllegalInstruction {
395                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
396                    });
397                };
398                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
399                    ::core::hint::cold_path();
400                    return Err(ExecutionError::IllegalInstruction {
401                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
402                    });
403                }
404                let group_regs = vtype.vlmul().register_count();
405                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
406                    program_counter,
407                    vd,
408                    group_regs,
409                )?;
410                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
411                    program_counter,
412                    vs2,
413                    group_regs,
414                )?;
415                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
416                    program_counter,
417                    vs1,
418                    group_regs,
419                )?;
420                if !vm && vd == VReg::V0 {
421                    ::core::hint::cold_path();
422                    return Err(ExecutionError::IllegalInstruction {
423                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
424                    });
425                }
426                let sew = vtype.vsew();
427                // SAFETY: alignment checked above; SEW < 64 checked above
428                unsafe {
429                    zvexx_muldiv_helpers::execute_arith_op(
430                        ext_state,
431                        vd,
432                        vs2,
433                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
434                        vm,
435                        sew,
436                        // vs2 is signed, vs1 is unsigned
437                        zvexx_muldiv_helpers::mulhsu_su,
438                    );
439                }
440            }
441            Self::VmulhsuVx {
442                vd,
443                vs2,
444                rs1: _,
445                vm,
446            } => {
447                if !ext_state.vector_instructions_allowed() {
448                    ::core::hint::cold_path();
449                    return Err(ExecutionError::IllegalInstruction {
450                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
451                    });
452                }
453                let Some(vtype) = ext_state.vtype() else {
454                    ::core::hint::cold_path();
455                    return Err(ExecutionError::IllegalInstruction {
456                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
457                    });
458                };
459                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
460                    ::core::hint::cold_path();
461                    return Err(ExecutionError::IllegalInstruction {
462                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
463                    });
464                }
465                let group_regs = vtype.vlmul().register_count();
466                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
467                    program_counter,
468                    vd,
469                    group_regs,
470                )?;
471                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
472                    program_counter,
473                    vs2,
474                    group_regs,
475                )?;
476                if !vm && vd == VReg::V0 {
477                    ::core::hint::cold_path();
478                    return Err(ExecutionError::IllegalInstruction {
479                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
480                    });
481                }
482                let sew = vtype.vsew();
483                // scalar from rs1 is the unsigned operand; vs2 elements are signed
484                let scalar = rs1_value.as_u64();
485                // SAFETY: alignment checked above; SEW < 64 checked above
486                unsafe {
487                    zvexx_muldiv_helpers::execute_arith_op(
488                        ext_state,
489                        vd,
490                        vs2,
491                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
492                        vm,
493                        sew,
494                        // vs2 is signed, scalar (rs1) is unsigned
495                        zvexx_muldiv_helpers::mulhsu_su,
496                    );
497                }
498            }
499            // vdivu.vv / vdivu.vx - unsigned divide
500            Self::VdivuVv { vd, vs2, vs1, vm } => {
501                if !ext_state.vector_instructions_allowed() {
502                    ::core::hint::cold_path();
503                    return Err(ExecutionError::IllegalInstruction {
504                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
505                    });
506                }
507                let Some(vtype) = ext_state.vtype() else {
508                    ::core::hint::cold_path();
509                    return Err(ExecutionError::IllegalInstruction {
510                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
511                    });
512                };
513                let group_regs = vtype.vlmul().register_count();
514                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
515                    program_counter,
516                    vd,
517                    group_regs,
518                )?;
519                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
520                    program_counter,
521                    vs2,
522                    group_regs,
523                )?;
524                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
525                    program_counter,
526                    vs1,
527                    group_regs,
528                )?;
529                if !vm && vd == VReg::V0 {
530                    ::core::hint::cold_path();
531                    return Err(ExecutionError::IllegalInstruction {
532                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
533                    });
534                }
535                let sew = vtype.vsew();
536                // SAFETY: alignment checked above
537                unsafe {
538                    zvexx_muldiv_helpers::execute_arith_op(
539                        ext_state,
540                        vd,
541                        vs2,
542                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
543                        vm,
544                        sew,
545                        // Division by zero: quotient = all-ones for the SEW width (spec §12.11)
546                        |a, b, sew| {
547                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
548                            let dividend = a & mask;
549                            let divisor = b & mask;
550                            dividend.checked_div(divisor).unwrap_or(mask)
551                        },
552                    );
553                }
554            }
555            Self::VdivuVx {
556                vd,
557                vs2,
558                rs1: _,
559                vm,
560            } => {
561                if !ext_state.vector_instructions_allowed() {
562                    ::core::hint::cold_path();
563                    return Err(ExecutionError::IllegalInstruction {
564                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
565                    });
566                }
567                let Some(vtype) = ext_state.vtype() else {
568                    ::core::hint::cold_path();
569                    return Err(ExecutionError::IllegalInstruction {
570                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
571                    });
572                };
573                let group_regs = vtype.vlmul().register_count();
574                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
575                    program_counter,
576                    vd,
577                    group_regs,
578                )?;
579                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
580                    program_counter,
581                    vs2,
582                    group_regs,
583                )?;
584                if !vm && vd == VReg::V0 {
585                    ::core::hint::cold_path();
586                    return Err(ExecutionError::IllegalInstruction {
587                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
588                    });
589                }
590                let sew = vtype.vsew();
591                let scalar = rs1_value.as_i64().cast_unsigned();
592                // SAFETY: alignment checked above
593                unsafe {
594                    zvexx_muldiv_helpers::execute_arith_op(
595                        ext_state,
596                        vd,
597                        vs2,
598                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
599                        vm,
600                        sew,
601                        |a, b, sew| {
602                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
603                            let dividend = a & mask;
604                            let divisor = b & mask;
605                            dividend.checked_div(divisor).unwrap_or(mask)
606                        },
607                    );
608                }
609            }
610            // vdiv.vv / vdiv.vx - signed divide
611            Self::VdivVv { vd, vs2, vs1, vm } => {
612                if !ext_state.vector_instructions_allowed() {
613                    ::core::hint::cold_path();
614                    return Err(ExecutionError::IllegalInstruction {
615                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
616                    });
617                }
618                let Some(vtype) = ext_state.vtype() else {
619                    ::core::hint::cold_path();
620                    return Err(ExecutionError::IllegalInstruction {
621                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
622                    });
623                };
624                let group_regs = vtype.vlmul().register_count();
625                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
626                    program_counter,
627                    vd,
628                    group_regs,
629                )?;
630                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
631                    program_counter,
632                    vs2,
633                    group_regs,
634                )?;
635                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
636                    program_counter,
637                    vs1,
638                    group_regs,
639                )?;
640                if !vm && vd == VReg::V0 {
641                    ::core::hint::cold_path();
642                    return Err(ExecutionError::IllegalInstruction {
643                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
644                    });
645                }
646                let sew = vtype.vsew();
647                // SAFETY: alignment checked above
648                unsafe {
649                    zvexx_muldiv_helpers::execute_arith_op(
650                        ext_state,
651                        vd,
652                        vs2,
653                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
654                        vm,
655                        sew,
656                        zvexx_muldiv_helpers::sdiv,
657                    );
658                }
659            }
660            Self::VdivVx {
661                vd,
662                vs2,
663                rs1: _,
664                vm,
665            } => {
666                if !ext_state.vector_instructions_allowed() {
667                    ::core::hint::cold_path();
668                    return Err(ExecutionError::IllegalInstruction {
669                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
670                    });
671                }
672                let Some(vtype) = ext_state.vtype() else {
673                    ::core::hint::cold_path();
674                    return Err(ExecutionError::IllegalInstruction {
675                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
676                    });
677                };
678                let group_regs = vtype.vlmul().register_count();
679                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
680                    program_counter,
681                    vd,
682                    group_regs,
683                )?;
684                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
685                    program_counter,
686                    vs2,
687                    group_regs,
688                )?;
689                if !vm && vd == VReg::V0 {
690                    ::core::hint::cold_path();
691                    return Err(ExecutionError::IllegalInstruction {
692                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
693                    });
694                }
695                let sew = vtype.vsew();
696                let scalar = rs1_value.as_i64().cast_unsigned();
697                // SAFETY: alignment checked above
698                unsafe {
699                    zvexx_muldiv_helpers::execute_arith_op(
700                        ext_state,
701                        vd,
702                        vs2,
703                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
704                        vm,
705                        sew,
706                        zvexx_muldiv_helpers::sdiv,
707                    );
708                }
709            }
710            // vremu.vv / vremu.vx - unsigned remainder
711            Self::VremuVv { vd, vs2, vs1, vm } => {
712                if !ext_state.vector_instructions_allowed() {
713                    ::core::hint::cold_path();
714                    return Err(ExecutionError::IllegalInstruction {
715                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
716                    });
717                }
718                let Some(vtype) = ext_state.vtype() else {
719                    ::core::hint::cold_path();
720                    return Err(ExecutionError::IllegalInstruction {
721                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
722                    });
723                };
724                let group_regs = vtype.vlmul().register_count();
725                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
726                    program_counter,
727                    vd,
728                    group_regs,
729                )?;
730                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
731                    program_counter,
732                    vs2,
733                    group_regs,
734                )?;
735                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
736                    program_counter,
737                    vs1,
738                    group_regs,
739                )?;
740                if !vm && vd == VReg::V0 {
741                    ::core::hint::cold_path();
742                    return Err(ExecutionError::IllegalInstruction {
743                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
744                    });
745                }
746                let sew = vtype.vsew();
747                // SAFETY: alignment checked above
748                unsafe {
749                    zvexx_muldiv_helpers::execute_arith_op(
750                        ext_state,
751                        vd,
752                        vs2,
753                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
754                        vm,
755                        sew,
756                        // Division by zero: remainder = dividend (spec §12.11)
757                        |a, b, sew| {
758                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
759                            let dividend = a & mask;
760                            let divisor = b & mask;
761                            if divisor == 0 {
762                                dividend
763                            } else {
764                                dividend % divisor
765                            }
766                        },
767                    );
768                }
769            }
770            Self::VremuVx {
771                vd,
772                vs2,
773                rs1: _,
774                vm,
775            } => {
776                if !ext_state.vector_instructions_allowed() {
777                    ::core::hint::cold_path();
778                    return Err(ExecutionError::IllegalInstruction {
779                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
780                    });
781                }
782                let Some(vtype) = ext_state.vtype() else {
783                    ::core::hint::cold_path();
784                    return Err(ExecutionError::IllegalInstruction {
785                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
786                    });
787                };
788                let group_regs = vtype.vlmul().register_count();
789                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
790                    program_counter,
791                    vd,
792                    group_regs,
793                )?;
794                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
795                    program_counter,
796                    vs2,
797                    group_regs,
798                )?;
799                if !vm && vd == VReg::V0 {
800                    ::core::hint::cold_path();
801                    return Err(ExecutionError::IllegalInstruction {
802                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
803                    });
804                }
805                let sew = vtype.vsew();
806                let scalar = rs1_value.as_i64().cast_unsigned();
807                // SAFETY: alignment checked above
808                unsafe {
809                    zvexx_muldiv_helpers::execute_arith_op(
810                        ext_state,
811                        vd,
812                        vs2,
813                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
814                        vm,
815                        sew,
816                        |a, b, sew| {
817                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
818                            let dividend = a & mask;
819                            let divisor = b & mask;
820                            if divisor == 0 {
821                                dividend
822                            } else {
823                                dividend % divisor
824                            }
825                        },
826                    );
827                }
828            }
829            // vrem.vv / vrem.vx - signed remainder
830            Self::VremVv { vd, vs2, vs1, vm } => {
831                if !ext_state.vector_instructions_allowed() {
832                    ::core::hint::cold_path();
833                    return Err(ExecutionError::IllegalInstruction {
834                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
835                    });
836                }
837                let Some(vtype) = ext_state.vtype() else {
838                    ::core::hint::cold_path();
839                    return Err(ExecutionError::IllegalInstruction {
840                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
841                    });
842                };
843                let group_regs = vtype.vlmul().register_count();
844                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
845                    program_counter,
846                    vd,
847                    group_regs,
848                )?;
849                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
850                    program_counter,
851                    vs2,
852                    group_regs,
853                )?;
854                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
855                    program_counter,
856                    vs1,
857                    group_regs,
858                )?;
859                if !vm && vd == VReg::V0 {
860                    ::core::hint::cold_path();
861                    return Err(ExecutionError::IllegalInstruction {
862                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
863                    });
864                }
865                let sew = vtype.vsew();
866                // SAFETY: alignment checked above
867                unsafe {
868                    zvexx_muldiv_helpers::execute_arith_op(
869                        ext_state,
870                        vd,
871                        vs2,
872                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
873                        vm,
874                        sew,
875                        zvexx_muldiv_helpers::srem,
876                    );
877                }
878            }
879            Self::VremVx {
880                vd,
881                vs2,
882                rs1: _,
883                vm,
884            } => {
885                if !ext_state.vector_instructions_allowed() {
886                    ::core::hint::cold_path();
887                    return Err(ExecutionError::IllegalInstruction {
888                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
889                    });
890                }
891                let Some(vtype) = ext_state.vtype() else {
892                    ::core::hint::cold_path();
893                    return Err(ExecutionError::IllegalInstruction {
894                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
895                    });
896                };
897                let group_regs = vtype.vlmul().register_count();
898                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
899                    program_counter,
900                    vd,
901                    group_regs,
902                )?;
903                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
904                    program_counter,
905                    vs2,
906                    group_regs,
907                )?;
908                if !vm && vd == VReg::V0 {
909                    ::core::hint::cold_path();
910                    return Err(ExecutionError::IllegalInstruction {
911                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
912                    });
913                }
914                let sew = vtype.vsew();
915                let scalar = rs1_value.as_i64().cast_unsigned();
916                // SAFETY: alignment checked above
917                unsafe {
918                    zvexx_muldiv_helpers::execute_arith_op(
919                        ext_state,
920                        vd,
921                        vs2,
922                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
923                        vm,
924                        sew,
925                        zvexx_muldiv_helpers::srem,
926                    );
927                }
928            }
929            // vwmulu.vv / vwmulu.vx - unsigned widening multiply; illegal for SEW=64
930            Self::VwmuluVv { vd, vs2, vs1, vm } => {
931                if !ext_state.vector_instructions_allowed() {
932                    ::core::hint::cold_path();
933                    return Err(ExecutionError::IllegalInstruction {
934                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
935                    });
936                }
937                let Some(vtype) = ext_state.vtype() else {
938                    ::core::hint::cold_path();
939                    return Err(ExecutionError::IllegalInstruction {
940                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
941                    });
942                };
943                // Widening produces 2*SEW result; SEW=64 would require 128-bit output
944                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
945                    ::core::hint::cold_path();
946                    return Err(ExecutionError::IllegalInstruction {
947                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
948                    });
949                }
950                let group_regs = vtype.vlmul().register_count();
951                // dest_group_regs encodes EMUL=2*LMUL; None means EMUL>8, which is illegal
952                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
953                    vtype.vlmul(),
954                )
955                .ok_or(ExecutionError::IllegalInstruction {
956                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
957                })?;
958                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
959                    program_counter,
960                    vd,
961                    dest_group_regs,
962                )?;
963                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
964                    program_counter,
965                    vs2,
966                    group_regs,
967                )?;
968                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
969                    program_counter,
970                    vs1,
971                    group_regs,
972                )?;
973                if !vm && vd == VReg::V0 {
974                    ::core::hint::cold_path();
975                    return Err(ExecutionError::IllegalInstruction {
976                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
977                    });
978                }
979                // vd and vs2/vs1 must not overlap
980                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
981                    program_counter,
982                    vd,
983                    vs2,
984                    dest_group_regs,
985                    group_regs,
986                )?;
987                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
988                    program_counter,
989                    vd,
990                    vs1,
991                    dest_group_regs,
992                    group_regs,
993                )?;
994                let sew = vtype.vsew();
995                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
996                unsafe {
997                    zvexx_muldiv_helpers::execute_widening_op(
998                        ext_state,
999                        vd,
1000                        vs2,
1001                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
1002                        vm,
1003                        sew,
1004                        |a, b, sew| {
1005                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
1006                            (a & mask).wrapping_mul(b & mask)
1007                        },
1008                    );
1009                }
1010            }
1011            Self::VwmuluVx {
1012                vd,
1013                vs2,
1014                rs1: _,
1015                vm,
1016            } => {
1017                if !ext_state.vector_instructions_allowed() {
1018                    ::core::hint::cold_path();
1019                    return Err(ExecutionError::IllegalInstruction {
1020                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1021                    });
1022                }
1023                let Some(vtype) = ext_state.vtype() else {
1024                    ::core::hint::cold_path();
1025                    return Err(ExecutionError::IllegalInstruction {
1026                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1027                    });
1028                };
1029                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1030                    ::core::hint::cold_path();
1031                    return Err(ExecutionError::IllegalInstruction {
1032                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1033                    });
1034                }
1035                let group_regs = vtype.vlmul().register_count();
1036                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1037                    vtype.vlmul(),
1038                )
1039                .ok_or(ExecutionError::IllegalInstruction {
1040                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1041                })?;
1042                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1043                    program_counter,
1044                    vd,
1045                    dest_group_regs,
1046                )?;
1047                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1048                    program_counter,
1049                    vs2,
1050                    group_regs,
1051                )?;
1052                if !vm && vd == VReg::V0 {
1053                    ::core::hint::cold_path();
1054                    return Err(ExecutionError::IllegalInstruction {
1055                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1056                    });
1057                }
1058                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1059                    program_counter,
1060                    vd,
1061                    vs2,
1062                    dest_group_regs,
1063                    group_regs,
1064                )?;
1065                let sew = vtype.vsew();
1066                let scalar = rs1_value.as_u64();
1067                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1068                unsafe {
1069                    zvexx_muldiv_helpers::execute_widening_op(
1070                        ext_state,
1071                        vd,
1072                        vs2,
1073                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
1074                        vm,
1075                        sew,
1076                        |a, b, sew| {
1077                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
1078                            (a & mask).wrapping_mul(b & mask)
1079                        },
1080                    );
1081                }
1082            }
1083            // vwmulsu.vv / vwmulsu.vx - signed×unsigned widening multiply; illegal for SEW=64
1084            Self::VwmulsuVv { vd, vs2, vs1, vm } => {
1085                if !ext_state.vector_instructions_allowed() {
1086                    ::core::hint::cold_path();
1087                    return Err(ExecutionError::IllegalInstruction {
1088                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1089                    });
1090                }
1091                let Some(vtype) = ext_state.vtype() else {
1092                    ::core::hint::cold_path();
1093                    return Err(ExecutionError::IllegalInstruction {
1094                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1095                    });
1096                };
1097                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1098                    ::core::hint::cold_path();
1099                    return Err(ExecutionError::IllegalInstruction {
1100                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1101                    });
1102                }
1103                let group_regs = vtype.vlmul().register_count();
1104                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1105                    vtype.vlmul(),
1106                )
1107                .ok_or(ExecutionError::IllegalInstruction {
1108                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1109                })?;
1110                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1111                    program_counter,
1112                    vd,
1113                    dest_group_regs,
1114                )?;
1115                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1116                    program_counter,
1117                    vs2,
1118                    group_regs,
1119                )?;
1120                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1121                    program_counter,
1122                    vs1,
1123                    group_regs,
1124                )?;
1125                if !vm && vd == VReg::V0 {
1126                    ::core::hint::cold_path();
1127                    return Err(ExecutionError::IllegalInstruction {
1128                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1129                    });
1130                }
1131                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1132                    program_counter,
1133                    vd,
1134                    vs2,
1135                    dest_group_regs,
1136                    group_regs,
1137                )?;
1138                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1139                    program_counter,
1140                    vd,
1141                    vs1,
1142                    dest_group_regs,
1143                    group_regs,
1144                )?;
1145                let sew = vtype.vsew();
1146                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1147                unsafe {
1148                    zvexx_muldiv_helpers::execute_widening_op(
1149                        ext_state,
1150                        vd,
1151                        vs2,
1152                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
1153                        vm,
1154                        sew,
1155                        // vs2 is signed, vs1 is unsigned; widen both to full u64 before multiply
1156                        |a, b, sew| {
1157                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
1158                            let ub = b & zvexx_muldiv_helpers::sew_mask(sew);
1159                            sa.cast_unsigned().wrapping_mul(ub)
1160                        },
1161                    );
1162                }
1163            }
1164            Self::VwmulsuVx {
1165                vd,
1166                vs2,
1167                rs1: _,
1168                vm,
1169            } => {
1170                if !ext_state.vector_instructions_allowed() {
1171                    ::core::hint::cold_path();
1172                    return Err(ExecutionError::IllegalInstruction {
1173                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1174                    });
1175                }
1176                let Some(vtype) = ext_state.vtype() else {
1177                    ::core::hint::cold_path();
1178                    return Err(ExecutionError::IllegalInstruction {
1179                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1180                    });
1181                };
1182                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1183                    ::core::hint::cold_path();
1184                    return Err(ExecutionError::IllegalInstruction {
1185                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1186                    });
1187                }
1188                let group_regs = vtype.vlmul().register_count();
1189                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1190                    vtype.vlmul(),
1191                )
1192                .ok_or(ExecutionError::IllegalInstruction {
1193                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1194                })?;
1195                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1196                    program_counter,
1197                    vd,
1198                    dest_group_regs,
1199                )?;
1200                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1201                    program_counter,
1202                    vs2,
1203                    group_regs,
1204                )?;
1205                if !vm && vd == VReg::V0 {
1206                    ::core::hint::cold_path();
1207                    return Err(ExecutionError::IllegalInstruction {
1208                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1209                    });
1210                }
1211                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1212                    program_counter,
1213                    vd,
1214                    vs2,
1215                    dest_group_regs,
1216                    group_regs,
1217                )?;
1218                let sew = vtype.vsew();
1219                // scalar from rs1 is the unsigned operand; vs2 elements are signed
1220                let scalar = rs1_value.as_u64();
1221                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1222                unsafe {
1223                    zvexx_muldiv_helpers::execute_widening_op(
1224                        ext_state,
1225                        vd,
1226                        vs2,
1227                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
1228                        vm,
1229                        sew,
1230                        |a, b, sew| {
1231                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
1232                            let ub = b & zvexx_muldiv_helpers::sew_mask(sew);
1233                            sa.cast_unsigned().wrapping_mul(ub)
1234                        },
1235                    );
1236                }
1237            }
1238            // vwmul.vv / vwmul.vx - signed widening multiply; illegal for SEW=64
1239            Self::VwmulVv { vd, vs2, vs1, vm } => {
1240                if !ext_state.vector_instructions_allowed() {
1241                    ::core::hint::cold_path();
1242                    return Err(ExecutionError::IllegalInstruction {
1243                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1244                    });
1245                }
1246                let Some(vtype) = ext_state.vtype() else {
1247                    ::core::hint::cold_path();
1248                    return Err(ExecutionError::IllegalInstruction {
1249                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1250                    });
1251                };
1252                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1253                    ::core::hint::cold_path();
1254                    return Err(ExecutionError::IllegalInstruction {
1255                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1256                    });
1257                }
1258                let group_regs = vtype.vlmul().register_count();
1259                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1260                    vtype.vlmul(),
1261                )
1262                .ok_or(ExecutionError::IllegalInstruction {
1263                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1264                })?;
1265                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1266                    program_counter,
1267                    vd,
1268                    dest_group_regs,
1269                )?;
1270                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1271                    program_counter,
1272                    vs2,
1273                    group_regs,
1274                )?;
1275                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1276                    program_counter,
1277                    vs1,
1278                    group_regs,
1279                )?;
1280                if !vm && vd == VReg::V0 {
1281                    ::core::hint::cold_path();
1282                    return Err(ExecutionError::IllegalInstruction {
1283                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1284                    });
1285                }
1286                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1287                    program_counter,
1288                    vd,
1289                    vs2,
1290                    dest_group_regs,
1291                    group_regs,
1292                )?;
1293                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1294                    program_counter,
1295                    vd,
1296                    vs1,
1297                    dest_group_regs,
1298                    group_regs,
1299                )?;
1300                let sew = vtype.vsew();
1301                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1302                unsafe {
1303                    zvexx_muldiv_helpers::execute_widening_op(
1304                        ext_state,
1305                        vd,
1306                        vs2,
1307                        zvexx_muldiv_helpers::OpSrc::Vreg(vs1),
1308                        vm,
1309                        sew,
1310                        // Both operands sign-extended; full 2*SEW product fits in u64
1311                        |a, b, sew| {
1312                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
1313                            let sb = zvexx_muldiv_helpers::sign_extend(b, sew);
1314                            sa.cast_unsigned().wrapping_mul(sb.cast_unsigned())
1315                        },
1316                    );
1317                }
1318            }
1319            Self::VwmulVx {
1320                vd,
1321                vs2,
1322                rs1: _,
1323                vm,
1324            } => {
1325                if !ext_state.vector_instructions_allowed() {
1326                    ::core::hint::cold_path();
1327                    return Err(ExecutionError::IllegalInstruction {
1328                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1329                    });
1330                }
1331                let Some(vtype) = ext_state.vtype() else {
1332                    ::core::hint::cold_path();
1333                    return Err(ExecutionError::IllegalInstruction {
1334                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1335                    });
1336                };
1337                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1338                    ::core::hint::cold_path();
1339                    return Err(ExecutionError::IllegalInstruction {
1340                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1341                    });
1342                }
1343                let group_regs = vtype.vlmul().register_count();
1344                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1345                    vtype.vlmul(),
1346                )
1347                .ok_or(ExecutionError::IllegalInstruction {
1348                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1349                })?;
1350                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1351                    program_counter,
1352                    vd,
1353                    dest_group_regs,
1354                )?;
1355                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1356                    program_counter,
1357                    vs2,
1358                    group_regs,
1359                )?;
1360                if !vm && vd == VReg::V0 {
1361                    ::core::hint::cold_path();
1362                    return Err(ExecutionError::IllegalInstruction {
1363                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1364                    });
1365                }
1366                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1367                    program_counter,
1368                    vd,
1369                    vs2,
1370                    dest_group_regs,
1371                    group_regs,
1372                )?;
1373                let sew = vtype.vsew();
1374                // scalar from rs1 is sign-extended to XLEN; treat as signed SEW-wide
1375                let scalar = rs1_value.as_u64();
1376                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1377                unsafe {
1378                    zvexx_muldiv_helpers::execute_widening_op(
1379                        ext_state,
1380                        vd,
1381                        vs2,
1382                        zvexx_muldiv_helpers::OpSrc::Scalar(scalar),
1383                        vm,
1384                        sew,
1385                        |a, b, sew| {
1386                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
1387                            let sb = zvexx_muldiv_helpers::sign_extend(b, sew);
1388                            sa.cast_unsigned().wrapping_mul(sb.cast_unsigned())
1389                        },
1390                    );
1391                }
1392            }
1393            // vmacc.vv / vmacc.vx - vd = vd + vs1 * vs2
1394            Self::VmaccVv { vd, vs1, vs2, vm } => {
1395                if !ext_state.vector_instructions_allowed() {
1396                    ::core::hint::cold_path();
1397                    return Err(ExecutionError::IllegalInstruction {
1398                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1399                    });
1400                }
1401                let Some(vtype) = ext_state.vtype() else {
1402                    ::core::hint::cold_path();
1403                    return Err(ExecutionError::IllegalInstruction {
1404                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1405                    });
1406                };
1407                let group_regs = vtype.vlmul().register_count();
1408                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1409                    program_counter,
1410                    vd,
1411                    group_regs,
1412                )?;
1413                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1414                    program_counter,
1415                    vs2,
1416                    group_regs,
1417                )?;
1418                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1419                    program_counter,
1420                    vs1,
1421                    group_regs,
1422                )?;
1423                if !vm && vd == VReg::V0 {
1424                    ::core::hint::cold_path();
1425                    return Err(ExecutionError::IllegalInstruction {
1426                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1427                    });
1428                }
1429                let sew = vtype.vsew();
1430                // SAFETY: alignment checked above
1431                unsafe {
1432                    zvexx_muldiv_helpers::execute_muladd_op(
1433                        ext_state,
1434                        vd,
1435                        vs1,
1436                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1437                        vm,
1438                        sew,
1439                        // vmacc: vd[i] = vd[i] + vs1[i] * vs2[i]
1440                        |acc, a, b, _| acc.wrapping_add(a.wrapping_mul(b)),
1441                    );
1442                }
1443            }
1444            Self::VmaccVx {
1445                vd,
1446                rs1: _,
1447                vs2,
1448                vm,
1449            } => {
1450                if !ext_state.vector_instructions_allowed() {
1451                    ::core::hint::cold_path();
1452                    return Err(ExecutionError::IllegalInstruction {
1453                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1454                    });
1455                }
1456                let Some(vtype) = ext_state.vtype() else {
1457                    ::core::hint::cold_path();
1458                    return Err(ExecutionError::IllegalInstruction {
1459                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1460                    });
1461                };
1462                let group_regs = vtype.vlmul().register_count();
1463                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1464                    program_counter,
1465                    vd,
1466                    group_regs,
1467                )?;
1468                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1469                    program_counter,
1470                    vs2,
1471                    group_regs,
1472                )?;
1473                if !vm && vd == VReg::V0 {
1474                    ::core::hint::cold_path();
1475                    return Err(ExecutionError::IllegalInstruction {
1476                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1477                    });
1478                }
1479                let sew = vtype.vsew();
1480                let scalar = rs1_value.as_i64().cast_unsigned();
1481                // SAFETY: alignment checked above
1482                unsafe {
1483                    zvexx_muldiv_helpers::execute_muladd_scalar_op(
1484                        ext_state,
1485                        vd,
1486                        scalar,
1487                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1488                        vm,
1489                        sew,
1490                        |acc, a, b, _| acc.wrapping_add(a.wrapping_mul(b)),
1491                    );
1492                }
1493            }
1494            // vnmsac.vv / vnmsac.vx - vd = vd - vs1 * vs2
1495            Self::VnmsacVv { vd, vs1, vs2, vm } => {
1496                if !ext_state.vector_instructions_allowed() {
1497                    ::core::hint::cold_path();
1498                    return Err(ExecutionError::IllegalInstruction {
1499                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1500                    });
1501                }
1502                let Some(vtype) = ext_state.vtype() else {
1503                    ::core::hint::cold_path();
1504                    return Err(ExecutionError::IllegalInstruction {
1505                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1506                    });
1507                };
1508                let group_regs = vtype.vlmul().register_count();
1509                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1510                    program_counter,
1511                    vd,
1512                    group_regs,
1513                )?;
1514                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1515                    program_counter,
1516                    vs2,
1517                    group_regs,
1518                )?;
1519                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1520                    program_counter,
1521                    vs1,
1522                    group_regs,
1523                )?;
1524                if !vm && vd == VReg::V0 {
1525                    ::core::hint::cold_path();
1526                    return Err(ExecutionError::IllegalInstruction {
1527                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1528                    });
1529                }
1530                let sew = vtype.vsew();
1531                // SAFETY: alignment checked above
1532                unsafe {
1533                    zvexx_muldiv_helpers::execute_muladd_op(
1534                        ext_state,
1535                        vd,
1536                        vs1,
1537                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1538                        vm,
1539                        sew,
1540                        // vnmsac: vd[i] = vd[i] - vs1[i] * vs2[i]
1541                        |acc, a, b, _| acc.wrapping_sub(a.wrapping_mul(b)),
1542                    );
1543                }
1544            }
1545            Self::VnmsacVx {
1546                vd,
1547                rs1: _,
1548                vs2,
1549                vm,
1550            } => {
1551                if !ext_state.vector_instructions_allowed() {
1552                    ::core::hint::cold_path();
1553                    return Err(ExecutionError::IllegalInstruction {
1554                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1555                    });
1556                }
1557                let Some(vtype) = ext_state.vtype() else {
1558                    ::core::hint::cold_path();
1559                    return Err(ExecutionError::IllegalInstruction {
1560                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1561                    });
1562                };
1563                let group_regs = vtype.vlmul().register_count();
1564                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1565                    program_counter,
1566                    vd,
1567                    group_regs,
1568                )?;
1569                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1570                    program_counter,
1571                    vs2,
1572                    group_regs,
1573                )?;
1574                if !vm && vd == VReg::V0 {
1575                    ::core::hint::cold_path();
1576                    return Err(ExecutionError::IllegalInstruction {
1577                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1578                    });
1579                }
1580                let sew = vtype.vsew();
1581                let scalar = rs1_value.as_i64().cast_unsigned();
1582                // SAFETY: alignment checked above
1583                unsafe {
1584                    zvexx_muldiv_helpers::execute_muladd_scalar_op(
1585                        ext_state,
1586                        vd,
1587                        scalar,
1588                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1589                        vm,
1590                        sew,
1591                        |acc, a, b, _| acc.wrapping_sub(a.wrapping_mul(b)),
1592                    );
1593                }
1594            }
1595            // vmadd.vv / vmadd.vx - vd = vs1 * vd + vs2
1596            Self::VmaddVv { vd, vs1, vs2, vm } => {
1597                if !ext_state.vector_instructions_allowed() {
1598                    ::core::hint::cold_path();
1599                    return Err(ExecutionError::IllegalInstruction {
1600                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1601                    });
1602                }
1603                let Some(vtype) = ext_state.vtype() else {
1604                    ::core::hint::cold_path();
1605                    return Err(ExecutionError::IllegalInstruction {
1606                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1607                    });
1608                };
1609                let group_regs = vtype.vlmul().register_count();
1610                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1611                    program_counter,
1612                    vd,
1613                    group_regs,
1614                )?;
1615                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1616                    program_counter,
1617                    vs2,
1618                    group_regs,
1619                )?;
1620                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1621                    program_counter,
1622                    vs1,
1623                    group_regs,
1624                )?;
1625                if !vm && vd == VReg::V0 {
1626                    ::core::hint::cold_path();
1627                    return Err(ExecutionError::IllegalInstruction {
1628                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1629                    });
1630                }
1631                let sew = vtype.vsew();
1632                // SAFETY: alignment checked above
1633                unsafe {
1634                    zvexx_muldiv_helpers::execute_muladd_op(
1635                        ext_state,
1636                        vd,
1637                        vs1,
1638                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1639                        vm,
1640                        sew,
1641                        // vmadd: vd[i] = vs1[i] * vd[i] + vs2[i]; acc=vd, a=vs1, b=vs2
1642                        |acc, a, b, _| a.wrapping_mul(acc).wrapping_add(b),
1643                    );
1644                }
1645            }
1646            Self::VmaddVx {
1647                vd,
1648                rs1: _,
1649                vs2,
1650                vm,
1651            } => {
1652                if !ext_state.vector_instructions_allowed() {
1653                    ::core::hint::cold_path();
1654                    return Err(ExecutionError::IllegalInstruction {
1655                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1656                    });
1657                }
1658                let Some(vtype) = ext_state.vtype() else {
1659                    ::core::hint::cold_path();
1660                    return Err(ExecutionError::IllegalInstruction {
1661                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1662                    });
1663                };
1664                let group_regs = vtype.vlmul().register_count();
1665                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1666                    program_counter,
1667                    vd,
1668                    group_regs,
1669                )?;
1670                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1671                    program_counter,
1672                    vs2,
1673                    group_regs,
1674                )?;
1675                if !vm && vd == VReg::V0 {
1676                    ::core::hint::cold_path();
1677                    return Err(ExecutionError::IllegalInstruction {
1678                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1679                    });
1680                }
1681                let sew = vtype.vsew();
1682                let scalar = rs1_value.as_i64().cast_unsigned();
1683                // SAFETY: alignment checked above
1684                unsafe {
1685                    zvexx_muldiv_helpers::execute_muladd_scalar_op(
1686                        ext_state,
1687                        vd,
1688                        scalar,
1689                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1690                        vm,
1691                        sew,
1692                        // vmadd: vd[i] = rs1 * vd[i] + vs2[i]
1693                        |acc, a, b, _| a.wrapping_mul(acc).wrapping_add(b),
1694                    );
1695                }
1696            }
1697            // vnmsub.vv / vnmsub.vx - vd = -(vs1 * vd) + vs2
1698            Self::VnmsubVv { vd, vs1, vs2, vm } => {
1699                if !ext_state.vector_instructions_allowed() {
1700                    ::core::hint::cold_path();
1701                    return Err(ExecutionError::IllegalInstruction {
1702                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1703                    });
1704                }
1705                let Some(vtype) = ext_state.vtype() else {
1706                    ::core::hint::cold_path();
1707                    return Err(ExecutionError::IllegalInstruction {
1708                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1709                    });
1710                };
1711                let group_regs = vtype.vlmul().register_count();
1712                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1713                    program_counter,
1714                    vd,
1715                    group_regs,
1716                )?;
1717                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1718                    program_counter,
1719                    vs2,
1720                    group_regs,
1721                )?;
1722                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1723                    program_counter,
1724                    vs1,
1725                    group_regs,
1726                )?;
1727                if !vm && vd == VReg::V0 {
1728                    ::core::hint::cold_path();
1729                    return Err(ExecutionError::IllegalInstruction {
1730                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1731                    });
1732                }
1733                let sew = vtype.vsew();
1734                // SAFETY: alignment checked above
1735                unsafe {
1736                    zvexx_muldiv_helpers::execute_muladd_op(
1737                        ext_state,
1738                        vd,
1739                        vs1,
1740                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1741                        vm,
1742                        sew,
1743                        // vnmsub: vd[i] = -(vs1[i] * vd[i]) + vs2[i]; acc=vd, a=vs1, b=vs2
1744                        |acc, a, b, _| b.wrapping_sub(a.wrapping_mul(acc)),
1745                    );
1746                }
1747            }
1748            Self::VnmsubVx {
1749                vd,
1750                rs1: _,
1751                vs2,
1752                vm,
1753            } => {
1754                if !ext_state.vector_instructions_allowed() {
1755                    ::core::hint::cold_path();
1756                    return Err(ExecutionError::IllegalInstruction {
1757                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1758                    });
1759                }
1760                let Some(vtype) = ext_state.vtype() else {
1761                    ::core::hint::cold_path();
1762                    return Err(ExecutionError::IllegalInstruction {
1763                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1764                    });
1765                };
1766                let group_regs = vtype.vlmul().register_count();
1767                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1768                    program_counter,
1769                    vd,
1770                    group_regs,
1771                )?;
1772                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1773                    program_counter,
1774                    vs2,
1775                    group_regs,
1776                )?;
1777                if !vm && vd == VReg::V0 {
1778                    ::core::hint::cold_path();
1779                    return Err(ExecutionError::IllegalInstruction {
1780                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1781                    });
1782                }
1783                let sew = vtype.vsew();
1784                let scalar = rs1_value.as_i64().cast_unsigned();
1785                // SAFETY: alignment checked above
1786                unsafe {
1787                    zvexx_muldiv_helpers::execute_muladd_scalar_op(
1788                        ext_state,
1789                        vd,
1790                        scalar,
1791                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1792                        vm,
1793                        sew,
1794                        // vnmsub: vd[i] = -(rs1 * vd[i]) + vs2[i]
1795                        |acc, a, b, _| b.wrapping_sub(a.wrapping_mul(acc)),
1796                    );
1797                }
1798            }
1799            // vwmaccu.vv / vwmaccu.vx - unsigned widening multiply-add; illegal for SEW=64
1800            Self::VwmaccuVv { vd, vs1, vs2, vm } => {
1801                if !ext_state.vector_instructions_allowed() {
1802                    ::core::hint::cold_path();
1803                    return Err(ExecutionError::IllegalInstruction {
1804                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1805                    });
1806                }
1807                let Some(vtype) = ext_state.vtype() else {
1808                    ::core::hint::cold_path();
1809                    return Err(ExecutionError::IllegalInstruction {
1810                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1811                    });
1812                };
1813                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1814                    ::core::hint::cold_path();
1815                    return Err(ExecutionError::IllegalInstruction {
1816                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1817                    });
1818                }
1819                let group_regs = vtype.vlmul().register_count();
1820                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1821                    vtype.vlmul(),
1822                )
1823                .ok_or(ExecutionError::IllegalInstruction {
1824                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1825                })?;
1826                // vd holds the 2*SEW accumulator
1827                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1828                    program_counter,
1829                    vd,
1830                    dest_group_regs,
1831                )?;
1832                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1833                    program_counter,
1834                    vs2,
1835                    group_regs,
1836                )?;
1837                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1838                    program_counter,
1839                    vs1,
1840                    group_regs,
1841                )?;
1842                if !vm && vd == VReg::V0 {
1843                    ::core::hint::cold_path();
1844                    return Err(ExecutionError::IllegalInstruction {
1845                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1846                    });
1847                }
1848                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1849                    program_counter,
1850                    vd,
1851                    vs2,
1852                    dest_group_regs,
1853                    group_regs,
1854                )?;
1855                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1856                    program_counter,
1857                    vd,
1858                    vs1,
1859                    dest_group_regs,
1860                    group_regs,
1861                )?;
1862                let sew = vtype.vsew();
1863                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1864                unsafe {
1865                    zvexx_muldiv_helpers::execute_widening_muladd_op(
1866                        ext_state,
1867                        vd,
1868                        vs1,
1869                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1870                        vm,
1871                        sew,
1872                        // vwmaccu: vd[i] = vd[i] + zext(vs1[i]) * zext(vs2[i])
1873                        |acc, a, b, sew| {
1874                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
1875                            acc.wrapping_add((a & mask).wrapping_mul(b & mask))
1876                        },
1877                    );
1878                }
1879            }
1880            Self::VwmaccuVx {
1881                vd,
1882                rs1: _,
1883                vs2,
1884                vm,
1885            } => {
1886                if !ext_state.vector_instructions_allowed() {
1887                    ::core::hint::cold_path();
1888                    return Err(ExecutionError::IllegalInstruction {
1889                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1890                    });
1891                }
1892                let Some(vtype) = ext_state.vtype() else {
1893                    ::core::hint::cold_path();
1894                    return Err(ExecutionError::IllegalInstruction {
1895                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1896                    });
1897                };
1898                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1899                    ::core::hint::cold_path();
1900                    return Err(ExecutionError::IllegalInstruction {
1901                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1902                    });
1903                }
1904                let group_regs = vtype.vlmul().register_count();
1905                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1906                    vtype.vlmul(),
1907                )
1908                .ok_or(ExecutionError::IllegalInstruction {
1909                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1910                })?;
1911                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1912                    program_counter,
1913                    vd,
1914                    dest_group_regs,
1915                )?;
1916                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1917                    program_counter,
1918                    vs2,
1919                    group_regs,
1920                )?;
1921                if !vm && vd == VReg::V0 {
1922                    ::core::hint::cold_path();
1923                    return Err(ExecutionError::IllegalInstruction {
1924                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1925                    });
1926                }
1927                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
1928                    program_counter,
1929                    vd,
1930                    vs2,
1931                    dest_group_regs,
1932                    group_regs,
1933                )?;
1934                let sew = vtype.vsew();
1935                let scalar = rs1_value.as_u64();
1936                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
1937                unsafe {
1938                    zvexx_muldiv_helpers::execute_widening_muladd_scalar_op(
1939                        ext_state,
1940                        vd,
1941                        scalar,
1942                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
1943                        vm,
1944                        sew,
1945                        |acc, a, b, sew| {
1946                            let mask = zvexx_muldiv_helpers::sew_mask(sew);
1947                            acc.wrapping_add((a & mask).wrapping_mul(b & mask))
1948                        },
1949                    );
1950                }
1951            }
1952            // vwmacc.vv / vwmacc.vx - signed widening multiply-add; illegal for SEW=64
1953            Self::VwmaccVv { vd, vs1, vs2, vm } => {
1954                if !ext_state.vector_instructions_allowed() {
1955                    ::core::hint::cold_path();
1956                    return Err(ExecutionError::IllegalInstruction {
1957                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1958                    });
1959                }
1960                let Some(vtype) = ext_state.vtype() else {
1961                    ::core::hint::cold_path();
1962                    return Err(ExecutionError::IllegalInstruction {
1963                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1964                    });
1965                };
1966                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
1967                    ::core::hint::cold_path();
1968                    return Err(ExecutionError::IllegalInstruction {
1969                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1970                    });
1971                }
1972                let group_regs = vtype.vlmul().register_count();
1973                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
1974                    vtype.vlmul(),
1975                )
1976                .ok_or(ExecutionError::IllegalInstruction {
1977                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1978                })?;
1979                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1980                    program_counter,
1981                    vd,
1982                    dest_group_regs,
1983                )?;
1984                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1985                    program_counter,
1986                    vs2,
1987                    group_regs,
1988                )?;
1989                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
1990                    program_counter,
1991                    vs1,
1992                    group_regs,
1993                )?;
1994                if !vm && vd == VReg::V0 {
1995                    ::core::hint::cold_path();
1996                    return Err(ExecutionError::IllegalInstruction {
1997                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
1998                    });
1999                }
2000                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2001                    program_counter,
2002                    vd,
2003                    vs2,
2004                    dest_group_regs,
2005                    group_regs,
2006                )?;
2007                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2008                    program_counter,
2009                    vd,
2010                    vs1,
2011                    dest_group_regs,
2012                    group_regs,
2013                )?;
2014                let sew = vtype.vsew();
2015                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
2016                unsafe {
2017                    zvexx_muldiv_helpers::execute_widening_muladd_op(
2018                        ext_state,
2019                        vd,
2020                        vs1,
2021                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
2022                        vm,
2023                        sew,
2024                        // vwmacc: vd[i] = vd[i] + sext(vs1[i]) * sext(vs2[i])
2025                        |acc, a, b, sew| {
2026                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
2027                            let sb = zvexx_muldiv_helpers::sign_extend(b, sew);
2028                            acc.wrapping_add(sa.cast_unsigned().wrapping_mul(sb.cast_unsigned()))
2029                        },
2030                    );
2031                }
2032            }
2033            Self::VwmaccVx {
2034                vd,
2035                rs1: _,
2036                vs2,
2037                vm,
2038            } => {
2039                if !ext_state.vector_instructions_allowed() {
2040                    ::core::hint::cold_path();
2041                    return Err(ExecutionError::IllegalInstruction {
2042                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2043                    });
2044                }
2045                let Some(vtype) = ext_state.vtype() else {
2046                    ::core::hint::cold_path();
2047                    return Err(ExecutionError::IllegalInstruction {
2048                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2049                    });
2050                };
2051                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
2052                    ::core::hint::cold_path();
2053                    return Err(ExecutionError::IllegalInstruction {
2054                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2055                    });
2056                }
2057                let group_regs = vtype.vlmul().register_count();
2058                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
2059                    vtype.vlmul(),
2060                )
2061                .ok_or(ExecutionError::IllegalInstruction {
2062                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2063                })?;
2064                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2065                    program_counter,
2066                    vd,
2067                    dest_group_regs,
2068                )?;
2069                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2070                    program_counter,
2071                    vs2,
2072                    group_regs,
2073                )?;
2074                if !vm && vd == VReg::V0 {
2075                    ::core::hint::cold_path();
2076                    return Err(ExecutionError::IllegalInstruction {
2077                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2078                    });
2079                }
2080                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2081                    program_counter,
2082                    vd,
2083                    vs2,
2084                    dest_group_regs,
2085                    group_regs,
2086                )?;
2087                let sew = vtype.vsew();
2088                let scalar = rs1_value.as_u64();
2089                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
2090                unsafe {
2091                    zvexx_muldiv_helpers::execute_widening_muladd_scalar_op(
2092                        ext_state,
2093                        vd,
2094                        scalar,
2095                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
2096                        vm,
2097                        sew,
2098                        |acc, a, b, sew| {
2099                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
2100                            let sb = zvexx_muldiv_helpers::sign_extend(b, sew);
2101                            acc.wrapping_add(sa.cast_unsigned().wrapping_mul(sb.cast_unsigned()))
2102                        },
2103                    );
2104                }
2105            }
2106            // vwmaccsu.vv / vwmaccsu.vx - signed×unsigned widening multiply-add; illegal for SEW=64
2107            Self::VwmaccsuVv { vd, vs1, vs2, vm } => {
2108                if !ext_state.vector_instructions_allowed() {
2109                    ::core::hint::cold_path();
2110                    return Err(ExecutionError::IllegalInstruction {
2111                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2112                    });
2113                }
2114                let Some(vtype) = ext_state.vtype() else {
2115                    ::core::hint::cold_path();
2116                    return Err(ExecutionError::IllegalInstruction {
2117                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2118                    });
2119                };
2120                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
2121                    ::core::hint::cold_path();
2122                    return Err(ExecutionError::IllegalInstruction {
2123                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2124                    });
2125                }
2126                let group_regs = vtype.vlmul().register_count();
2127                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
2128                    vtype.vlmul(),
2129                )
2130                .ok_or(ExecutionError::IllegalInstruction {
2131                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2132                })?;
2133                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2134                    program_counter,
2135                    vd,
2136                    dest_group_regs,
2137                )?;
2138                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2139                    program_counter,
2140                    vs2,
2141                    group_regs,
2142                )?;
2143                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2144                    program_counter,
2145                    vs1,
2146                    group_regs,
2147                )?;
2148                if !vm && vd == VReg::V0 {
2149                    ::core::hint::cold_path();
2150                    return Err(ExecutionError::IllegalInstruction {
2151                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2152                    });
2153                }
2154                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2155                    program_counter,
2156                    vd,
2157                    vs2,
2158                    dest_group_regs,
2159                    group_regs,
2160                )?;
2161                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2162                    program_counter,
2163                    vd,
2164                    vs1,
2165                    dest_group_regs,
2166                    group_regs,
2167                )?;
2168                let sew = vtype.vsew();
2169                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
2170                unsafe {
2171                    zvexx_muldiv_helpers::execute_widening_muladd_op(
2172                        ext_state,
2173                        vd,
2174                        vs1,
2175                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
2176                        vm,
2177                        sew,
2178                        // vwmaccsu: vd[i] = vd[i] + sext(vs1[i]) * zext(vs2[i])
2179                        |acc, a, b, sew| {
2180                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
2181                            let ub = b & zvexx_muldiv_helpers::sew_mask(sew);
2182                            acc.wrapping_add(sa.cast_unsigned().wrapping_mul(ub))
2183                        },
2184                    );
2185                }
2186            }
2187            Self::VwmaccsuVx {
2188                vd,
2189                rs1: _,
2190                vs2,
2191                vm,
2192            } => {
2193                if !ext_state.vector_instructions_allowed() {
2194                    ::core::hint::cold_path();
2195                    return Err(ExecutionError::IllegalInstruction {
2196                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2197                    });
2198                }
2199                let Some(vtype) = ext_state.vtype() else {
2200                    ::core::hint::cold_path();
2201                    return Err(ExecutionError::IllegalInstruction {
2202                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2203                    });
2204                };
2205                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
2206                    ::core::hint::cold_path();
2207                    return Err(ExecutionError::IllegalInstruction {
2208                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2209                    });
2210                }
2211                let group_regs = vtype.vlmul().register_count();
2212                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
2213                    vtype.vlmul(),
2214                )
2215                .ok_or(ExecutionError::IllegalInstruction {
2216                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2217                })?;
2218                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2219                    program_counter,
2220                    vd,
2221                    dest_group_regs,
2222                )?;
2223                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2224                    program_counter,
2225                    vs2,
2226                    group_regs,
2227                )?;
2228                if !vm && vd == VReg::V0 {
2229                    ::core::hint::cold_path();
2230                    return Err(ExecutionError::IllegalInstruction {
2231                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2232                    });
2233                }
2234                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2235                    program_counter,
2236                    vd,
2237                    vs2,
2238                    dest_group_regs,
2239                    group_regs,
2240                )?;
2241                let sew = vtype.vsew();
2242                // scalar (rs1) is the signed operand; vs2 elements are unsigned
2243                let scalar = rs1_value.as_u64();
2244                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
2245                unsafe {
2246                    zvexx_muldiv_helpers::execute_widening_muladd_scalar_op(
2247                        ext_state,
2248                        vd,
2249                        scalar,
2250                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
2251                        vm,
2252                        sew,
2253                        // vwmaccsu.vx: vd[i] = vd[i] + sext(rs1) * zext(vs2[i])
2254                        // Helper passes (acc, scalar_as_a, vs2_as_b, sew): a=rs1 (signed),
2255                        // b=vs2 (unsigned)
2256                        |acc, a, b, sew| {
2257                            let sa = zvexx_muldiv_helpers::sign_extend(a, sew);
2258                            let ub = b & zvexx_muldiv_helpers::sew_mask(sew);
2259                            acc.wrapping_add(sa.cast_unsigned().wrapping_mul(ub))
2260                        },
2261                    );
2262                }
2263            }
2264            // vwmaccus.vx - unsigned×signed widening multiply-add (vx only); illegal for SEW=64
2265            Self::VwmaccusVx {
2266                vd,
2267                rs1: _,
2268                vs2,
2269                vm,
2270            } => {
2271                if !ext_state.vector_instructions_allowed() {
2272                    ::core::hint::cold_path();
2273                    return Err(ExecutionError::IllegalInstruction {
2274                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2275                    });
2276                }
2277                let Some(vtype) = ext_state.vtype() else {
2278                    ::core::hint::cold_path();
2279                    return Err(ExecutionError::IllegalInstruction {
2280                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2281                    });
2282                };
2283                if u32::from(vtype.vsew().bits_width()) == u64::BITS {
2284                    ::core::hint::cold_path();
2285                    return Err(ExecutionError::IllegalInstruction {
2286                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2287                    });
2288                }
2289                let group_regs = vtype.vlmul().register_count();
2290                let dest_group_regs = zvexx_muldiv_helpers::widening_dest_register_count(
2291                    vtype.vlmul(),
2292                )
2293                .ok_or(ExecutionError::IllegalInstruction {
2294                    address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2295                })?;
2296                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2297                    program_counter,
2298                    vd,
2299                    dest_group_regs,
2300                )?;
2301                zvexx_muldiv_helpers::check_vreg_group_alignment::<Reg, _, _, _>(
2302                    program_counter,
2303                    vs2,
2304                    group_regs,
2305                )?;
2306                if !vm && vd == VReg::V0 {
2307                    ::core::hint::cold_path();
2308                    return Err(ExecutionError::IllegalInstruction {
2309                        address: program_counter.old_pc(zvexx_helpers::INSTRUCTION_SIZE),
2310                    });
2311                }
2312                zvexx_muldiv_helpers::check_no_widening_overlap::<Reg, _, _, _>(
2313                    program_counter,
2314                    vd,
2315                    vs2,
2316                    dest_group_regs,
2317                    group_regs,
2318                )?;
2319                let sew = vtype.vsew();
2320                // scalar (rs1) is the unsigned operand; vs2 elements are signed
2321                let scalar = rs1_value.as_u64();
2322                // SAFETY: alignment and overlap checked above; SEW < 64 checked above
2323                unsafe {
2324                    zvexx_muldiv_helpers::execute_widening_muladd_scalar_op(
2325                        ext_state,
2326                        vd,
2327                        scalar,
2328                        zvexx_muldiv_helpers::OpSrc::Vreg(vs2),
2329                        vm,
2330                        sew,
2331                        // vwmaccus.vx: vd[i] = vd[i] + zext(rs1) * sext(vs2[i])
2332                        // Helper passes (acc, scalar_as_a, vs2_as_b, sew): a=rs1 (unsigned),
2333                        // b=vs2 (signed)
2334                        |acc, a, b, sew| {
2335                            let ua = a & zvexx_muldiv_helpers::sew_mask(sew);
2336                            let sb = zvexx_muldiv_helpers::sign_extend(b, sew);
2337                            acc.wrapping_add(sb.cast_unsigned().wrapping_mul(ua))
2338                        },
2339                    );
2340                }
2341            }
2342        }
2343
2344        Ok(ControlFlow::Continue(Default::default()))
2345    }
2346}