1use crate::v::vector_registers::{VectorRegisterFile, VectorRegistersExt};
4pub use crate::v::zvexx::arith::zvexx_arith_helpers::{
5 OpSrc, check_vreg_group_alignment, sew_mask, sign_extend,
6};
7use crate::v::zvexx::arith::zvexx_arith_helpers::{read_element_u64, write_element_u64};
8use crate::v::zvexx::fixed_point::zvexx_fixed_point_helpers::read_wide_element_u64;
9use crate::v::zvexx::load::zvexx_load_helpers::{mask_bit, snapshot_mask};
10use crate::v::zvexx::zvexx_helpers::INSTRUCTION_SIZE;
11use crate::{ExecutionError, ProgramCounter};
12use ab_riscv_primitives::prelude::*;
13use core::fmt;
14use core::hint::cold_path;
15use core::num::NonZeroU8;
16
17#[inline(always)]
26#[doc(hidden)]
27#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
28pub fn widening_dest_register_count(vlmul: Vlmul) -> Option<NonZeroU8> {
29 let (lmul_num, lmul_den) = vlmul.as_fraction();
30 let Some(emul_num) = 2u8.checked_mul(lmul_num.get()) else {
32 cold_path();
33 return None;
34 };
35 let emul_den = lmul_den.get();
36 let g = emul_num.min(emul_den);
38 let (n, d) = (emul_num / g, emul_den / g);
39 let legal = matches!(
41 (n, d),
42 (1, 8) | (1, 4) | (1, 2) | (1, 1) | (2, 1) | (4, 1) | (8, 1)
43 );
44 if !legal {
45 cold_path();
46 return None;
47 }
48 Some(NonZeroU8::new(if d > 1 { 1 } else { n }).expect("Not zero; qed"))
50}
51
52#[inline(always)]
71#[doc(hidden)]
72#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
73pub fn check_no_widening_overlap<Reg, Memory, PC, CustomError>(
74 program_counter: &PC,
75 vd: VReg,
76 vs: VReg,
77 dest_group_regs: NonZeroU8,
78 src_group_regs: NonZeroU8,
79) -> Result<(), ExecutionError<Reg::Type, CustomError>>
80where
81 Reg: Register,
82 PC: ProgramCounter<Reg::Type, Memory, CustomError>,
83{
84 let dest_group_regs = dest_group_regs.get();
85 let src_group_regs = src_group_regs.get();
86 let vd_start = vd.to_bits();
87 let vd_end = vd_start + dest_group_regs;
88 let vs_start = vs.to_bits();
89 let vs_end = vs_start + src_group_regs;
90 if vs_start >= vd_end || vd_start >= vs_end {
92 return Ok(());
93 }
94 if dest_group_regs > src_group_regs && vs_start == vd_end - src_group_regs {
98 return Ok(());
99 }
100
101 cold_path();
102 Err(ExecutionError::IllegalInstruction {
103 address: program_counter.old_pc(INSTRUCTION_SIZE),
104 })
105}
106
107#[inline(always)]
113#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
114unsafe fn write_wide_element_u64<const VLEN: Vlen>(
115 vregs: &mut VectorRegisterFile<VLEN>,
116 base_reg: VReg,
117 elem_i: u16,
118 sew: Vsew,
119 value: u64,
120) {
121 let wide_bytes = u32::from(sew.bytes_width()) * 2;
122 let elems_per_reg = VLEN.bytes() / wide_bytes;
123 let reg_off = u32::from(elem_i) / elems_per_reg;
124 let byte_off = (u32::from(elem_i) % elems_per_reg) * wide_bytes;
125 let buf = value.to_le_bytes();
126 let reg = unsafe {
128 vregs.get_mut(VReg::from_bits(base_reg.to_bits() + reg_off as u8).unwrap_unchecked())
129 };
130 let dst = unsafe { reg.get_unchecked_mut(byte_off as usize..(byte_off + wide_bytes) as usize) };
132 dst.copy_from_slice(unsafe { buf.get_unchecked(..wide_bytes as usize) });
134}
135
136#[inline(always)]
146#[doc(hidden)]
147pub unsafe fn execute_arith_op<Reg, ExtState, CustomError, F>(
149 ext_state: &mut ExtState,
150 vd: VReg,
151 vs2: VReg,
152 src: OpSrc,
153 vm: bool,
154 sew: Vsew,
155 op: F,
156) where
157 Reg: Register,
158 ExtState: VectorRegistersExt<Reg, CustomError>,
159 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
160 CustomError: fmt::Debug,
161 F: Fn(u64, u64, Vsew) -> u64,
162{
163 let vl = ext_state.vl();
164 let vstart = ext_state.vstart();
165 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
167 for i in vstart.range_to(vl) {
168 if !mask_bit(&mask_buf, i) {
169 continue;
170 }
171 let a = unsafe { read_element_u64(ext_state.read_vregs(), vs2, i, sew) };
173 let b = match src {
174 OpSrc::Vreg(vs1_base) => unsafe {
176 read_element_u64(ext_state.read_vregs(), vs1_base, i, sew)
177 },
178 OpSrc::Scalar(val) => val,
179 };
180 let result = op(a, b, sew);
181 unsafe {
183 write_element_u64(ext_state.write_vregs(), vd, i, sew, result);
184 }
185 }
186 ext_state.mark_vs_dirty();
187 ext_state.reset_vstart();
188}
189
190#[inline(always)]
202#[doc(hidden)]
203pub unsafe fn execute_widening_op<Reg, ExtState, CustomError, F>(
205 ext_state: &mut ExtState,
206 vd: VReg,
207 vs2: VReg,
208 src: OpSrc,
209 vm: bool,
210 sew: Vsew,
211 op: F,
212) where
213 Reg: Register,
214 ExtState: VectorRegistersExt<Reg, CustomError>,
215 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
216 CustomError: fmt::Debug,
217 F: Fn(u64, u64, Vsew) -> u64,
218{
219 let vl = ext_state.vl();
220 let vstart = ext_state.vstart();
221 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
223 for i in vstart.range_to(vl) {
224 if !mask_bit(&mask_buf, i) {
225 continue;
226 }
227 let a = unsafe { read_element_u64(ext_state.read_vregs(), vs2, i, sew) };
229 let b = match src {
230 OpSrc::Vreg(vs1_base) => unsafe {
232 read_element_u64(ext_state.read_vregs(), vs1_base, i, sew)
233 },
234 OpSrc::Scalar(val) => val,
235 };
236 let result = op(a, b, sew);
237 unsafe {
241 write_wide_element_u64(ext_state.write_vregs(), vd, i, sew, result);
242 }
243 }
244 ext_state.mark_vs_dirty();
245 ext_state.reset_vstart();
246}
247
248#[inline(always)]
258#[doc(hidden)]
259pub unsafe fn execute_muladd_op<Reg, ExtState, CustomError, F>(
261 ext_state: &mut ExtState,
262 vd: VReg,
263 a_reg: VReg,
264 src: OpSrc,
265 vm: bool,
266 sew: Vsew,
267 op: F,
268) where
269 Reg: Register,
270 ExtState: VectorRegistersExt<Reg, CustomError>,
271 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
272 CustomError: fmt::Debug,
273 F: Fn(u64, u64, u64, Vsew) -> u64,
274{
275 let vl = ext_state.vl();
276 let vstart = ext_state.vstart();
277 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
279 for i in vstart.range_to(vl) {
280 if !mask_bit(&mask_buf, i) {
281 continue;
282 }
283 let acc = unsafe { read_element_u64(ext_state.read_vregs(), vd, i, sew) };
285 let a = unsafe { read_element_u64(ext_state.read_vregs(), a_reg, i, sew) };
287 let b = match src {
288 OpSrc::Vreg(b_reg) => unsafe {
290 read_element_u64(ext_state.read_vregs(), b_reg, i, sew)
291 },
292 OpSrc::Scalar(val) => val,
293 };
294 let result = op(acc, a, b, sew);
295 unsafe {
297 write_element_u64(ext_state.write_vregs(), vd, i, sew, result);
298 }
299 }
300 ext_state.mark_vs_dirty();
301 ext_state.reset_vstart();
302}
303
304#[inline(always)]
311#[doc(hidden)]
312pub unsafe fn execute_muladd_scalar_op<Reg, ExtState, CustomError, F>(
314 ext_state: &mut ExtState,
315 vd: VReg,
316 scalar: u64,
317 src: OpSrc,
318 vm: bool,
319 sew: Vsew,
320 op: F,
321) where
322 Reg: Register,
323 ExtState: VectorRegistersExt<Reg, CustomError>,
324 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
325 CustomError: fmt::Debug,
326 F: Fn(u64, u64, u64, Vsew) -> u64,
327{
328 let vl = ext_state.vl();
329 let vstart = ext_state.vstart();
330 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
332 for i in vstart.range_to(vl) {
333 if !mask_bit(&mask_buf, i) {
334 continue;
335 }
336 let acc = unsafe { read_element_u64(ext_state.read_vregs(), vd, i, sew) };
338 let b = match src {
339 OpSrc::Vreg(b_reg) => unsafe {
341 read_element_u64(ext_state.read_vregs(), b_reg, i, sew)
342 },
343 OpSrc::Scalar(val) => val,
344 };
345 let result = op(acc, scalar, b, sew);
346 unsafe {
348 write_element_u64(ext_state.write_vregs(), vd, i, sew, result);
349 }
350 }
351 ext_state.mark_vs_dirty();
352 ext_state.reset_vstart();
353}
354
355#[inline(always)]
368#[doc(hidden)]
369pub unsafe fn execute_widening_muladd_op<Reg, ExtState, CustomError, F>(
371 ext_state: &mut ExtState,
372 vd: VReg,
373 a_reg: VReg,
374 src: OpSrc,
375 vm: bool,
376 sew: Vsew,
377 op: F,
378) where
379 Reg: Register,
380 ExtState: VectorRegistersExt<Reg, CustomError>,
381 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
382 CustomError: fmt::Debug,
383 F: Fn(u64, u64, u64, Vsew) -> u64,
384{
385 let vl = ext_state.vl();
386 let vstart = ext_state.vstart();
387 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
389 for i in vstart.range_to(vl) {
390 if !mask_bit(&mask_buf, i) {
391 continue;
392 }
393 let acc = unsafe { read_wide_element_u64(ext_state.read_vregs(), vd, i, sew) };
397 let a = unsafe { read_element_u64(ext_state.read_vregs(), a_reg, i, sew) };
399 let b = match src {
400 OpSrc::Vreg(b_reg) => unsafe {
402 read_element_u64(ext_state.read_vregs(), b_reg, i, sew)
403 },
404 OpSrc::Scalar(val) => val,
405 };
406 let result = op(acc, a, b, sew);
407 unsafe {
409 write_wide_element_u64(ext_state.write_vregs(), vd, i, sew, result);
410 }
411 }
412 ext_state.mark_vs_dirty();
413 ext_state.reset_vstart();
414}
415
416#[inline(always)]
423#[doc(hidden)]
424pub unsafe fn execute_widening_muladd_scalar_op<Reg, ExtState, CustomError, F>(
426 ext_state: &mut ExtState,
427 vd: VReg,
428 scalar: u64,
429 src: OpSrc,
430 vm: bool,
431 sew: Vsew,
432 op: F,
433) where
434 Reg: Register,
435 ExtState: VectorRegistersExt<Reg, CustomError>,
436 [(); SUPPORTED_ELEN_VLEN::<{ ExtState::ELEN }, { ExtState::VLEN }>]:,
437 CustomError: fmt::Debug,
438 F: Fn(u64, u64, u64, Vsew) -> u64,
439{
440 let vl = ext_state.vl();
441 let vstart = ext_state.vstart();
442 let mask_buf = unsafe { snapshot_mask(ext_state.read_vregs(), vm, vl) };
444 for i in vstart.range_to(vl) {
445 if !mask_bit(&mask_buf, i) {
446 continue;
447 }
448 let acc = unsafe { read_wide_element_u64(ext_state.read_vregs(), vd, i, sew) };
451 let b = match src {
452 OpSrc::Vreg(b_reg) => unsafe {
454 read_element_u64(ext_state.read_vregs(), b_reg, i, sew)
455 },
456 OpSrc::Scalar(val) => val,
457 };
458 let result = op(acc, scalar, b, sew);
459 unsafe {
461 write_wide_element_u64(ext_state.write_vregs(), vd, i, sew, result);
462 }
463 }
464 ext_state.mark_vs_dirty();
465 ext_state.reset_vstart();
466}
467
468#[inline(always)]
473#[doc(hidden)]
474#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
475pub fn mulh_ss(a: u64, b: u64, sew: Vsew) -> u64 {
476 let sa = i128::from(sign_extend(a, sew));
477 let sb = i128::from(sign_extend(b, sew));
478 let product = sa.wrapping_mul(sb);
479 let high = (product >> u32::from(sew.bits_width())).cast_unsigned() as u64;
481 high & sew_mask(sew)
482}
483
484#[inline(always)]
486#[doc(hidden)]
487#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
488pub fn mulhu_uu(a: u64, b: u64, sew: Vsew) -> u64 {
489 let ua = u128::from(a & sew_mask(sew));
490 let ub = u128::from(b & sew_mask(sew));
491 let product = ua.wrapping_mul(ub);
492 let high = (product >> u32::from(sew.bits_width())) as u64;
493 high & sew_mask(sew)
494}
495
496#[inline(always)]
500#[doc(hidden)]
501#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
502pub fn mulhsu_su(a: u64, b: u64, sew: Vsew) -> u64 {
503 let sa = i128::from(sign_extend(a, sew));
504 let ub = u128::from(b & sew_mask(sew));
505 let product = sa.wrapping_mul(ub.cast_signed());
507 let high = (product >> u32::from(sew.bits_width())).cast_unsigned() as u64;
508 high & sew_mask(sew)
509}
510
511#[inline(always)]
516#[doc(hidden)]
517pub fn sdiv(a: u64, b: u64, sew: Vsew) -> u64 {
519 let sa = sign_extend(a, sew);
520 let sb = sign_extend(b, sew);
521 if sb == 0 {
523 return sew_mask(sew);
524 }
525 let sew_min = i64::MIN >> (u64::BITS - u32::from(sew.bits_width()));
527 if sa == sew_min && sb == -1 {
528 return sew_min.cast_unsigned() & sew_mask(sew);
529 }
530 (sa / sb).cast_unsigned() & sew_mask(sew)
531}
532
533#[inline(always)]
539#[doc(hidden)]
540#[expect(
541 clippy::modulo_arithmetic,
542 reason = "This is what the code is supposed to do"
543)]
544pub fn srem(a: u64, b: u64, sew: Vsew) -> u64 {
546 let sa = sign_extend(a, sew);
547 let sb = sign_extend(b, sew);
548 if sb == 0 {
550 return a & sew_mask(sew);
551 }
552 let sew_min = i64::MIN >> (u64::BITS - u32::from(sew.bits_width()));
554 if sa == sew_min && sb == -1 {
555 return 0;
556 }
557 (sa % sb).cast_unsigned() & sew_mask(sew)
558}