1use crate::v::vector_registers::{VRegGroup, VectorRegistersExt};
4use crate::v::zvexx::arith::zvexx_arith_helpers::sign_extend;
5pub use crate::v::zvexx::arith::zvexx_arith_helpers::{OpSrc, sew_mask};
6use crate::v::zvexx::load::zvexx_load_helpers::{mask_bit, snapshot_mask};
7use crate::v::zvexx::zvexx_helpers::{INSTRUCTION_SIZE, WideningSew};
8use crate::{ExecutionError, PackedAddress, ProgramCounter};
9use ab_riscv_primitives::prelude::*;
10use core::hint::cold_path;
11
12#[inline(always)]
17#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
18fn round_increment(val: u64, shift: u32, mode: Vxrm, current_result_lsb: u64) -> u64 {
19 if shift == 0 {
20 return 0;
21 }
22 let d_minus1_bit = (val >> (shift - 1)) & 1;
24 let sticky = if shift >= 2 {
26 (val & ((1u64 << (shift - 1)).wrapping_sub(1))) != 0
28 } else {
29 false
30 };
31 match mode {
32 Vxrm::Rnu => d_minus1_bit,
34 Vxrm::Rne => d_minus1_bit & u64::from(sticky || current_result_lsb != 0),
36 Vxrm::Rdn => 0,
38 Vxrm::Rod => u64::from(current_result_lsb == 0 && (d_minus1_bit != 0 || sticky)),
40 }
41}
42
43#[inline(always)]
47#[doc(hidden)]
48#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
49pub fn rounded_srl(val: u64, shift: u32, mode: Vxrm) -> u64 {
50 let truncated = val >> shift;
51 let r = round_increment(val, shift, mode, truncated & 1);
52 truncated.wrapping_add(r)
53}
54
55#[inline(always)]
59#[doc(hidden)]
60#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
61pub fn rounded_sra(val: u64, shift: u32, mode: Vxrm, sew: Vsew) -> u64 {
62 let signed = sign_extend(val, sew);
63 let truncated_signed = signed >> shift;
67 let r = round_increment(val, shift, mode, truncated_signed.cast_unsigned() & 1);
68 truncated_signed.cast_unsigned().wrapping_add(r)
69}
70
71#[inline(always)]
75#[doc(hidden)]
76#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
77pub fn sat_addu(a: u64, b: u64, sew: Vsew, vxsat: &mut bool) -> u64 {
78 let mask = sew_mask(sew);
79 let a_w = a & mask;
80 let b_w = b & mask;
81 let result = a_w.wrapping_add(b_w);
82 if result & mask < a_w {
83 *vxsat = true;
85 mask
86 } else {
87 result & mask
88 }
89}
90
91#[inline(always)]
95#[doc(hidden)]
96#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
97pub fn sat_add(a: u64, b: u64, sew: Vsew, vxsat: &mut bool) -> u64 {
98 let sa = i128::from(sign_extend(a, sew));
99 let sb = i128::from(sign_extend(b, sew));
100 let result = sa.wrapping_add(sb);
101 let min_val = i128::MIN >> (i128::BITS - u32::from(sew.bits_width()));
102 let max_val = i128::MAX >> (i128::BITS - u32::from(sew.bits_width()));
103 if result < min_val {
104 *vxsat = true;
105 (min_val as i64).cast_unsigned() & sew_mask(sew)
106 } else if result > max_val {
107 *vxsat = true;
108 (max_val as i64).cast_unsigned() & sew_mask(sew)
109 } else {
110 (result as i64).cast_unsigned() & sew_mask(sew)
111 }
112}
113
114#[inline(always)]
118#[doc(hidden)]
119#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
120pub fn sat_subu(a: u64, b: u64, sew: Vsew, vxsat: &mut bool) -> u64 {
121 let mask = sew_mask(sew);
122 let a_w = a & mask;
123 let b_w = b & mask;
124 if a_w < b_w {
125 *vxsat = true;
126 0
127 } else {
128 (a_w - b_w) & mask
129 }
130}
131
132#[inline(always)]
136#[doc(hidden)]
137#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
138pub fn sat_sub(a: u64, b: u64, sew: Vsew, vxsat: &mut bool) -> u64 {
139 let sa = i128::from(sign_extend(a, sew));
140 let sb = i128::from(sign_extend(b, sew));
141 let result = sa.wrapping_sub(sb);
142 let min_val = i128::MIN >> (i128::BITS - u32::from(sew.bits_width()));
143 let max_val = i128::MAX >> (i128::BITS - u32::from(sew.bits_width()));
144 if result < min_val {
145 *vxsat = true;
146 (min_val as i64).cast_unsigned() & sew_mask(sew)
147 } else if result > max_val {
148 *vxsat = true;
149 (max_val as i64).cast_unsigned() & sew_mask(sew)
150 } else {
151 (result as i64).cast_unsigned() & sew_mask(sew)
152 }
153}
154
155#[inline(always)]
159#[doc(hidden)]
160#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
161pub fn avg_addu(a: u64, b: u64, sew: Vsew, mode: Vxrm) -> u64 {
162 let mask = sew_mask(sew);
163 let a_w = a & mask;
164 let b_w = b & mask;
165 let sum = a_w.wrapping_add(b_w);
169 let carry = u64::from(sum & mask < a_w);
171 let r = round_increment(sum & mask, 1, mode, (sum >> 1u8) & 1);
175 let shifted = (carry << (u32::from(sew.bits_width()) - 1)) | ((sum & mask) >> 1u8);
177 (shifted.wrapping_add(r)) & mask
178}
179
180#[inline(always)]
184#[doc(hidden)]
185#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
186pub fn avg_add(a: u64, b: u64, sew: Vsew, mode: Vxrm) -> u64 {
187 let sa = sign_extend(a, sew);
188 let sb = sign_extend(b, sew);
189 let sum = i128::from(sa).wrapping_add(i128::from(sb));
191 let r = match mode {
193 Vxrm::Rnu => (sum & 1).cast_unsigned() as u64,
194 Vxrm::Rne => {
195 let result_lsb = ((sum >> 1u8) & 1).cast_unsigned() as u64;
198 ((sum & 1).cast_unsigned() as u64) & result_lsb
199 }
200 Vxrm::Rdn => 0,
201 Vxrm::Rod => {
202 let result_lsb = (sum >> 1u8) & 1;
204 u64::from(result_lsb == 0 && (sum & 1) != 0)
205 }
206 };
207 let result = (sum >> 1u8) + i128::from(r);
208 (result as i64).cast_unsigned() & sew_mask(sew)
209}
210
211#[inline(always)]
215#[doc(hidden)]
216#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
217pub fn avg_subu(a: u64, b: u64, sew: Vsew, mode: Vxrm) -> u64 {
218 let mask = sew_mask(sew);
219 let a_w = a & mask;
220 let b_w = b & mask;
221 let diff = a_w.wrapping_sub(b_w);
223 let borrow = u64::from(a_w < b_w);
225 let r = round_increment(diff & mask, 1, mode, (diff >> 1u8) & 1);
229 let sign_fill = borrow.wrapping_neg(); let shifted = (sign_fill << (u32::from(sew.bits_width()) - 1)) | ((diff & mask) >> 1u8);
236 (shifted.wrapping_add(r)) & mask
237}
238
239#[inline(always)]
243#[doc(hidden)]
244#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
245pub fn avg_sub(a: u64, b: u64, sew: Vsew, mode: Vxrm) -> u64 {
246 let sa = sign_extend(a, sew);
247 let sb = sign_extend(b, sew);
248 let diff = i128::from(sa).wrapping_sub(i128::from(sb));
249 let r = match mode {
250 Vxrm::Rnu => (diff & 1).cast_unsigned() as u64,
251 Vxrm::Rne => {
252 let result_lsb = ((diff >> 1u8) & 1).cast_unsigned() as u64;
253 ((diff & 1).cast_unsigned() as u64) & result_lsb
254 }
255 Vxrm::Rdn => 0,
256 Vxrm::Rod => {
257 let result_lsb = (diff >> 1u8) & 1;
258 u64::from(result_lsb == 0 && (diff & 1) != 0)
259 }
260 };
261 let result = (diff >> 1u8) + i128::from(r);
262 (result as i64).cast_unsigned() & sew_mask(sew)
263}
264
265#[inline(always)]
273#[doc(hidden)]
274#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
275pub fn smul(a: u64, b: u64, sew: Vsew, mode: Vxrm, vxsat: &mut bool) -> u64 {
276 let min_sew = i64::MIN >> (i64::BITS - u32::from(sew.bits_width()));
278 let max_sew = i64::MAX >> (i64::BITS - u32::from(sew.bits_width()));
279 let sa = i128::from(sign_extend(a, sew));
280 let sb = i128::from(sign_extend(b, sew));
281 if sa == i128::from(min_sew) && sb == i128::from(min_sew) {
284 cold_path();
285 *vxsat = true;
286 return max_sew.cast_unsigned() & sew_mask(sew);
287 }
288 let product = sa * sb;
291 let doubled = product << 1u8;
294 let shift = u32::from(sew.bits_width());
297 let low_bits = (doubled.cast_unsigned() & u128::from(sew_mask(sew))) as u64;
298 let truncated = doubled >> shift;
300 let r = round_increment(
301 low_bits,
302 shift.min(64),
303 mode,
304 (truncated.cast_unsigned() as u64) & 1,
305 );
306 let result = (truncated as i64).wrapping_add(r.cast_signed());
308 if result < min_sew {
310 *vxsat = true;
311 min_sew.cast_unsigned() & sew_mask(sew)
312 } else if result > max_sew {
313 *vxsat = true;
314 max_sew.cast_unsigned() & sew_mask(sew)
315 } else {
316 result.cast_unsigned() & sew_mask(sew)
317 }
318}
319
320#[inline(always)]
329#[doc(hidden)]
330#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
331pub fn nclipu(vs2_elem: u64, shamt: u32, sew: Vsew, mode: Vxrm, vxsat: &mut bool) -> u64 {
332 let shifted = rounded_srl(vs2_elem, shamt, mode);
334 let max_dst = sew_mask(sew);
336 if shifted > max_dst {
337 *vxsat = true;
338 max_dst
339 } else {
340 shifted & max_dst
341 }
342}
343
344#[inline(always)]
349#[doc(hidden)]
350#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
351pub fn nclip(vs2_elem: u64, shamt: u32, sew: Vsew, mode: Vxrm, vxsat: &mut bool) -> u64 {
352 let double_sew_bits = sew.bits_width() * 2;
356 let shift_amt = i64::BITS - u32::from(double_sew_bits);
357 let signed_wide = (vs2_elem.cast_signed() << shift_amt) >> shift_amt;
358 let low_bits = signed_wide.cast_unsigned()
361 & if double_sew_bits == 64 {
362 u64::MAX
363 } else {
364 (1u64 << double_sew_bits) - 1
365 };
366 let truncated = signed_wide >> shamt;
367 let r = round_increment(low_bits, shamt, mode, (truncated.cast_unsigned()) & 1);
368 let rounded = truncated.wrapping_add(r.cast_signed());
369 let min_dst = i64::MIN >> (i64::BITS - u32::from(sew.bits_width()));
371 let max_dst = i64::MAX >> (i64::BITS - u32::from(sew.bits_width()));
372 if rounded < min_dst {
373 *vxsat = true;
374 min_dst.cast_unsigned() & sew_mask(sew)
375 } else if rounded > max_dst {
376 *vxsat = true;
377 max_dst.cast_unsigned() & sew_mask(sew)
378 } else {
379 rounded.cast_unsigned() & sew_mask(sew)
380 }
381}
382
383#[inline(always)]
391#[doc(hidden)]
392#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
393pub fn execute_fixed_point_op<Reg, Env, F>(
394 env: &mut Env,
395 vd: VRegGroup<Env::Hart>,
396 vs2: VRegGroup<Env::Hart>,
397 src: OpSrc<VRegGroup<Env::Hart>>,
398 vm: bool,
399 sew: Vsew,
400 op: F,
401) where
402 Reg: Register,
403 Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
404 F: Fn(u64, u64, Vsew, Vxrm, &mut bool) -> u64,
406{
407 let vl = vd.vl();
408 let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
409 cold_path();
410 return;
411 };
412 let vxrm = env.vxrm();
413 let mask_buf = snapshot_mask(env.read_vregs(), vm);
414 let mut any_sat = false;
415 for i in vl.indices() {
416 if !mask_bit(&mask_buf, i) {
417 continue;
418 }
419 let a = env
420 .read_vregs()
421 .read(vs2, i)
422 .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
423 let b = match src.vreg {
424 Some(vs1) => env
425 .read_vregs()
426 .read(vs1, i)
427 .expect("`vs1` has the same `vl` as `vd`, checked above; qed"),
428 None => src.scalar,
429 };
430 let result = op(a, b, sew, vxrm, &mut any_sat);
431 env.write_vregs()
432 .write(vd, i, result)
433 .expect("`i < vl` of `vd`; qed");
434 }
435 if any_sat {
436 env.set_vxsat(true);
438 }
439 env.mark_vs_dirty();
440}
441
442#[inline(always)]
450#[doc(hidden)]
451#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
452pub fn execute_narrowing_clip_op<Reg, Env, F>(
453 env: &mut Env,
454 vd: VRegGroup<Env::Hart>,
455 vs2: VRegGroup<Env::Hart>,
456 src: OpSrc<VRegGroup<Env::Hart>>,
457 vm: bool,
458 sew: WideningSew<Env::Hart>,
459 op: F,
460) where
461 Reg: Register,
462 Env: VectorRegistersExt<Hart: HartConfig<Reg = Reg>>,
463 F: Fn(u64, u32, Vsew, Vxrm, &mut bool) -> u64,
465{
466 let sew = sew.narrow();
467 let vl = vd.vl();
468 let (Some(vs2), Some(src)) = (vs2.with_same_vl(vl), src.with_same_vl(vl)) else {
469 cold_path();
470 return;
471 };
472 let vxrm = env.vxrm();
473 let mask_buf = snapshot_mask(env.read_vregs(), vm);
474 let mut any_sat = false;
475 let shamt_mask = u64::from(sew.bits_width() * 2 - 1);
477 for i in vl.indices() {
478 if !mask_bit(&mask_buf, i) {
479 continue;
480 }
481 let wide_a = env
483 .read_vregs()
484 .read(vs2, i)
485 .expect("`vs2` has the same `vl` as `vd`, checked above; qed");
486 let shamt = match src.vreg {
487 Some(vs1) => {
488 let raw = env
489 .read_vregs()
490 .read(vs1, i)
491 .expect("`vs1` has the same `vl` as `vd`, checked above; qed");
492 (raw & shamt_mask) as u32
493 }
494 None => (src.scalar & shamt_mask) as u32,
495 };
496 let result = op(wide_a, shamt, sew, vxrm, &mut any_sat);
497 env.write_vregs()
498 .write(vd, i, result)
499 .expect("`i < vl` of `vd`; qed");
500 }
501 if any_sat {
502 env.set_vxsat(true);
503 }
504 env.mark_vs_dirty();
505}
506
507#[inline(always)]
512#[doc(hidden)]
513#[cfg_attr(feature = "no-panic", no_panic_const::no_panic)]
514pub fn check_narrowing_sew<Hart, Reg, Memory, PC>(
515 program_counter: &PC,
516 sew: Vsew,
517) -> Result<WideningSew<Hart>, ExecutionError<Reg::Type>>
518where
519 Hart: VectorHartConfig,
520 Reg: Register,
521 PC: ProgramCounter<Reg::Type, Memory>,
522{
523 WideningSew::<Hart>::new(sew).ok_or_else(|| {
524 cold_path();
525 ExecutionError::IllegalInstruction {
526 address: PackedAddress::new(program_counter.old_pc(INSTRUCTION_SIZE)),
527 }
528 })
529}