engine/rr: hardware FP8 saturation for the f32 conversions

RADV lowers saturatedConvertEXT to FP8 as a NaN-preserving clamp per element
(v_minimummaximum_f32) followed by v_cvt_pk_fp8_f32, because the hardware's
FP16_OVFL saturation turns infinities into NaN instead of +-448. Run over all
2^32 f32 inputs on an RX 9070 XT, FP16_OVFL gives the clamp's result for
every finite value and NaN; only +-Inf differ.

The f32 values the native stages convert are never infinite (sums of FP8
products and binary16 biases, far below 3.4e38). Writing the conversion as
"an infinity becomes NaN, then saturate" (FINITE in rr_exact.glsl) makes it
the pattern nir_opt_algebraic turns into f2e4m3fn_satfn (vkd3d-proton's,
Mesa >= 25.2, the same release that added the FP8 conversions): the
hardware conversion with FP16_OVFL and no clamp. RR_SAT_CLAMP=1 builds the
old form.

168 - 232 fewer instructions per network stage (enc0 960 -> 128 clamps).
Every image byte-identical on the replay inputs (also robustness2). RX 9070
XT, 720p -> 1440p: frame 3.92 -> 3.79 ms, every network stage faster
(dec0_1 607 -> 576 us, enc0 752 -> 727, enc1 262 -> 247).

Signed-off-by: itchyfeetleech <[email protected]>

engine/rr/rr_exact.glsl

@@ -31,6 +31,19 @@ uint inter(uint j) { return (j & ~31u) | (((j & 31u) >> 2u) << 1u) | (j & 1u) |
 uint uninter(uint c) { return (c & ~31u) | (((c & 15u) >> 1u) << 2u) | (c & 1u) | (((c >> 4u) & 1u) << 1u); }
 int mirror(int v, int n) { v = abs(v); return min(v, 2 * n - 2 - v); }
 
+// FINITE(v): v, but an infinity (which these f32 values never are: sums of FP8 products and binary16 biases stay far
+// below 3.4e38) becomes NaN. Written so, a saturating FP8 conversion is RADV's f2e4m3fn_satfn (nir_opt_algebraic's
+// vkd3d-proton pattern): the hardware conversion with FP16_OVFL, which saturates every finite value the same way,
+// instead of a NaN-preserving clamp per element and then the conversion. RR_SAT_CLAMP=1 keeps the clamp, and so does
+// RR_MIX, whose binary16 accumulators can overflow.
+#if defined(RR_SAT_CLAMP) || defined(RR_MIX)
+#define FINITE(v) (v)
+#define FINITE2(v) (v)
+#else
+#define FINITE(v) (isinf(v) ? uintBitsToFloat(0x7fc00000u) : (v))
+#define FINITE2(v) vec2(FINITE((v).x), FINITE((v).y))
+#endif
+
 #ifdef RR_GFX11
 #include "rr_gfx11.glsl"
 #define F8 uint
@@ -52,7 +65,7 @@ floate4m3_t q8(float16_t h)
 }
 // the same from f32 / f16 values in one saturating hardware conversion (satfinite; for an f32 value that is a binary16
 // number the result equals the f16 conversion's)
-floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
+floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, FINITE(x)); return r; }
 floate4m3_t q8h(float16_t x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
 
 uint code(floate4m3_t v) { return uint(floate4m3BitsToUintEXT(v)); }
@@ -124,13 +137,13 @@ float16_t fma16q(float16_t a, float16_t b, float16_t c, inout bool mid)
 }
 #ifndef RR_GFX11
 // The saturating FP8 conversion of a whole accumulator, two elements per conversion instruction
-#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, vec2((src)[i_], (src)[i_ + 1])); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(vec2((src)[i_], (src)[i_ + 1]))); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 // q8(act(h)) from the packed table (enc1_pack.act_table; the weight buffer's W8U view at W_ACT)
 #define ACTQ(h) uintBitsToFloate4m3EXT(w8u.w[W_ACT + bh(h)])
 // the activation of a whole accumulator into FP8. RR_NATIVE (D4R_RR_RELAXED=3): in f32, two elements per
 // instruction, straight into FP8 - no binary16 in between and no table (not the reference's roundings)
 #ifdef RR_NATIVE
-#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 #else
 #define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; ++i_) (dst)[i_] = ACTQ(float16_t((h)[i_]));
 #endif