diff --git a/engine/rr/README.md b/engine/rr/README.md index 67007ce..735a468 100644 --- a/engine/rr/README.md +++ b/engine/rr/README.md @@ -555,7 +555,18 @@ registers to the waves its shared memory leaves room for (enc1 needed 97 vector enc0 747 -> 729 us, dec0_1 598 -> 592. The output passes keep both ORs (they are not passed through `spv_exact_convert.py`, and their ISA already uses `v_fma_mix*_f16` for plain conversions, so it cannot be checked the same way). -Frame 4.09 -> 3.97 ms (network 3.39 -> 3.28). Tried without gain: the decoders' first kernels' expansion in KVS (16 / +- **Hardware saturation for the FP8 conversions of f32 values** (`rr_exact.glsl` `FINITE`). RADV lowers a saturating + conversion (`saturatedConvertEXT`) as a NaN-preserving clamp per element (`v_minimummaximum_f32`) and then the + conversion, because the hardware's FP16_OVFL saturation turns infinities into NaN instead of +-448 (ACO's comment; + confirmed on all 2^32 f32 inputs, RX 9070 XT: FP16_OVFL is identical to the clamp for every finite value and NaN, + only +-Inf differ). The values converted here never are infinite: sums of FP8 products (each at most 448^2) and + binary16 biases stay far below 3.4e38. Written as "an infinity becomes NaN, then saturate", the conversion is the + pattern `nir_opt_algebraic` turns into `f2e4m3fn_satfn` (vkd3d-proton's; Mesa >= 25.2, `has_f2e4m3fn_satfn`): + the hardware conversion with FP16_OVFL and no clamp. 168 - 232 clamps fewer per network stage (enc0 960 -> 128); + frame 3.92 -> 3.79 ms, every stage faster (dec0_1 607 -> 576 us, enc0 752 -> 727, enc1 262 -> 247). The binary16 + conversions (the decoders' first kernels' input) keep the clamp: NIR moves the infinity test through the widening + and the pattern no longer matches. `RR_SAT_CLAMP=1` builds the old form. +Frame 4.09 -> 3.84 ms. Tried without gain: the decoders' first kernels' expansion in KVS (16 / 12 / 8 KiB, but the deep levels have too few windows for the waves to matter and the extra barrier cost 4 - 5 us each), all twelve tiles of a 1440p frame in one group (sixteen tile slots: the same), the output passes on 32-wide waves (0.67 -> 0.82 ms) and unrolled head / branch loops in the 160-channel stages (+10 - 20 us each). Per stage diff --git a/engine/rr/rr2_m.comp b/engine/rr/rr2_m.comp index bb2b5a5..5a214ef 100644 --- a/engine/rr/rr2_m.comp +++ b/engine/rr/rr2_m.comp @@ -90,7 +90,7 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena; #define E4V(w, b) float(uintBitsToFloate4m3EXT(uint8_t(((w) >> (8u * (b))) & 0xffu))) #define RESID(d, a, s, c) const float d = fma(float(a), float(s), float(c)); #define Q8R(x) q8s(x) -#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } +#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } #else #define RVT float16_t #define E4V(w, b) e4((w), (b)) diff --git a/engine/rr/rr2_w.comp b/engine/rr/rr2_w.comp index c359f78..58ad9e3 100644 --- a/engine/rr/rr2_w.comp +++ b/engine/rr/rr2_w.comp @@ -55,7 +55,7 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena; #define LDB(dst, base, n, kt, nt) coopMatLoad((dst), w8.w, (base) + 256u * ((kt) * ((n) / 16u) + (nt)), 16u, RM) #define K32(acc, a0, b0, a1, b1) { (acc) = coopMatMulAdd((a1), (b1), coopMatMulAdd((a0), (b0), (acc))); } #define Q8(dst, src) Q8PAIRS(dst, src) -#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } +#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } const uint HEAD = W_WQK1 - W_WQK0, BRANCH = W_WD1 - W_WD0; #if KIND == 4 && defined(RR2_XF_SHARED) @@ -370,7 +370,7 @@ void main() [[unroll]] for (uint m = 0u; m < MT; ++m) { floate4m3_t Tb[8]; [[unroll]] for (int i = 0; i < 8; i += 2) { - fe4m3vec2 q; saturatedConvertEXT(q, vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1]))); + fe4m3vec2 q; saturatedConvertEXT(q, FINITE2(vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1])))); Tb[i] = q.x; Tb[i+1] = q.y; } #if KIND == 4 diff --git a/engine/rr/rr_exact.glsl b/engine/rr/rr_exact.glsl index 9dfb47f..193cd83 100644 --- a/engine/rr/rr_exact.glsl +++ b/engine/rr/rr_exact.glsl @@ -31,6 +31,19 @@ uint inter(uint j) { return (j & ~31u) | (((j & 31u) >> 2u) << 1u) | (j & 1u) | uint uninter(uint c) { return (c & ~31u) | (((c & 15u) >> 1u) << 2u) | (c & 1u) | (((c >> 4u) & 1u) << 1u); } int mirror(int v, int n) { v = abs(v); return min(v, 2 * n - 2 - v); } +// FINITE(v): v, but an infinity (which these f32 values never are: sums of FP8 products and binary16 biases stay far +// below 3.4e38) becomes NaN. Written so, a saturating FP8 conversion is RADV's f2e4m3fn_satfn (nir_opt_algebraic's +// vkd3d-proton pattern): the hardware conversion with FP16_OVFL, which saturates every finite value the same way, +// instead of a NaN-preserving clamp per element and then the conversion. RR_SAT_CLAMP=1 keeps the clamp, and so does +// RR_MIX, whose binary16 accumulators can overflow. +#if defined(RR_SAT_CLAMP) || defined(RR_MIX) +#define FINITE(v) (v) +#define FINITE2(v) (v) +#else +#define FINITE(v) (isinf(v) ? uintBitsToFloat(0x7fc00000u) : (v)) +#define FINITE2(v) vec2(FINITE((v).x), FINITE((v).y)) +#endif + #ifdef RR_GFX11 #include "rr_gfx11.glsl" #define F8 uint @@ -52,7 +65,7 @@ floate4m3_t q8(float16_t h) } // the same from f32 / f16 values in one saturating hardware conversion (satfinite; for an f32 value that is a binary16 // number the result equals the f16 conversion's) -floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; } +floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, FINITE(x)); return r; } floate4m3_t q8h(float16_t x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; } uint code(floate4m3_t v) { return uint(floate4m3BitsToUintEXT(v)); } @@ -124,13 +137,13 @@ float16_t fma16q(float16_t a, float16_t b, float16_t c, inout bool mid) } #ifndef RR_GFX11 // The saturating FP8 conversion of a whole accumulator, two elements per conversion instruction -#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, vec2((src)[i_], (src)[i_ + 1])); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } +#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(vec2((src)[i_], (src)[i_ + 1]))); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } // q8(act(h)) from the packed table (enc1_pack.act_table; the weight buffer's W8U view at W_ACT) #define ACTQ(h) uintBitsToFloate4m3EXT(w8u.w[W_ACT + bh(h)]) // the activation of a whole accumulator into FP8. RR_NATIVE (D4R_RR_RELAXED=3): in f32, two elements per // instruction, straight into FP8 - no binary16 in between and no table (not the reference's roundings) #ifdef RR_NATIVE -#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } +#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; } #else #define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; ++i_) (dst)[i_] = ACTQ(float16_t((h)[i_])); #endif