engine/rr: hardware FP8 saturation for the f32 conversions
RADV lowers saturatedConvertEXT to FP8 as a NaN-preserving clamp per element (v_minimummaximum_f32) followed by v_cvt_pk_fp8_f32, because the hardware's FP16_OVFL saturation turns infinities into NaN instead of +-448. Run over all 2^32 f32 inputs on an RX 9070 XT, FP16_OVFL gives the clamp's result for every finite value and NaN; only +-Inf differ. The f32 values the native stages convert are never infinite (sums of FP8 products and binary16 biases, far below 3.4e38). Writing the conversion as "an infinity becomes NaN, then saturate" (FINITE in rr_exact.glsl) makes it the pattern nir_opt_algebraic turns into f2e4m3fn_satfn (vkd3d-proton's, Mesa >= 25.2, the same release that added the FP8 conversions): the hardware conversion with FP16_OVFL and no clamp. RR_SAT_CLAMP=1 builds the old form. 168 - 232 fewer instructions per network stage (enc0 960 -> 128 clamps). Every image byte-identical on the replay inputs (also robustness2). RX 9070 XT, 720p -> 1440p: frame 3.92 -> 3.79 ms, every network stage faster (dec0_1 607 -> 576 us, enc0 752 -> 727, enc1 262 -> 247). Signed-off-by: itchyfeetleech <[email protected]>
This commit is contained in:
itchyfeetleech committed
1 parent
2fa4561bc1
commit
e46c589c68
4 files changed
+31
-7
No files matched your search
+12
-1
@@ -555,7 +555,18 @@ registers to the waves its shared memory leaves room for (enc1 needed 97 vector
|
||||
enc0 747 -> 729 us, dec0_1 598 -> 592. The output passes keep both ORs (they are not passed through
|
||||
`spv_exact_convert.py`, and their ISA already uses `v_fma_mix*_f16` for plain conversions, so it cannot be checked
|
||||
the same way).
|
||||
Frame 4.09 -> 3.97 ms (network 3.39 -> 3.28). Tried without gain: the decoders' first kernels' expansion in KVS (16 /
|
||||
- **Hardware saturation for the FP8 conversions of f32 values** (`rr_exact.glsl` `FINITE`). RADV lowers a saturating
|
||||
conversion (`saturatedConvertEXT`) as a NaN-preserving clamp per element (`v_minimummaximum_f32`) and then the
|
||||
conversion, because the hardware's FP16_OVFL saturation turns infinities into NaN instead of +-448 (ACO's comment;
|
||||
confirmed on all 2^32 f32 inputs, RX 9070 XT: FP16_OVFL is identical to the clamp for every finite value and NaN,
|
||||
only +-Inf differ). The values converted here never are infinite: sums of FP8 products (each at most 448^2) and
|
||||
binary16 biases stay far below 3.4e38. Written as "an infinity becomes NaN, then saturate", the conversion is the
|
||||
pattern `nir_opt_algebraic` turns into `f2e4m3fn_satfn` (vkd3d-proton's; Mesa >= 25.2, `has_f2e4m3fn_satfn`):
|
||||
the hardware conversion with FP16_OVFL and no clamp. 168 - 232 clamps fewer per network stage (enc0 960 -> 128);
|
||||
frame 3.92 -> 3.79 ms, every stage faster (dec0_1 607 -> 576 us, enc0 752 -> 727, enc1 262 -> 247). The binary16
|
||||
conversions (the decoders' first kernels' input) keep the clamp: NIR moves the infinity test through the widening
|
||||
and the pattern no longer matches. `RR_SAT_CLAMP=1` builds the old form.
|
||||
Frame 4.09 -> 3.84 ms. Tried without gain: the decoders' first kernels' expansion in KVS (16 /
|
||||
12 / 8 KiB, but the deep levels have too few windows for the waves to matter and the extra barrier cost 4 - 5 us
|
||||
each), all twelve tiles of a 1440p frame in one group (sixteen tile slots: the same), the output passes on 32-wide
|
||||
waves (0.67 -> 0.82 ms) and unrolled head / branch loops in the 160-channel stages (+10 - 20 us each). Per stage
|
||||
|
||||
@@ -90,7 +90,7 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena;
|
||||
#define E4V(w, b) float(uintBitsToFloate4m3EXT(uint8_t(((w) >> (8u * (b))) & 0xffu)))
|
||||
#define RESID(d, a, s, c) const float d = fma(float(a), float(s), float(c));
|
||||
#define Q8R(x) q8s(x)
|
||||
#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#else
|
||||
#define RVT float16_t
|
||||
#define E4V(w, b) e4((w), (b))
|
||||
|
||||
@@ -55,7 +55,7 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena;
|
||||
#define LDB(dst, base, n, kt, nt) coopMatLoad((dst), w8.w, (base) + 256u * ((kt) * ((n) / 16u) + (nt)), 16u, RM)
|
||||
#define K32(acc, a0, b0, a1, b1) { (acc) = coopMatMulAdd((a1), (b1), coopMatMulAdd((a0), (b0), (acc))); }
|
||||
#define Q8(dst, src) Q8PAIRS(dst, src)
|
||||
#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
const uint HEAD = W_WQK1 - W_WQK0, BRANCH = W_WD1 - W_WD0;
|
||||
|
||||
#if KIND == 4 && defined(RR2_XF_SHARED)
|
||||
@@ -370,7 +370,7 @@ void main()
|
||||
[[unroll]] for (uint m = 0u; m < MT; ++m) {
|
||||
floate4m3_t Tb[8];
|
||||
[[unroll]] for (int i = 0; i < 8; i += 2) {
|
||||
fe4m3vec2 q; saturatedConvertEXT(q, vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1])));
|
||||
fe4m3vec2 q; saturatedConvertEXT(q, FINITE2(vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1]))));
|
||||
Tb[i] = q.x; Tb[i+1] = q.y;
|
||||
}
|
||||
#if KIND == 4
|
||||
|
||||
+16
-3
@@ -31,6 +31,19 @@ uint inter(uint j) { return (j & ~31u) | (((j & 31u) >> 2u) << 1u) | (j & 1u) |
|
||||
uint uninter(uint c) { return (c & ~31u) | (((c & 15u) >> 1u) << 2u) | (c & 1u) | (((c >> 4u) & 1u) << 1u); }
|
||||
int mirror(int v, int n) { v = abs(v); return min(v, 2 * n - 2 - v); }
|
||||
|
||||
// FINITE(v): v, but an infinity (which these f32 values never are: sums of FP8 products and binary16 biases stay far
|
||||
// below 3.4e38) becomes NaN. Written so, a saturating FP8 conversion is RADV's f2e4m3fn_satfn (nir_opt_algebraic's
|
||||
// vkd3d-proton pattern): the hardware conversion with FP16_OVFL, which saturates every finite value the same way,
|
||||
// instead of a NaN-preserving clamp per element and then the conversion. RR_SAT_CLAMP=1 keeps the clamp, and so does
|
||||
// RR_MIX, whose binary16 accumulators can overflow.
|
||||
#if defined(RR_SAT_CLAMP) || defined(RR_MIX)
|
||||
#define FINITE(v) (v)
|
||||
#define FINITE2(v) (v)
|
||||
#else
|
||||
#define FINITE(v) (isinf(v) ? uintBitsToFloat(0x7fc00000u) : (v))
|
||||
#define FINITE2(v) vec2(FINITE((v).x), FINITE((v).y))
|
||||
#endif
|
||||
|
||||
#ifdef RR_GFX11
|
||||
#include "rr_gfx11.glsl"
|
||||
#define F8 uint
|
||||
@@ -52,7 +65,7 @@ floate4m3_t q8(float16_t h)
|
||||
}
|
||||
// the same from f32 / f16 values in one saturating hardware conversion (satfinite; for an f32 value that is a binary16
|
||||
// number the result equals the f16 conversion's)
|
||||
floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
|
||||
floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, FINITE(x)); return r; }
|
||||
floate4m3_t q8h(float16_t x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
|
||||
|
||||
uint code(floate4m3_t v) { return uint(floate4m3BitsToUintEXT(v)); }
|
||||
@@ -124,13 +137,13 @@ float16_t fma16q(float16_t a, float16_t b, float16_t c, inout bool mid)
|
||||
}
|
||||
#ifndef RR_GFX11
|
||||
// The saturating FP8 conversion of a whole accumulator, two elements per conversion instruction
|
||||
#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, vec2((src)[i_], (src)[i_ + 1])); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(vec2((src)[i_], (src)[i_ + 1]))); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
// q8(act(h)) from the packed table (enc1_pack.act_table; the weight buffer's W8U view at W_ACT)
|
||||
#define ACTQ(h) uintBitsToFloate4m3EXT(w8u.w[W_ACT + bh(h)])
|
||||
// the activation of a whole accumulator into FP8. RR_NATIVE (D4R_RR_RELAXED=3): in f32, two elements per
|
||||
// instruction, straight into FP8 - no binary16 in between and no table (not the reference's roundings)
|
||||
#ifdef RR_NATIVE
|
||||
#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
|
||||
#else
|
||||
#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; ++i_) (dst)[i_] = ACTQ(float16_t((h)[i_]));
|
||||
#endif
|
||||
|
||||
Reference in new issue
Block a user