woutermans
8eb7a94533
- engine/compile_rr2.py, runtime_rr2.inc, engine/rr/rr2_m.comp, exposure_f.comp, gen_rr2_enc0.py, rr2_pack.py, rr2_ref.py: the rr2_* model (nvngx_dlssd.dll 310.9's third network) as d4r's own shaders - exposure, enc0 gather/block split, enc1-5 and dec4-0 through one matrix shader, translated output passes; weights packed from the DLL, argument blocks from the capture - ptx2rr: match the driver's fused roundings - opaque-zero OR around cvt.rn.f16.f32 against v_fma_mix, half-away-from-zero formatted stores, UNORM 10:10:10:2 as RGBA32F of q/1023, OCML tanhF with the AMDGPU exp lowering, binary16 fma and fetch roundings; --probe for register bisection against ZLUDA - dec0_m/swin_m/enc0_*: RR_MIX build option (D4R_RR_MIX=1), one v_fma_mix rounding per 32-deep step, ~5% faster, not bit-exact; default build stays exact - runtime_rr.inc: RModel/REngine dispatch to the D4RR0002 model; replay_rr: RGBA32F and A2B10G10R1010 texel sizes, roughnessPacked bit, larger debug staging - shim: engine\rr-f model slot, 310.9 version probe so preset F is only claimed when the CUDA path has it; d4r.ini / d4r_config.py Model = F; run_rr_engine_compare.sh D4R_RR_DLL - kernels/rr/swin_native.h + rr_swin_native_test.cpp: HIP port of encoder 1 for the FP8-product and rounding cost measurements - engine/rr/README: preset F's block, options, capture-by-capture results, timings and the open camera-matrix difference
71 lines
4.0 KiB
GLSL
71 lines
4.0 KiB
GLSL
// The attention core shared by the 32-channel-head stages (engine/rr/enc1_m.comp, dec0_m.comp), included inside the
|
|
// head loop: this wave's 16 query slots against the window's 96 key slots. Expects KS (K, token slot x 32 FP8
|
|
// channels in k order) and VS (V^T, 32 channels x 96 token slots) filled for head hd, the bias at W_BIAS0 + HEAD * hd
|
|
// (enc1_pack: 96 x 64, [key slot][query slot]) and the cubic table at W_CUBIC. Leaves Ob[2]: O^T (32 channels x
|
|
// this wave's queries) as the out-projection's B operand. RR_KS / RR_VS (+ word offsets RR_KS0 / RR_VS0) name
|
|
// the arrays when they are not KS and VS.
|
|
#ifndef RR_KS
|
|
#define RR_KS KS
|
|
#define RR_KS0 0u
|
|
#endif
|
|
#ifndef RR_VS
|
|
#define RR_VS VS
|
|
#define RR_VS0 0u
|
|
#endif
|
|
B8 Qb[2];
|
|
[[unroll]] for (uint c = 0u; c < 2u; ++c) coopMatLoad(Qb[c], RR_KS, RR_KS0 + wv * 128u + 4u * c, 8u, CM);
|
|
// scores S^T (keys x this wave's queries) + bias, then the cubic softmax down each column
|
|
B8 Pb[6];
|
|
{
|
|
float16_t e[6][8];
|
|
[[unroll]] for (uint kt = 0u; kt < 6u; ++kt) {
|
|
HAC c;
|
|
coopMatLoad(c, w16.w, (W_BIAS0 + HEAD * hd) / 2u + kt * 1024u + 16u * wv, 64u, RM);
|
|
A8 k0, k1;
|
|
coopMatLoad(k0, RR_KS, RR_KS0 + kt * 128u, 8u, RM);
|
|
coopMatLoad(k1, RR_KS, RR_KS0 + kt * 128u + 4u, 8u, RM);
|
|
K32H(c, k0, Qb[0], k1, Qb[1])
|
|
// the cubic numerator from enc1_pack.cubic_table, indexed by the score's binary16 bits
|
|
[[unroll]] for (int i = 0; i < 8; ++i) e[kt][i] = hb(uint(w16u.w[W_CUBIC / 2u + bh(c[i])]));
|
|
}
|
|
// key slot 16kt + 8hl + i is NVIDIA's column 8j + 2tt + half with j = 4(kt >> 1) + (kt & 1) + 2((i >> 1) & 1),
|
|
// tt = 2hl + (i >> 2), half = i & 1 (enc1_ref.softmax_cubic's tree)
|
|
// packed: component h of a pair is column parity h (elements i = 4tq + h, i + 2), so the tree runs on f16x2
|
|
f16vec2 ep[6][4];
|
|
[[unroll]] for (uint kt = 0u; kt < 6u; ++kt)
|
|
[[unroll]] for (uint q = 0u; q < 4u; ++q) ep[kt][q] = f16vec2(e[kt][2u * q], e[kt][2u * q + 1u]);
|
|
precise f16vec2 st[2];
|
|
[[unroll]] for (uint tq = 0u; tq < 2u; ++tq) { // pairs q = 2tq (i0) and 2tq + 1 (i1 = i0 + 2)
|
|
const uint q0 = 2u * tq, q1 = q0 + 1u;
|
|
precise f16vec2 ev = ((ep[0][q0] + ep[0][q1]) + (ep[2][q0] + ep[2][q1])) + (ep[4][q0] + ep[4][q1]);
|
|
precise f16vec2 od = ((ep[1][q0] + ep[1][q1]) + (ep[3][q0] + ep[3][q1])) + (ep[5][q0] + ep[5][q1]);
|
|
st[tq] = ev + od;
|
|
}
|
|
precise f16vec2 loc = st[0] + st[1];
|
|
const uint other = subgroupShuffleXor(packFloat2x16(loc), 16u);
|
|
precise f16vec2 o = loc + unpackFloat2x16(other);
|
|
precise float16_t rs = o.x + o.y;
|
|
// f16(1 / rs) for a binary16 rs: x * m = 1 + d with |d| >= 2^-22 for every binary16 midpoint m, so the f32
|
|
// reciprocal (within an ulp) already rounds correctly
|
|
const float16_t r = float16_t(1.0 / float(rs));
|
|
const f16vec2 r2 = f16vec2(r, r);
|
|
[[unroll]] for (uint kt = 0u; kt < 6u; ++kt)
|
|
[[unroll]] for (uint q = 0u; q < 4u; ++q) {
|
|
precise f16vec2 pv = ep[kt][q] * r2;
|
|
// a probability is in [0, 1]: the plain conversion, no saturation clamp, both in one instruction
|
|
{ const fe4m3vec2 t_ = fe4m3vec2(pv); Pb[kt][2u * q] = t_.x; Pb[kt][2u * q + 1u] = t_.y; }
|
|
}
|
|
}
|
|
// O^T = V^T P^T, then the out-projection onto the residual
|
|
B8 Ob[2];
|
|
[[unroll]] for (uint c = 0u; c < 2u; ++c) {
|
|
RAC o = RAC(RV(float16_t(0.0)));
|
|
[[unroll]] for (uint ks = 0u; ks < 3u; ++ks) {
|
|
A8 v0, v1;
|
|
coopMatLoad(v0, RR_VS, RR_VS0 + c * 384u + 8u * ks, 24u, RM);
|
|
coopMatLoad(v1, RR_VS, RR_VS0 + c * 384u + 8u * ks + 4u, 24u, RM);
|
|
K32(o, v0, Pb[2u * ks], v1, Pb[2u * ks + 1u])
|
|
}
|
|
Q8(Ob[c], o)
|
|
}
|