Token-lane block for the RDNA4 native-FP8 layers: enc1, dec1 and the enc3 tube

swin_tok.h passes the weights as the first WMMA operand and the activations as the second, so each GEMM's encoded result is already the next one's operand and stays in registers (swin_block.h transposes every result through LDS). One wave owns one 4x4 window for the whole block: norms, attention and the MLP need no LDS and no barriers; only the patch expand and the patch merge go through an e4m3 image in LDS. Outputs are bit-identical to swin_block.h (swapped WMMA operand roles, verified on an RX 9070 XT and on recorded launches). mix2 rounds in place (accumulator is addend and destination, no copy per pair). -DSWIN_NO_TOKEN_LANES builds these layers from swin_block.h again; enc2 and dec2 are unchanged. RX 9070 XT accuracy set, 2560x1440 render: whole evaluation 8.56 -> 7.18 ms.
This commit is contained in:
woutermans committed 2026-10-06 22:10:41 +02:00
1 parent 7c60d53cd0
commit a7d565d1cd
7 files changed
+730

No files matched your search

+8
View File
@@ -441,10 +441,18 @@ __device__ __forceinline__ T16 t16_splat(half_t h)
// acc = f16(P + acc), rounded once (v_fma_mix: verified == f16(exact sum) on 8.4M cases)
__device__ __forceinline__ hv2 mix2(float p0, float p1, hv2 c)
{
#ifdef MIX_COPY
hv2 r = c;
asm(MIX_PAD "v_fma_mixlo_f16 %0, %1, 1.0, %2 op_sel_hi:[0,0,1]" : "+v"(r) : "v"(p0), "v"(c));
asm("v_fma_mixhi_f16 %0, %1, 1.0, %2 op_sel:[0,0,1] op_sel_hi:[0,0,1]" : "+v"(r) : "v"(p1), "v"(c));
return r;
#else
// in place: mixlo writes only the low half and mixhi reads only the high half of the addend, so the
// accumulator is both the addend and the destination (no copy of it per pair)
asm(MIX_PAD "v_fma_mixlo_f16 %0, %1, 1.0, %0 op_sel_hi:[0,0,1]" : "+v"(c) : "v"(p0));
asm("v_fma_mixhi_f16 %0, %1, 1.0, %0 op_sel:[0,0,1] op_sel_hi:[0,0,1]" : "+v"(c) : "v"(p1));
return c;
#endif
}
__device__ __forceinline__ void mixacc(T16& acc, f8v P)
{