Add RDNA4 (gfx12) support and a 0.1.2 optimization round

RDNA4 (experimental; untested on RDNA4 hardware) - kernels/common/wmma_layout.h picks the WMMA operand/accumulator layout per target; K (pwin_*) and M (swin_*) index through it, so the same sources build for gfx11 and gfx12. D4R_WMMA_LAYOUT=12 on gfx11 is a test shim that runs the gfx12 indexing on gfx11 WMMA. - M can use native e4m3 FP8 WMMA on gfx12 (D4R_NATIVE_FP8=1). Texture tails (enc0_tail, dec0_head) get gfx12 f16 and FP8 paths that match ZLUDA's gfx12 lowering. - ZLUDA patch 0007: gfx12 WMMA helpers, optional native FP8 MMA (D4R_ZLUDA_WMMA_FP8_NATIVE), a layout override for testing, and an architecture argument for d4r_emit. Texture kernels are now compiled offline per target through d4r_emit (D4R_ZLUDA_EMIT) instead of on the local GPU. - New d4r.ini option [Kernels] NativeFp8 (default on), wired through the shim and d4r_config.py. On gfx12 the bridge serves kernels/<arch>-fp8 when it is on. - The bridge picks the KFD GPU with the most SIMDs (so an iGPU is not chosen over a discrete GPU); D4R_GPU_ARCH overrides it. - kernels/build.sh and package_release.sh build gfx1100-gfx1103 and gfx1200/gfx1201 (plus -fp8 variants), and D4R_BUNDLE_TEX takes per-target folders. Performance (RDNA3; harness byte-identical unless noted) - ZLUDA patch 0006: m16n8k8 f16 MMAs on WMMA (E; not bit-exact, 52-55 dB per layer, D4R_ZLUDA_WMMA_K8=0 disables it), per-key weight-image slots for native prep kernels (d4r_prep_key_at, d4r_prep_key_slots), and wave64 builds of MMA-free kernels (D4R_ZLUDA_WAVE64). - M: packed FP8 encoding, conflict-free patch-merge reads, scalar weight loads in enc1, and weight prep once per weight set for the six enc3 tube blocks. - K: CU mode for the six non-wide pwin layers. - Texture kernels: every DLSS flag variant is built natively (24 K output, 8 M post, 4 M downsample, 4 M enc0), not only Townfall's; post and K output run as wave64. - M direct output: rrlite_downsample_kernel writes the shim's output buffer; the shim's default direct-output presets are now 11,13. Tools - dump_runner takes the prep grid from d4r_prep_blocks and refuses texture/surface dumps. - New pwin_check.py, swin_check.py, model_enc3.py and tail_check.hip. Docs: README GPU table and RDNA4 notes, building (patches 0002-0007, d4r_emit), native-kernels (RDNA4 section), performance (new numbers), packaging README. Version 0.1.2.
2026-09-29 20:36:10 -04:00
1 parent f310d52b69
commit 4334a0c3c4
@@ -1,14 +1,15 @@
-// Native RDNA3 (gfx11 wave32 WMMA) kernels for the DLSS 4 (preset K) DltssPaddedWinLayer blocks.
+// Native kernels (gfx11 / gfx12 wave32 WMMA) for the DLSS 4 (preset K) DltssPaddedWinLayer blocks.
 // Semantics: kernels/tools/pwin_model.py (verified against the PTX interpreter, kernels/tools/ptxsim.py).
 //
 // Layout conventions (transposed WMMA, as in the preset M kernels): a GEMM Y = X W is computed as
-// Y^T = W^T X^T with the weights as the WMMA A operand (lane l: output channel l & 15, 16 K values)
-// and the activations as the B operand (lane l: token l & 15, 16 K values). The f32 result then has
-// lane l = token l & 15 and VGPR i = output channel 2i + (l >> 4) of the 16-channel tile ("D^T").
+// Y^T = W^T X^T with the weights as the WMMA A operand (lane l: output channel l & 15) and the activations
+// as the B operand (lane l: token l & 15). The f32 result then has lane l = token l & 15 and VGPR i =
+// output channel wm_acc_row(i) of the 16-channel tile ("D^T"); ../common/wmma_layout.h has the per-target
+// register layouts (gfx11: channel 2i + (l >> 4); gfx12: channel i + 8 (l >> 4)).
 #pragma once
 #include <hip/hip_runtime.h>
 #include <stdint.h>
-
+#include "../common/wmma_layout.h"
 #pragma clang fp contract(off)

 typedef _Float16 half_t;
@@ -23,16 +24,13 @@ __device__ __forceinline__ uint32_t lane_id()
    return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
 }

-__device__ __forceinline__ f8v wmma(u8v a, u8v b, f8v c)
-{
-    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(h16, a), __builtin_bit_cast(h16, b), c);
-}
+typedef wm_op op_t; // WMMA operand: u8v (gfx11) or u4v (gfx12)

 // one k16 step with the f16 accumulator of NVIDIA's f16 wmma (rounded after the step)
 // PWIN_F32ACC: keep the accumulator in f32 through the chain (rounded to f16 where the values are used)
-__device__ __forceinline__ f8v mma16(u8v a, u8v b, f8v c)
+__device__ __forceinline__ f8v mma16(const op_t& a, const op_t& b, f8v c)
 {
-    f8v d = wmma(a, b, c);
+    f8v d = wm_mma(a, b, c);
 #ifndef PWIN_F32ACC
 #pragma unroll
    for (int i = 0; i < 8; ++i)
@@ -65,50 +63,19 @@ __device__ __forceinline__ half_t hi16(uint32_t v)
    return __builtin_bit_cast(hv2, v)[1];
 }

-// D^T f16 values (own[i] = channel 2i + hf of token l & 15) -> B operand of the same token (all 16
-// channels in order): the other parity comes from lane l ^ 16
-__device__ __forceinline__ u8v operand_from_dt(const half_t own[8])
+// D^T f16 values (own[i] = channel wm_acc_row(i) of token l & 15) -> operand of the same token (the
+// channels become K)
+__device__ __forceinline__ op_t operand_from_dt(const half_t own[8])
 {
-    const uint32_t hf = lane_id() >> 4;
-    uint32_t mine[4], theirs[4];
-#pragma unroll
-    for (int j = 0; j < 4; ++j)
-    {
-        mine[j] = pack2(own[2 * j], own[2 * j + 1]);
-        theirs[j] = other_half(mine[j]);
-    }
-#ifndef PWIN_NO_PERM
-    // v_perm_b32(theirs, mine): bytes 0-3 = mine, 4-7 = theirs. Even j: the low halves, odd j: the high
-    // halves, in the order (mine, theirs) for hf = 0 and (theirs, mine) for hf = 1.
-    const uint32_t sel_lo = hf ? 0x01000504u : 0x05040100u, sel_hi = hf ? 0x03020706u : 0x07060302u;
-    u8v r;
-#pragma unroll
-    for (int j = 0; j < 8; ++j)
-        r[j] = __builtin_amdgcn_perm(theirs[j >> 1], mine[j >> 1], (j & 1) ? sel_hi : sel_lo);
-    return r;
-#else
-    u8v r;
-#pragma unroll
-    for (int j = 0; j < 8; ++j)
-    {
-        const half_t a = (j & 1) ? hi16(mine[j >> 1]) : lo16(mine[j >> 1]);
-        const half_t b = (j & 1) ? hi16(theirs[j >> 1]) : lo16(theirs[j >> 1]);
-        r[j] = hf ? pack2(b, a) : pack2(a, b);
-    }
-    return r;
-#endif
+    return wm_op_from_acc(own);
 }

-__device__ __forceinline__ u8v operand_from_f8(f8v d)
+__device__ __forceinline__ op_t operand_from_f8(f8v d)
 {
-    half_t h[8];
-#pragma unroll
-    for (int i = 0; i < 8; ++i)
-        h[i] = (half_t)d[i];
-    return operand_from_dt(h);
+    return wm_op_from_f8(d);
 }

-// element c (0..15) of a B operand (16 halves)
+// element c (0..15) of a row of 16 halves (VALU code; not a WMMA operand)
 __device__ __forceinline__ half_t op_get(const u8v& v, int c)
 {
    return (c & 1) ? hi16(v[c >> 1]) : lo16(v[c >> 1]);
@@ -145,6 +112,58 @@ __device__ __forceinline__ void gstore_row16(half_t* dst, const u8v& v)
    gstore16(dst + 8, (u4v){v[4], v[5], v[6], v[7]});
 }

+// WMMA operands from rows of 16 channels: LDS, global (PWIN_NT_LOAD as above), prep images (32-byte slots)
+__device__ __forceinline__ op_t op_lds(const half_t* p)
+{
+    return wm_op_load(p);
+}
+__device__ __forceinline__ op_t op_gload(const half_t* p)
+{
+#if D4R_WMMA_LAYOUT == 12
+#ifdef PWIN_NT_LOAD
+    return __builtin_nontemporal_load((const u4v*)(p + 8 * wm_half()));
+#else
+    return *(const u4v*)(p + 8 * wm_half());
+#endif
+#else
+    return gload_row16(p);
+#endif
+}
+__device__ __forceinline__ op_t op_img(const u8v* __restrict__ img, int idx)
+{
+    return wm_op_image(img, idx);
+}
+// operand -> slot idx of an operand image in LDS (read back with op_img)
+__device__ __forceinline__ void op_img_store(u8v* img, int idx, const op_t& v)
+{
+#if D4R_WMMA_LAYOUT == 12
+    ((u4v*)img)[2 * idx + wm_half()] = v;
+#else
+    if (!wm_half())
+        img[idx] = v;
+#endif
+}
+// operand (16 channels of this lane's token) -> row of `count` (8 or 16) channels in global memory, if ok
+__device__ __forceinline__ void op_gstore(half_t* dst, const op_t& v, bool ok, int count = 16)
+{
+#if D4R_WMMA_LAYOUT == 12
+    if (ok && (wm_half() == 0 || count >= 16))
+        gstore16(dst + 8 * wm_half(), v);
+#else
+    if (!wm_half() && ok)
+    {
+        gstore16(dst, (u4v){v[0], v[1], v[2], v[3]});
+        if (count >= 16)
+            gstore16(dst + 8, (u4v){v[4], v[5], v[6], v[7]});
+    }
+#endif
+}
+// operand -> row of 16 channels in LDS
+__device__ __forceinline__ void op_store(half_t* dst, const op_t& v)
+{
+    wm_op_store(dst, v);
+}
+
 __device__ __forceinline__ void block_sync()
 {
    __syncthreads();
@@ -162,27 +181,3 @@ __device__ __forceinline__ int frag_offset(int k, int n)
 {
    return 64 * (n & 7) + 16 * ((k & 7) >> 1) + 8 * (n >> 3) + 4 * (k >> 3) + 2 * (k & 1);
 }
-
-// A-operand image of a K x N weight: tile (kt, nt) = 16 lanes x 8 dwords (lane = output channel)
-struct WDesc
-{
-    int src;       // byte offset of the fragments in the weight buffer
-    int kt_stride; // bytes between k tiles
-    int nt_stride; // bytes between n tiles
-    int KT, NT;    // tiles
-    int dst;       // u8v index of the image in the prep buffer
-};
-
-// weight operand tile `tile` of an A-operand image (16 lanes x 32 bytes): lanes l and l + 16 need the same
-// row, so each half-wave loads one 16-byte half and the halves are swapped across (halves L2 traffic)
-__device__ __forceinline__ u8v wtile(const u8v* __restrict__ img, int tile)
-{
-    const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
-    const u4v mine = ((const u4v*)&img[tile * 16 + m])[hf];
-    u4v other;
-#pragma unroll
-    for (int j = 0; j < 4; ++j)
-        other[j] = other_half(mine[j]);
-    const u4v lo = hf ? other : mine, hi = hf ? mine : other;
-    return (u8v){lo[0], lo[1], lo[2], lo[3], hi[0], hi[1], hi[2], hi[3]};
-}