Add RDNA4 (gfx12) support and a 0.1.2 optimization round
RDNA4 (experimental; untested on RDNA4 hardware) - kernels/common/wmma_layout.h picks the WMMA operand/accumulator layout per target; K (pwin_*) and M (swin_*) index through it, so the same sources build for gfx11 and gfx12. D4R_WMMA_LAYOUT=12 on gfx11 is a test shim that runs the gfx12 indexing on gfx11 WMMA. - M can use native e4m3 FP8 WMMA on gfx12 (D4R_NATIVE_FP8=1). Texture tails (enc0_tail, dec0_head) get gfx12 f16 and FP8 paths that match ZLUDA's gfx12 lowering. - ZLUDA patch 0007: gfx12 WMMA helpers, optional native FP8 MMA (D4R_ZLUDA_WMMA_FP8_NATIVE), a layout override for testing, and an architecture argument for d4r_emit. Texture kernels are now compiled offline per target through d4r_emit (D4R_ZLUDA_EMIT) instead of on the local GPU. - New d4r.ini option [Kernels] NativeFp8 (default on), wired through the shim and d4r_config.py. On gfx12 the bridge serves kernels/<arch>-fp8 when it is on. - The bridge picks the KFD GPU with the most SIMDs (so an iGPU is not chosen over a discrete GPU); D4R_GPU_ARCH overrides it. - kernels/build.sh and package_release.sh build gfx1100-gfx1103 and gfx1200/gfx1201 (plus -fp8 variants), and D4R_BUNDLE_TEX takes per-target folders. Performance (RDNA3; harness byte-identical unless noted) - ZLUDA patch 0006: m16n8k8 f16 MMAs on WMMA (E; not bit-exact, 52-55 dB per layer, D4R_ZLUDA_WMMA_K8=0 disables it), per-key weight-image slots for native prep kernels (d4r_prep_key_at, d4r_prep_key_slots), and wave64 builds of MMA-free kernels (D4R_ZLUDA_WAVE64). - M: packed FP8 encoding, conflict-free patch-merge reads, scalar weight loads in enc1, and weight prep once per weight set for the six enc3 tube blocks. - K: CU mode for the six non-wide pwin layers. - Texture kernels: every DLSS flag variant is built natively (24 K output, 8 M post, 4 M downsample, 4 M enc0), not only Townfall's; post and K output run as wave64. - M direct output: rrlite_downsample_kernel writes the shim's output buffer; the shim's default direct-output presets are now 11,13. Tools - dump_runner takes the prep grid from d4r_prep_blocks and refuses texture/surface dumps. - New pwin_check.py, swin_check.py, model_enc3.py and tail_check.hip. Docs: README GPU table and RDNA4 notes, building (patches 0002-0007, d4r_emit), native-kernels (RDNA4 section), performance (new numbers), packaging README. Version 0.1.2.
This commit is contained in:
1 parent
f310d52b69
commit
4334a0c3c4
42 files changed
+3068
-529
No files matched your search
@@ -3,6 +3,10 @@
|
||||
// only plain clang builtins are used (compiled with -nogpuinc -nogpulib).
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
#include "../common/wmma_layout.h"
|
||||
#if defined(D4R_TEX_FP8) && D4R_WMMA_LAYOUT != 12 && defined(__HIP_DEVICE_COMPILE__)
|
||||
#error "D4R_TEX_FP8 needs the gfx12 WMMA layout"
|
||||
#endif
|
||||
|
||||
#pragma clang fp contract(off)
|
||||
|
||||
@@ -65,6 +69,75 @@ __attribute__((device)) static inline f8v wmma(u8v a, u8v b, f8v c)
|
||||
return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(h16, a), __builtin_bit_cast(h16, b), c);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- gfx12 m16n8k32 e4m3 steps
|
||||
// On gfx12 the GEMM tails do a k32 step as ZLUDA's gfx12 lowering of NVIDIA's m16n8k32 e4m3 MMA does, so they
|
||||
// stay bit-identical to ZLUDA's compile of the kernel they replace: a lane's k16 operand holds this half's
|
||||
// e4m3 k 8h .. 8h + 7 (ZLUDA's gfx12 slot order), either widened as above with C * 2^-16 inside the f32
|
||||
// WMMAs (d4r_wmma12_*), or with D4R_TEX_FP8 (ZLUDA's D4R_ZLUDA_WMMA_FP8_NATIVE) as the bytes of a native FP8
|
||||
// WMMA with no scaling (d4r_wmma12f8_*). (The gfx11 tails use operand16 and wmma above.)
|
||||
#if D4R_WMMA_LAYOUT == 12
|
||||
// 8 e4m3 bytes (k in order) widened to value * 2^-8 f16 pairs, k in order
|
||||
__attribute__((device)) static inline u4v widen8(uint2_t d)
|
||||
{
|
||||
uint32_t a0, a1, b0, b1;
|
||||
e4m3x4_scaled(d.x, a0, a1);
|
||||
e4m3x4_scaled(d.y, b0, b1);
|
||||
return (u4v){a0, a1, b0, b1};
|
||||
}
|
||||
#ifdef D4R_TEX_FP8
|
||||
typedef uint2_t kop;
|
||||
typedef u4v kslot; // weight image slot: the 16 e4m3 bytes of one k16 operand row
|
||||
__attribute__((device)) static inline kop kop_from8(uint2_t d)
|
||||
{
|
||||
return d;
|
||||
}
|
||||
__attribute__((device)) static inline kslot kslot_from16(u4v d)
|
||||
{
|
||||
return d;
|
||||
}
|
||||
#else
|
||||
typedef u4v kop;
|
||||
typedef u8v kslot; // weight image slot: the 16 widened values of one k16 operand row, k in order
|
||||
__attribute__((device)) static inline kop kop_from8(uint2_t d)
|
||||
{
|
||||
return widen8(d);
|
||||
}
|
||||
__attribute__((device)) static inline kslot kslot_from16(u4v d)
|
||||
{
|
||||
const u4v lo = widen8((uint2_t){d[0], d[1]}), hi = widen8((uint2_t){d[2], d[3]});
|
||||
return (u8v){lo[0], lo[1], lo[2], lo[3], hi[0], hi[1], hi[2], hi[3]};
|
||||
}
|
||||
#endif
|
||||
// this half's k16 operand from 16 consecutive e4m3 bytes of NVIDIA's k order
|
||||
__attribute__((device)) static inline kop kop_from16(u4v d)
|
||||
{
|
||||
return kop_from8(lane_id() >= 16 ? (uint2_t){d[2], d[3]} : (uint2_t){d[0], d[1]});
|
||||
}
|
||||
// this half's operand from slot idx of a weight image
|
||||
__attribute__((device)) static inline kop kop_image(const kslot* img, int idx)
|
||||
{
|
||||
return ((const kop*)img)[2 * idx + (lane_id() >> 4)];
|
||||
}
|
||||
// f32 C (true scale) + one k32 step
|
||||
__attribute__((device)) static inline f8v k32_e4m3(f8v c, const kop& a0, const kop& b0, const kop& a1, const kop& b1)
|
||||
{
|
||||
#if defined(D4R_TEX_FP8) && defined(__GFX12__)
|
||||
typedef int i2v __attribute__((ext_vector_type(2)));
|
||||
c = __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12(__builtin_bit_cast(i2v, a0), __builtin_bit_cast(i2v, b0), c);
|
||||
return __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12(__builtin_bit_cast(i2v, a1), __builtin_bit_cast(i2v, b1), c);
|
||||
#elif defined(D4R_TEX_FP8)
|
||||
// layout shim, like ZLUDA's: each FP8 WMMA as the scaled widening (exact powers of two)
|
||||
c = wm_mma_zluda(widen8(a0), widen8(b0), c * 0x1p-16f) * 0x1p16f;
|
||||
return wm_mma_zluda(widen8(a1), widen8(b1), c * 0x1p-16f) * 0x1p16f;
|
||||
#else
|
||||
c = c * 0x1p-16f;
|
||||
c = wm_mma_zluda(a0, b0, c);
|
||||
c = wm_mma_zluda(a1, b1, c);
|
||||
return c * 0x1p16f;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
// ZLUDA's f16x2_to_e4m3x2_satfinite_bits: RNE satfinite e4m3 of both halves, low half -> low byte.
|
||||
__attribute__((device)) static inline uint32_t e4m3x2(uint32_t bits)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user