Correct M attention precision and validate every temporal transformer block
This commit is contained in:
Codex committed
1 parent
e15f6a73a6
commit
b8d801ae4c
14 files changed
+623
-18
No files matched your search
+8
-2
@@ -50,7 +50,7 @@ foreach(family k m)
|
|||||||
list(APPEND native_flags -DD4R_K_FP16_BASELINE)
|
list(APPEND native_flags -DD4R_K_FP16_BASELINE)
|
||||||
endif()
|
endif()
|
||||||
if(family STREQUAL "m" AND D4R_M_FP16_BASELINE)
|
if(family STREQUAL "m" AND D4R_M_FP16_BASELINE)
|
||||||
list(APPEND native_flags -DSWIN_EXACT)
|
list(APPEND native_flags -DSWIN_EXACT -DSWIN_EXACT_PV)
|
||||||
endif()
|
endif()
|
||||||
if(flags_line)
|
if(flags_line)
|
||||||
separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}")
|
separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}")
|
||||||
@@ -68,7 +68,7 @@ foreach(family k m)
|
|||||||
endforeach()
|
endforeach()
|
||||||
endforeach()
|
endforeach()
|
||||||
add_custom_target(d4r_native_gfx1201_compile ALL DEPENDS ${native_objects})
|
add_custom_target(d4r_native_gfx1201_compile ALL DEPENDS ${native_objects})
|
||||||
foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe)
|
foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_mma_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe)
|
||||||
add_executable(d4r_${probe} tools/windows/${probe}.cpp)
|
add_executable(d4r_${probe} tools/windows/${probe}.cpp)
|
||||||
target_compile_definitions(d4r_${probe} PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
|
target_compile_definitions(d4r_${probe} PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
|
||||||
target_include_directories(d4r_${probe} PRIVATE "${D4R_HIP_ROOT}/include")
|
target_include_directories(d4r_${probe} PRIVATE "${D4R_HIP_ROOT}/include")
|
||||||
@@ -147,6 +147,7 @@ install(FILES tools/windows/k_ptx_validate.py DESTINATION .)
|
|||||||
install(FILES tools/windows/frame_compare.py DESTINATION .)
|
install(FILES tools/windows/frame_compare.py DESTINATION .)
|
||||||
install(FILES tools/windows/m_replay_validate.py DESTINATION .)
|
install(FILES tools/windows/m_replay_validate.py DESTINATION .)
|
||||||
install(FILES tools/windows/m_ptx_validate.py DESTINATION .)
|
install(FILES tools/windows/m_ptx_validate.py DESTINATION .)
|
||||||
|
install(FILES tools/windows/m_temporal_validate.py tools/windows/m_capture_compare.py DESTINATION .)
|
||||||
install(FILES scripts/windows/test-m-replay.ps1 DESTINATION .)
|
install(FILES scripts/windows/test-m-replay.ps1 DESTINATION .)
|
||||||
install(FILES scripts/windows/test-m-captures.ps1 DESTINATION .)
|
install(FILES scripts/windows/test-m-captures.ps1 DESTINATION .)
|
||||||
install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py
|
install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py
|
||||||
@@ -182,6 +183,11 @@ if(EXISTS "${D4R_ZLUDA_ROOT}/nvcuda.dll")
|
|||||||
set_tests_properties(cuda_${context} PROPERTIES TIMEOUT 180 LABELS "hardware;zluda")
|
set_tests_properties(cuda_${context} PROPERTIES TIMEOUT 180 LABELS "hardware;zluda")
|
||||||
endforeach()
|
endforeach()
|
||||||
if(EXISTS "${D4R_ZLUDA_ROOT}/build-info.json")
|
if(EXISTS "${D4R_ZLUDA_ROOT}/build-info.json")
|
||||||
|
add_test(NAME cuda_f16_mma_reference COMMAND d4r_cuda_mma_probe
|
||||||
|
--hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll")
|
||||||
|
set_tests_properties(cuda_f16_mma_reference PROPERTIES TIMEOUT 120
|
||||||
|
LABELS "hardware;zluda;mma;reference"
|
||||||
|
ENVIRONMENT "D4R_ZLUDA_WMMA=1;D4R_ZLUDA_WMMA_F16_REFERENCE=1")
|
||||||
add_test(NAME cuda_images COMMAND d4r_cuda_image_probe
|
add_test(NAME cuda_images COMMAND d4r_cuda_image_probe
|
||||||
--hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32)
|
--hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32)
|
||||||
set_tests_properties(cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors")
|
set_tests_properties(cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors")
|
||||||
|
|||||||
@@ -13,8 +13,12 @@ frames on Windows. M now passes all five synthetic and real-weight layers agains
|
|||||||
NumPy and independent PTX, and four native CUDA frames. The standalone D3D12
|
NumPy and independent PTX, and four native CUDA frames. The standalone D3D12
|
||||||
harness now produces four K and four M frames using imported VRAM and shared
|
harness now produces four K and four M frames using imported VRAM and shared
|
||||||
fences, with bit-exact RGB agreement against the corresponding CUDA harnesses.
|
fences, with bit-exact RGB agreement against the corresponding CUDA harnesses.
|
||||||
Game command-list/queue integration remains in progress. The fully translated M
|
Game command-list/queue integration remains in progress. A full temporal M scan
|
||||||
comparison is being rerun with the correct WMMA options and a writable JIT cache.
|
found rare attention P/V accumulation mismatches that the earlier twelve-block
|
||||||
|
sampling missed. `SWIN_EXACT_PV` fixes the native baseline; ZLUDA patch `0013`
|
||||||
|
adds the corresponding opt-in f16 reference accumulation. All spatial blocks in
|
||||||
|
40 actual native M launches now exactly match NumPy. Four complete native M RGB
|
||||||
|
frames exactly match fully translated M with the corrected reference option.
|
||||||
Do not integrate NGX until the integer PTX workload is stable on the real GPU.
|
Do not integrate NGX until the integer PTX workload is stable on the real GPU.
|
||||||
|
|
||||||
| Gate | Status |
|
| Gate | Status |
|
||||||
@@ -25,11 +29,70 @@ Do not integrate NGX until the integer PTX workload is stable on the real GPU.
|
|||||||
| M3: Windows NGX initialization and simple DLSS path | Init, SR capabilities, Create/Evaluate PASS; E, K and M CUDA harnesses execute |
|
| M3: Windows NGX initialization and simple DLSS path | Init, SR capabilities, Create/Evaluate PASS; E, K and M CUDA harnesses execute |
|
||||||
| M4: D3D12 / HIP external memory and fence round trip | PASS with TheRock; stable 7.2 has mapped-view leak |
|
| M4: D3D12 / HIP external memory and fence round trip | PASS with TheRock; stable 7.2 has mapped-view leak |
|
||||||
| M5: independently validated gfx12 WMMA backend | PASS: raw + legacy adapter + upstream layout, max abs/relative error 0 |
|
| M5: independently validated gfx12 WMMA backend | PASS: raw + legacy adapter + upstream layout, max abs/relative error 0 |
|
||||||
| M6: K layers, full transformer and image validation | CUDA harness PASS: 11 real-weight layers, native launches for all layers, four finite frames, 81.85–92.91 dB versus translated K; D3D12 integration pending |
|
| M6: K layers, full transformer and image validation | PASS: 11 real-weight layers, 44 native launches, four D3D12 frames bit-exact against native CUDA, 81.85–92.91 dB versus translated K |
|
||||||
| M7: M FP16-equivalent baseline and full transformer | CUDA harness PASS: 5 real-weight layers exactly match NumPy/PTX, 4 finite frames, 40 native Swin launches; D3D12 and full translated M comparison pending |
|
| M7: M FP16-equivalent baseline and full transformer | PASS: all blocks of 40 native launches exactly match NumPy; targeted independent PTX regression matches; four D3D12 frames bit-exact against corrected translated M |
|
||||||
| M8: standalone Windows D3D12 NGX harness | PASS at explicit submitted queue boundaries: K/M, four frames each, no shim CPU image copies, bit-exact RGB against CUDA harness |
|
| M8: standalone Windows D3D12 NGX harness | PASS at explicit submitted queue boundaries: K/M, four frames each, no shim CPU image copies, bit-exact RGB against CUDA harness |
|
||||||
| M9: OptiScaler integration, profiling, installation | Pending |
|
| M9: OptiScaler integration, profiling, installation | Pending |
|
||||||
|
|
||||||
|
## M temporal precision regression (2026-09-30)
|
||||||
|
|
||||||
|
The earlier finite-frame and sampled-layer results did not cover every spatial
|
||||||
|
block. The second enc1 launch has a cancellation-sensitive P/V dot in block
|
||||||
|
`0,4`: the old native f32 WMMA route differed by up to 21 e4m3 code steps.
|
||||||
|
NumPy and the independent PTX interpreter agree exactly on that block.
|
||||||
|
`SWIN_EXACT_PV` stores each wave's sixteen P rows and V rows in LDS, sums sixteen
|
||||||
|
f16 products in f64, rounds directly to f16, then applies the existing e4m3
|
||||||
|
quantisation. Other GEMMs retain the gfx12 WMMA layout and FP16 widening.
|
||||||
|
The flag is enabled by `D4R_M_FP16_BASELINE`; upstream fast/Linux defaults are
|
||||||
|
preserved when that flag is absent. Intermediate snapshots are compiled only
|
||||||
|
with `D4R_SWIN_DIAGNOSTICS`; they are absent from packaged kernels.
|
||||||
|
|
||||||
|
The translated reference also differed: first frame, fourth enc3_tube launch,
|
||||||
|
block `0,0`, thirty stored codes, maximum fifteen e4m3 steps. Its inputs match
|
||||||
|
the corrected native launch exactly. Patch `0013` adds
|
||||||
|
`D4R_ZLUDA_WMMA_F16_REFERENCE=1` for gfx12 f16-accumulator MMA helpers, retaining
|
||||||
|
the established fragment gathers, pair fusion and conversions. It uses f64
|
||||||
|
sums and integer f16 RNE encoding, including subnormal bits. FP8 GEMMs continue
|
||||||
|
using the existing native WMMA widening. The default upstream helper behavior
|
||||||
|
remains available with this option unset or zero. Windows diagnostics select
|
||||||
|
the reference option by default. All d4r codegen options enter the JIT cache key.
|
||||||
|
|
||||||
|
This is a match to the independent mathematical NumPy/PTX baseline. NVIDIA's
|
||||||
|
[PTX floating-point MMA specification](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-instructions-mma)
|
||||||
|
leaves accumulation order, rounding and subnormal handling unspecified; these
|
||||||
|
checks do not establish bit-exact agreement with NVIDIA hardware.
|
||||||
|
|
||||||
|
Recorded results on the RX 9070 XT / TheRock runtime:
|
||||||
|
|
||||||
|
* Public `cuda_mma_probe`: 512 synthetic matrix cases, 65,536 f16 outputs,
|
||||||
|
zero differing bits with reference accumulation. The old fast WMMA route
|
||||||
|
differs in 23,308 outputs on the same cancellation/subnormal stress fixture.
|
||||||
|
* `m-native-exact-pv-temporal-reference`: all spatial blocks of 40 actual native
|
||||||
|
launches, every stored e4m3 value identical to NumPy, max abs/relative error 0.
|
||||||
|
* `m-exact-pv-enc1-regression-ptx`: original problem block `0,4`, independent
|
||||||
|
PTX versus native and versus NumPy both max absolute/relative error 0.
|
||||||
|
* `m-exact-pv-complete-f16-reference.log`: four complete RGB frames against
|
||||||
|
corrected fully translated M, max absolute/relative error 0, PSNR infinite.
|
||||||
|
* `m-translated-f16-temporal-reference`: all blocks of all forty translated
|
||||||
|
M launches exactly match NumPy; no sampled-window exemption.
|
||||||
|
* `m-temporal-final-ctest.log`: 11/11 native Windows hardware tests PASS.
|
||||||
|
Pinned fresh-source application of all twelve ZLUDA patches PASS.
|
||||||
|
* `d3d12-m-exact-pv-final.zip`: four finite frames, forty native launches,
|
||||||
|
no shim CPU image copies, frame age zero. This remains the explicit queue
|
||||||
|
boundary harness; open game command-list support is the next gate.
|
||||||
|
|
||||||
|
All captures, extracted PTX and real weights remain in ignored private result
|
||||||
|
directories. To repeat the full temporal scan without editing sources:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:PYTHONPATH="$PWD/.tools/python/vendor"
|
||||||
|
python tools/windows/m_temporal_validate.py --capture-dir test-results/ngx-m-exact-pv-capture/captures --output-dir test-results/m-temporal-reference --exact
|
||||||
|
```
|
||||||
|
|
||||||
|
The game selected for the next integration test is Silent Hill 2 at
|
||||||
|
`D:\Games\SILENT HILL 2\SHProto\Binaries\Win64\SHProto-Win64-Shipping.exe`.
|
||||||
|
No game files have been modified at this milestone.
|
||||||
|
|
||||||
## Source baseline (checked 2026-09-29)
|
## Source baseline (checked 2026-09-29)
|
||||||
|
|
||||||
* d4r: `ed1ab60`, main, 14 commits; https://github.com/countervolts/d4r.
|
* d4r: `ed1ab60`, main, 14 commits; https://github.com/countervolts/d4r.
|
||||||
|
|||||||
+54
-2
@@ -5,6 +5,20 @@
|
|||||||
// CIN patch-expand input channels (0 = none): decoder input = f16(q8(expand(low-res CIN)) + skip)
|
// CIN patch-expand input channels (0 = none): decoder input = f16(q8(expand(low-res CIN)) + skip)
|
||||||
#pragma once
|
#pragma once
|
||||||
#include "swin_common.h"
|
#include "swin_common.h"
|
||||||
|
#ifdef D4R_SWIN_DIAGNOSTICS
|
||||||
|
extern "C" {
|
||||||
|
__device__ int d4r_swin_debug_block[2] = {0, 4};
|
||||||
|
__device__ half_t d4r_swin_debug_values[4 * 64 * 128];
|
||||||
|
}
|
||||||
|
template <int C, int AST> __device__ __forceinline__ void swin_snapshot(
|
||||||
|
const half_t* activation, int stage, int bx, int by, int tid, bool pair_order) {
|
||||||
|
if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1] && tid < 64)
|
||||||
|
for (int c = 0; c < C; ++c)
|
||||||
|
d4r_swin_debug_values[(stage * 64 + tid) * 128 + c] =
|
||||||
|
activation[tid * AST + (pair_order ? gperm(c) : apos(c))];
|
||||||
|
__syncthreads();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
// Immediate lane masks need only a compile-time integer, not the host C++
|
// Immediate lane masks need only a compile-time integer, not the host C++
|
||||||
// standard library (which is absent from device-only Windows HIP builds).
|
// standard library (which is absent from device-only Windows HIP builds).
|
||||||
@@ -79,7 +93,12 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
static_assert(NW % NH == 0 && NW / NH <= 4, "waves per head");
|
static_assert(NW % NH == 0 && NW / NH <= 4, "waves per head");
|
||||||
constexpr int NPL = L::NPL, NT = L::NT, INNER = L::INNER;
|
constexpr int NPL = L::NPL, NT = L::NT, INNER = L::INNER;
|
||||||
constexpr int AST = (C > INNER ? C : INNER) + 8;
|
constexpr int AST = (C > INNER ? C : INNER) + 8;
|
||||||
constexpr int GST = 40, QST = 40, PST = 24;
|
constexpr int GST = 40, PST = 24;
|
||||||
|
#ifdef SWIN_EXACT_PV
|
||||||
|
constexpr int QST = 56;
|
||||||
|
#else
|
||||||
|
constexpr int QST = 40;
|
||||||
|
#endif
|
||||||
#ifndef SWIN_MLP_R
|
#ifndef SWIN_MLP_R
|
||||||
#define SWIN_MLP_R(NW) ((NW) / 2)
|
#define SWIN_MLP_R(NW) ((NW) / 2)
|
||||||
#endif
|
#endif
|
||||||
@@ -294,6 +313,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
__syncthreads();
|
__syncthreads();
|
||||||
|
|
||||||
// ------------------------------------------------ stage 2: Q/V projection + attention (wave = head)
|
// ------------------------------------------------ stage 2: Q/V projection + attention (wave = head)
|
||||||
|
#ifdef D4R_SWIN_DIAGNOSTICS
|
||||||
|
swin_snapshot<C, AST>(A, 0, bx, by, tid, false);
|
||||||
|
#endif
|
||||||
{
|
{
|
||||||
constexpr int MW = 4 / (NW / NH); // windows per wave
|
constexpr int MW = 4 / (NW / NH); // windows per wave
|
||||||
const int h = NW == NH ? wv : wv % NH, m0 = NW == NH ? 0 : MW * (wv / NH);
|
const int h = NW == NH ? wv : wv % NH, m0 = NW == NH ? 0 : MW * (wv / NH);
|
||||||
@@ -400,6 +422,13 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
Qs[mrow(2 * k + 1) * PST + l16] = pw[1];
|
Qs[mrow(2 * k + 1) * PST + l16] = pw[1];
|
||||||
}
|
}
|
||||||
wave_sync();
|
wave_sync();
|
||||||
|
#ifdef SWIN_EXACT_PV
|
||||||
|
half_t* Vs = Qs + 16 * PST;
|
||||||
|
for (int nt = 0; nt < 2; ++nt)
|
||||||
|
for (int i = 0; i < 8; ++i)
|
||||||
|
Vs[mrow(i) * 32 + 16 * nt + l16] = vp[m][nt][i / 2][i & 1];
|
||||||
|
wave_sync();
|
||||||
|
#endif
|
||||||
const sop pa = lds16(Qs + l16 * PST);
|
const sop pa = lds16(Qs + l16 * PST);
|
||||||
#pragma unroll
|
#pragma unroll
|
||||||
for (int nt = 0; nt < 2; ++nt)
|
for (int nt = 0; nt < 2; ++nt)
|
||||||
@@ -422,7 +451,19 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
}
|
}
|
||||||
const h16 vb = __builtin_bit_cast(h16, vw);
|
const h16 vb = __builtin_bit_cast(h16, vw);
|
||||||
#endif
|
#endif
|
||||||
const f8v o = wmma(pa, vb, splat(0.0f));
|
f8v o = wmma(pa, vb, splat(0.0f));
|
||||||
|
#ifdef SWIN_EXACT_PV
|
||||||
|
// FP16 P/V products need more than 24 significant bits in a
|
||||||
|
// cancellation-heavy dot. An f32 WMMA can round to the wrong
|
||||||
|
// FP16 neighbour at a midpoint, then flip the FP8 output code.
|
||||||
|
// This baseline sums the 16 FP16 products before one FP16 round.
|
||||||
|
for (int i = 0; i < 8; ++i) {
|
||||||
|
double dot = 0.0;
|
||||||
|
for (int k = 0; k < 16; ++k)
|
||||||
|
dot += (double)Qs[mrow(i) * PST + k] * (double)Vs[k * 32 + 16 * nt + l16];
|
||||||
|
o[i] = (float)(half_t)dot;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
const int ocol = apos(32 * h + ginv(16 * nt + l16));
|
const int ocol = apos(32 * h + ginv(16 * nt + l16));
|
||||||
#pragma unroll
|
#pragma unroll
|
||||||
for (int i = 0; i < 8; i += 2)
|
for (int i = 0; i < 8; i += 2)
|
||||||
@@ -438,6 +479,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
__syncthreads();
|
__syncthreads();
|
||||||
|
|
||||||
// ------------------------------------------------ stage 3: output projection + residual
|
// ------------------------------------------------ stage 3: output projection + residual
|
||||||
|
#ifdef D4R_SWIN_DIAGNOSTICS
|
||||||
|
swin_snapshot<INNER, AST>(A, 3, bx, by, tid, false);
|
||||||
|
#endif
|
||||||
#pragma unroll
|
#pragma unroll
|
||||||
for (int kc = 0; kc < NH; ++kc)
|
for (int kc = 0; kc < NH; ++kc)
|
||||||
{
|
{
|
||||||
@@ -471,6 +515,11 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
if (tid < 64)
|
if (tid < 64)
|
||||||
{
|
{
|
||||||
// the row as 128-bit LDS loads: element 8 j + c of n8 tile j
|
// the row as 128-bit LDS loads: element 8 j + c of n8 tile j
|
||||||
|
#ifdef D4R_SWIN_DIAGNOSTICS
|
||||||
|
if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1])
|
||||||
|
for (int c = 0; c < C; ++c)
|
||||||
|
d4r_swin_debug_values[(64 + tid) * 128 + c] = A[tid * AST + gperm(c)];
|
||||||
|
#endif
|
||||||
h8 row[L::NT8];
|
h8 row[L::NT8];
|
||||||
#pragma unroll
|
#pragma unroll
|
||||||
for (int j = 0; j < L::NT8; ++j)
|
for (int j = 0; j < L::NT8; ++j)
|
||||||
@@ -529,6 +578,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
|
|||||||
__syncthreads();
|
__syncthreads();
|
||||||
|
|
||||||
// ------------------------------------------------ stage 4: MLP, R chunks per round
|
// ------------------------------------------------ stage 4: MLP, R chunks per round
|
||||||
|
#ifdef D4R_SWIN_DIAGNOSTICS
|
||||||
|
swin_snapshot<C, AST>(A, 2, bx, by, tid, false);
|
||||||
|
#endif
|
||||||
{
|
{
|
||||||
constexpr int WPC = NW / R, MPW = 4 / WPC; // waves per chunk, fc1 m tiles per wave
|
constexpr int WPC = NW / R, MPW = 4 / WPC; // waves per chunk, fc1 m tiles per wave
|
||||||
const int cw = wv / WPC, mb = MPW * (wv % WPC);
|
const int cw = wv / WPC, mb = MPW * (wv % WPC);
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ def layout(C, heads, pre=0):
|
|||||||
L['end'] = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1)
|
L['end'] = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1)
|
||||||
return L
|
return L
|
||||||
|
|
||||||
def swin_block(P, x0, C, heads, pre=0):
|
def swin_block(P, x0, C, heads, pre=0, stages=None):
|
||||||
"""x0: 64 x C natural (f16 values). Returns (x2 pair order, out natural fp8 values)."""
|
"""x0: 64 x C natural (f16 values). Returns (x2 pair order, out natural fp8 values)."""
|
||||||
fp8 = lambda T: E4M3[P.wbuf[T]]
|
fp8 = lambda T: E4M3[P.wbuf[T]]
|
||||||
L = layout(C, heads, pre)
|
L = layout(C, heads, pre)
|
||||||
@@ -116,6 +116,8 @@ def swin_block(P, x0, C, heads, pre=0):
|
|||||||
Wo[32 * s:32 * s + 32] = woff_table(L['head'][s] + 64 * C + 512, 0, 512, 32, C)
|
Wo[32 * s:32 * s + 32] = woff_table(L['head'][s] + 64 * C + 512, 0, 512, 32, C)
|
||||||
x1 = mma_chain(pair_to_nat(q8(O)), fp8(Wo), C0=f16(nat_to_pair(x0) + P.f16vec(L['bo'], C)[None, :]))
|
x1 = mma_chain(pair_to_nat(q8(O)), fp8(Wo), C0=f16(nat_to_pair(x0) + P.f16vec(L['bo'], C)[None, :]))
|
||||||
h2 = pair_to_nat(rms_from_sum(x1, tree_norm2(x1), P.f16vec(L['g2'], C)))
|
h2 = pair_to_nat(rms_from_sum(x1, tree_norm2(x1), P.f16vec(L['g2'], C)))
|
||||||
|
if stages is not None:
|
||||||
|
stages.update(h1=h1, x1=pair_to_nat(x1), h2=h2, O=pair_to_nat(q8(O)))
|
||||||
x2 = f16(x1 + P.f16vec(L['b2'], C)[None, :])
|
x2 = f16(x1 + P.f16vec(L['b2'], C)[None, :])
|
||||||
idx = gp(np.arange(32))
|
idx = gp(np.arange(32))
|
||||||
for c, B1 in enumerate(L['b1']):
|
for c, B1 in enumerate(L['b1']):
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
|
||||||
|
index 5a54417..90c0034 100644
|
||||||
|
--- a/llvm_zluda/src/compile.rs
|
||||||
|
+++ b/llvm_zluda/src/compile.rs
|
||||||
|
@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
|
||||||
|
let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
|
||||||
|
if family == 12 {
|
||||||
|
flags |= 2;
|
||||||
|
+ // Explicit diagnostic/correctness mode; retain upstream's fast default.
|
||||||
|
+ if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") {
|
||||||
|
+ flags |= 8;
|
||||||
|
+ }
|
||||||
|
// native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
|
||||||
|
if flags & 1 != 0 && wmma_fp8_native_requested() {
|
||||||
|
flags |= 4;
|
||||||
|
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
|
||||||
|
index 07238db..d262fc6 100644
|
||||||
|
--- a/llvm_zluda/src/lib.cpp
|
||||||
|
+++ b/llvm_zluda/src/lib.cpp
|
||||||
|
@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M)
|
||||||
|
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
|
||||||
|
// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
|
||||||
|
// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
|
||||||
|
+// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs.
|
||||||
|
// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
|
||||||
|
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
|
||||||
|
{
|
||||||
|
@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
|
||||||
|
const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
|
||||||
|
const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
|
||||||
|
const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
|
||||||
|
+ const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16");
|
||||||
|
Helpers H;
|
||||||
|
H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
|
||||||
|
H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
|
||||||
|
- H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
|
||||||
|
+ H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2);
|
||||||
|
H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
|
||||||
|
H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
|
||||||
|
H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
|
||||||
|
- H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
|
||||||
|
+ H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
|
||||||
|
{
|
||||||
|
Type *F = Type::getFloatTy(Context);
|
||||||
|
Type *V4F = FixedVectorType::get(F, 4);
|
||||||
|
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
|
||||||
|
index 4ff25dd..36bd6b1 100644
|
||||||
|
--- a/ptx/lib/zluda_ptx_impl.cpp
|
||||||
|
+++ b/ptx/lib/zluda_ptx_impl.cpp
|
||||||
|
@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
+// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may
|
||||||
|
+// reach the other f16 neighbour around cancellation-heavy rounding midpoints.
|
||||||
|
+// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and
|
||||||
|
+// round directly to f16 once, matching d4r's independent NumPy/PTX reference.
|
||||||
|
+// The e4m3 and f32-shadow paths continue using native WMMA.
|
||||||
|
+__device__ static double wmma12_half_value(uint32_t bits)
|
||||||
|
+{
|
||||||
|
+ const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10;
|
||||||
|
+ const double value = exponent == 0 ? double(magnitude) * 0x1p-24 :
|
||||||
|
+ double(std::bit_cast<float>(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) :
|
||||||
|
+ ((magnitude << 13) + 0x38000000u)));
|
||||||
|
+ return bits & 0x8000u ? -value : value;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+// Preserve f16 subnormal result bits even when the target's default f16
|
||||||
|
+// arithmetic mode flushes them. No intermediate f32 rounding is involved.
|
||||||
|
+__device__ static uint32_t wmma12_half_round(double value)
|
||||||
|
+{
|
||||||
|
+ const uint64_t bits = std::bit_cast<uint64_t>(value);
|
||||||
|
+ const uint32_t sign = uint32_t(bits >> 48) & 0x8000u;
|
||||||
|
+ const uint32_t exponent = uint32_t((bits >> 52) & 2047u);
|
||||||
|
+ const uint64_t fraction = bits & 0xfffffffffffffu;
|
||||||
|
+ if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u);
|
||||||
|
+ const int power = int(exponent) - 1023;
|
||||||
|
+ if (power < -25) return sign;
|
||||||
|
+ if (power > 15) return sign | 0x7c00u;
|
||||||
|
+ const unsigned shift = power < -14 ? unsigned(28 - power) : 42u;
|
||||||
|
+ const uint64_t significand = fraction | 0x10000000000000ull;
|
||||||
|
+ uint32_t code = uint32_t(significand >> shift);
|
||||||
|
+ const uint64_t tail = significand & ((uint64_t(1) << shift) - 1);
|
||||||
|
+ const uint64_t midpoint = uint64_t(1) << (shift - 1);
|
||||||
|
+ code += tail > midpoint || (tail == midpoint && (code & 1u));
|
||||||
|
+ if (power >= -14) code += uint32_t(power + 14) << 10;
|
||||||
|
+ return sign | code;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired)
|
||||||
|
+{
|
||||||
|
+ const bool high = wmma12_half() != 0;
|
||||||
|
+ const uint2::Native_vec_ own = paired && high ? c1 : c0;
|
||||||
|
+ const uint2::Native_vec_ send = paired && !high ? c1 : c0;
|
||||||
|
+ const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
|
||||||
|
+ const uint2::Native_vec_ even = paired && high ? other : own;
|
||||||
|
+ const uint2::Native_vec_ odd = paired && high ? own : other;
|
||||||
|
+ return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u),
|
||||||
|
+ (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)};
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c)
|
||||||
|
+{
|
||||||
|
+ // x/y: packed even rows, z/w: packed odd rows, in this half-wave.
|
||||||
|
+ wmma_uint4 result = {};
|
||||||
|
+ const uint32_t half = wmma12_half();
|
||||||
|
+ for (uint32_t i = 0; i < 8; ++i)
|
||||||
|
+ {
|
||||||
|
+ const uint32_t row = i + 8 * half;
|
||||||
|
+ double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu);
|
||||||
|
+ for (uint32_t khalf = 0; khalf < 2; ++khalf)
|
||||||
|
+ for (uint32_t j = 0; j < 4; ++j)
|
||||||
|
+ {
|
||||||
|
+ const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]);
|
||||||
|
+ const uint32_t other = wmma12_other_half(b[j]);
|
||||||
|
+ const uint32_t bv = half == khalf ? b[j] : other;
|
||||||
|
+ sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu);
|
||||||
|
+ sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16);
|
||||||
|
+ }
|
||||||
|
+ // Do not round through f32: that can reintroduce the midpoint error.
|
||||||
|
+ result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1));
|
||||||
|
+ }
|
||||||
|
+ return result;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
|
||||||
|
// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
|
||||||
|
__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
|
||||||
|
@@ -1959,6 +2031,22 @@ extern "C"
|
||||||
|
return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
|
||||||
|
}
|
||||||
|
|
||||||
|
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
|
||||||
|
+ {
|
||||||
|
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false));
|
||||||
|
+ const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)};
|
||||||
|
+ return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y};
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
|
||||||
|
+ {
|
||||||
|
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true));
|
||||||
|
+ const bool high = wmma12_half() != 0;
|
||||||
|
+ const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z);
|
||||||
|
+ const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w);
|
||||||
|
+ return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1};
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
|
||||||
|
{
|
||||||
|
return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
|
||||||
@@ -16,7 +16,7 @@ foreach ($layer in @('enc1','enc2','enc3_tube','dec2','dec1')) {
|
|||||||
}
|
}
|
||||||
& python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll
|
& python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll
|
||||||
if ($LASTEXITCODE) { throw "Native manifest generation failed ($LASTEXITCODE)" }
|
if ($LASTEXITCODE) { throw "Native manifest generation failed ($LASTEXITCODE)" }
|
||||||
@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; FP16 widening; native FP8 disabled';
|
@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; SWIN_EXACT_PV; FP16 widening; native FP8 disabled';
|
||||||
dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash;
|
dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash;
|
||||||
validation='synthetic layers passed; actual DLSS capture/reference validation required';
|
validation='synthetic layers passed; actual DLSS capture/reference validation required';
|
||||||
modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} |
|
modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} |
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ $originalPythonPath = $env:PYTHONPATH
|
|||||||
$originalVerbose = $env:D4R_ZLUDA_VERBOSE
|
$originalVerbose = $env:D4R_ZLUDA_VERBOSE
|
||||||
$originalCache = $env:ZLUDA_CACHE_DIR
|
$originalCache = $env:ZLUDA_CACHE_DIR
|
||||||
$originalCodegen = @{}
|
$originalCodegen = @{}
|
||||||
foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) {
|
foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) {
|
||||||
$originalCodegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process')
|
$originalCodegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process')
|
||||||
}
|
}
|
||||||
$exitStatus = 1
|
$exitStatus = 1
|
||||||
@@ -103,14 +103,14 @@ function Invoke-Probe([string]$Name, [string]$Exe, [string[]]$Arguments) {
|
|||||||
try {
|
try {
|
||||||
if (!$env:ZLUDA_CACHE_DIR) { $env:ZLUDA_CACHE_DIR = Join-Path $repoRoot 'build/zluda-cache-windows' }
|
if (!$env:ZLUDA_CACHE_DIR) { $env:ZLUDA_CACHE_DIR = Join-Path $repoRoot 'build/zluda-cache-windows' }
|
||||||
$summary.zludaCacheDirectory = $env:ZLUDA_CACHE_DIR
|
$summary.zludaCacheDirectory = $env:ZLUDA_CACHE_DIR
|
||||||
foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) {
|
foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) {
|
||||||
if ($null -eq $originalCodegen[$setting]) {
|
if ($null -eq $originalCodegen[$setting]) {
|
||||||
[Environment]::SetEnvironmentVariable($setting, $(if ($setting -eq 'D4R_ZLUDA_WMMA_FP8_NATIVE') { '0' } else { '1' }), 'Process')
|
[Environment]::SetEnvironmentVariable($setting, $(if ($setting -eq 'D4R_ZLUDA_WMMA_FP8_NATIVE') { '0' } else { '1' }), 'Process')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$summary.codegen = @{}
|
$summary.codegen = @{}
|
||||||
foreach ($setting in $originalCodegen.Keys) { $summary.codegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process') }
|
foreach ($setting in $originalCodegen.Keys) { $summary.codegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process') }
|
||||||
Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE"
|
Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE f16 reference=$env:D4R_ZLUDA_WMMA_F16_REFERENCE"
|
||||||
if ($NgxOnly -and (!$NgxCore -or !$DlssDll)) { throw '-NgxOnly requires both locally supplied NVIDIA DLL paths.' }
|
if ($NgxOnly -and (!$NgxCore -or !$DlssDll)) { throw '-NgxOnly requires both locally supplied NVIDIA DLL paths.' }
|
||||||
if ($RequireNativeNetwork) {
|
if ($RequireNativeNetwork) {
|
||||||
if ($NgxMode -notin @('evaluate','d3d12') -or $Preset -notin @(11,13) -or !$env:D4R_ZLUDA_NATIVE_DIR) {
|
if ($NgxMode -notin @('evaluate','d3d12') -or $Preset -notin @(11,13) -or !$env:D4R_ZLUDA_NATIVE_DIR) {
|
||||||
@@ -197,6 +197,14 @@ try {
|
|||||||
if ($cudaOk -and ($NgxMode -eq 'evaluate' -or (Test-Path (Join-Path $ZludaRoot 'build-info.json')))) {
|
if ($cudaOk -and ($NgxMode -eq 'evaluate' -or (Test-Path (Join-Path $ZludaRoot 'build-info.json')))) {
|
||||||
$cudaOk = Invoke-Probe 'cuda-images' (Join-Path $bin 'd4r_cuda_image_probe.exe') @(
|
$cudaOk = Invoke-Probe 'cuda-images' (Join-Path $bin 'd4r_cuda_image_probe.exe') @(
|
||||||
'--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'), '--iterations', "$Iterations")
|
'--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'), '--iterations', "$Iterations")
|
||||||
|
if ($cudaOk) {
|
||||||
|
$previousF16 = $env:D4R_ZLUDA_WMMA_F16_REFERENCE
|
||||||
|
try {
|
||||||
|
$env:D4R_ZLUDA_WMMA_F16_REFERENCE = '1'
|
||||||
|
$cudaOk = Invoke-Probe 'cuda-f16-mma-reference' (Join-Path $bin 'd4r_cuda_mma_probe.exe') @(
|
||||||
|
'--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'))
|
||||||
|
} finally { $env:D4R_ZLUDA_WMMA_F16_REFERENCE = $previousF16 }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ($cudaOk) {
|
if ($cudaOk) {
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
#include "hip_api.h"
|
||||||
|
#include "cuda_api.h"
|
||||||
|
#include <array>
|
||||||
|
#include <cmath>
|
||||||
|
#include <cstring>
|
||||||
|
|
||||||
|
// Public synthetic matrix fragments, independent of NVIDIA DLLs and weights.
|
||||||
|
static constexpr char source[] = R"PTX(
|
||||||
|
.version 7.0
|
||||||
|
.target sm_75
|
||||||
|
.address_size 64
|
||||||
|
.visible .entry d4r_f16_mma_reference(
|
||||||
|
.param .u64 A, .param .u64 B, .param .u64 C, .param .u64 D)
|
||||||
|
{
|
||||||
|
.reg .b32 %r<24>;
|
||||||
|
.reg .b64 %rd<12>;
|
||||||
|
.reg .b16 %h<4>;
|
||||||
|
ld.param.u64 %rd0, [A];
|
||||||
|
ld.param.u64 %rd1, [B];
|
||||||
|
ld.param.u64 %rd2, [C];
|
||||||
|
ld.param.u64 %rd3, [D];
|
||||||
|
mov.u32 %r0, %ctaid.x;
|
||||||
|
mov.u32 %r1, %tid.x;
|
||||||
|
shr.u32 %r2, %r1, 2;
|
||||||
|
and.b32 %r3, %r1, 3;
|
||||||
|
shl.b32 %r3, %r3, 1;
|
||||||
|
mul.wide.u32 %rd4, %r0, 512;
|
||||||
|
add.u64 %rd0, %rd0, %rd4;
|
||||||
|
mul.wide.u32 %rd4, %r0, 256;
|
||||||
|
add.u64 %rd1, %rd1, %rd4;
|
||||||
|
add.u64 %rd2, %rd2, %rd4;
|
||||||
|
add.u64 %rd3, %rd3, %rd4;
|
||||||
|
mad.lo.u32 %r4, %r2, 16, %r3;
|
||||||
|
mul.wide.u32 %rd4, %r4, 2;
|
||||||
|
add.u64 %rd5, %rd0, %rd4;
|
||||||
|
ld.global.u32 %r8, [%rd5];
|
||||||
|
ld.global.u32 %r9, [%rd5+256];
|
||||||
|
ld.global.u32 %r10, [%rd5+16];
|
||||||
|
ld.global.u32 %r11, [%rd5+272];
|
||||||
|
mad.lo.u32 %r4, %r3, 8, %r2;
|
||||||
|
mul.wide.u32 %rd4, %r4, 2;
|
||||||
|
add.u64 %rd5, %rd1, %rd4;
|
||||||
|
ld.global.b16 %h0, [%rd5];
|
||||||
|
ld.global.b16 %h1, [%rd5+16];
|
||||||
|
mov.b32 %r12, {%h0,%h1};
|
||||||
|
ld.global.b16 %h2, [%rd5+128];
|
||||||
|
ld.global.b16 %h3, [%rd5+144];
|
||||||
|
mov.b32 %r13, {%h2,%h3};
|
||||||
|
mad.lo.u32 %r4, %r2, 8, %r3;
|
||||||
|
mul.wide.u32 %rd4, %r4, 2;
|
||||||
|
add.u64 %rd5, %rd2, %rd4;
|
||||||
|
ld.global.u32 %r14, [%rd5];
|
||||||
|
ld.global.u32 %r15, [%rd5+128];
|
||||||
|
mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16
|
||||||
|
{%r16,%r17}, {%r8,%r9,%r10,%r11}, {%r12,%r13}, {%r14,%r15};
|
||||||
|
add.u64 %rd5, %rd3, %rd4;
|
||||||
|
st.global.u32 [%rd5], %r16;
|
||||||
|
st.global.u32 [%rd5+128], %r17;
|
||||||
|
ret;
|
||||||
|
}
|
||||||
|
)PTX";
|
||||||
|
|
||||||
|
static uint16_t half_bits(double value) {
|
||||||
|
_Float16 half = static_cast<_Float16>(value);
|
||||||
|
uint16_t bits;
|
||||||
|
std::memcpy(&bits, &half, sizeof(bits));
|
||||||
|
return bits;
|
||||||
|
}
|
||||||
|
static double half_value(uint16_t bits) {
|
||||||
|
_Float16 half;
|
||||||
|
std::memcpy(&half, &bits, sizeof(bits));
|
||||||
|
return static_cast<double>(half);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char** argv) {
|
||||||
|
using namespace d4r::diag;
|
||||||
|
start();
|
||||||
|
try {
|
||||||
|
Args args(argc, argv);
|
||||||
|
HipApi hip(args.hip_root);
|
||||||
|
hipDeviceProp_t props{};
|
||||||
|
hip.select_gfx1201(args.device, props);
|
||||||
|
CudaApi cuda(args.cuda_dll);
|
||||||
|
cuda.check(cuda.cuInit(0), "cuInit");
|
||||||
|
CUdevice device = -1;
|
||||||
|
int count = 0;
|
||||||
|
cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
|
||||||
|
for (int i = 0; i < count; ++i) {
|
||||||
|
CUdevice candidate;
|
||||||
|
int bus, pci, domain;
|
||||||
|
cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
|
||||||
|
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
|
||||||
|
cuda.check(cuda.cuDeviceGetAttribute(&pci, 34, candidate), "PCI device");
|
||||||
|
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
|
||||||
|
if (bus == props.pciBusID && pci == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
|
||||||
|
}
|
||||||
|
if (device < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
|
||||||
|
CUcontext context = nullptr;
|
||||||
|
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, device), "primary context");
|
||||||
|
struct Context { CudaApi& api; CUdevice device; ~Context() { api.cuCtxSetCurrent(nullptr); api.cuDevicePrimaryCtxRelease_v2(device); } } context_owner{cuda, device};
|
||||||
|
cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent");
|
||||||
|
CUmodule module = nullptr;
|
||||||
|
cuda.check(cuda.cuModuleLoadData(&module, source), "MMA PTX JIT");
|
||||||
|
struct Module { CudaApi& api; CUmodule module; ~Module() { api.cuModuleUnload(module); } } module_owner{cuda, module};
|
||||||
|
CUfunction kernel = nullptr;
|
||||||
|
cuda.check(cuda.cuModuleGetFunction(&kernel, module, "d4r_f16_mma_reference"), "MMA kernel");
|
||||||
|
constexpr size_t cases = 512;
|
||||||
|
std::vector<uint16_t> a(cases * 256), b(cases * 128), c(cases * 128), d(cases * 128);
|
||||||
|
uint32_t random = 0x192a321u;
|
||||||
|
auto next = [&]() { random = random * 1664525u + 1013904223u; return random; };
|
||||||
|
for (size_t n = 0; n < cases; ++n) {
|
||||||
|
for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = uint16_t(0x2800u + (next() >> 20));
|
||||||
|
for (size_t i = 0; i < 128; ++i) {
|
||||||
|
b[n * 128 + i] = uint16_t((next() & 0x8000u) | (0x3000u + (next() >> 20)));
|
||||||
|
c[n * 128 + i] = half_bits(n % 3 ? 0. : 0.015625);
|
||||||
|
}
|
||||||
|
// Sweep large cancellation terms and a tiny surviving product
|
||||||
|
// across K positions; this exposes loss hidden by integer fixtures.
|
||||||
|
if (n < 256) {
|
||||||
|
for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = half_bits(1.);
|
||||||
|
std::fill(b.begin() + n * 128, b.begin() + (n + 1) * 128, half_bits(0.));
|
||||||
|
for (size_t col = 0; col < 8; ++col) {
|
||||||
|
b[n * 128 + (n % 16) * 8 + col] = half_bits(32768.);
|
||||||
|
b[n * 128 + ((n % 16 + 1) % 16) * 8 + col] = half_bits(-32768.);
|
||||||
|
b[n * 128 + ((n % 16 + 2 + n / 16 % 14) % 16) * 8 + col] = half_bits(0.0009765625);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
struct Allocation {
|
||||||
|
CudaApi& api; CUdeviceptr pointer = 0;
|
||||||
|
Allocation(CudaApi& api, size_t bytes) : api(api) { api.check(api.cuMemAlloc_v2(&pointer, bytes), "MMA allocate"); }
|
||||||
|
~Allocation() { if (pointer) api.cuMemFree_v2(pointer); }
|
||||||
|
} da(cuda, a.size() * 2), db(cuda, b.size() * 2), dc(cuda, c.size() * 2), dd(cuda, d.size() * 2);
|
||||||
|
cuda.check(cuda.cuMemcpyHtoD_v2(da.pointer, a.data(), a.size() * 2), "MMA A upload");
|
||||||
|
cuda.check(cuda.cuMemcpyHtoD_v2(db.pointer, b.data(), b.size() * 2), "MMA B upload");
|
||||||
|
cuda.check(cuda.cuMemcpyHtoD_v2(dc.pointer, c.data(), c.size() * 2), "MMA C upload");
|
||||||
|
std::fill(d.begin(), d.end(), uint16_t(0x7e00));
|
||||||
|
cuda.check(cuda.cuMemcpyHtoD_v2(dd.pointer, d.data(), d.size() * 2), "MMA NaN sentinel");
|
||||||
|
void* parameters[] = {&da.pointer, &db.pointer, &dc.pointer, &dd.pointer};
|
||||||
|
cuda.check(cuda.cuLaunchKernel(kernel, cases, 1, 1, 32, 1, 1, 0, nullptr, parameters, nullptr), "MMA launch");
|
||||||
|
cuda.check(cuda.cuCtxSynchronize(), "MMA synchronize");
|
||||||
|
cuda.check(cuda.cuMemcpyDtoH_v2(d.data(), dd.pointer, d.size() * 2), "MMA download");
|
||||||
|
size_t mismatch = 0;
|
||||||
|
for (size_t n = 0; n < cases; ++n)
|
||||||
|
for (size_t row = 0; row < 16; ++row)
|
||||||
|
for (size_t col = 0; col < 8; ++col) {
|
||||||
|
double sum = half_value(c[n * 128 + row * 8 + col]);
|
||||||
|
for (size_t k = 0; k < 16; ++k)
|
||||||
|
sum += half_value(a[n * 256 + row * 16 + k]) * half_value(b[n * 128 + k * 8 + col]);
|
||||||
|
const uint16_t expected = half_bits(sum);
|
||||||
|
const uint16_t actual = d[n * 128 + row * 8 + col];
|
||||||
|
if (expected != actual) {
|
||||||
|
if (mismatch++ < 8) std::fprintf(stderr, "MMA_DIFFERENT case=%zu row=%zu column=%zu expected=%04x actual=%04x\n", n, row, col, expected, actual);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::printf("MMA_REFERENCE cases=%zu elements=%zu differing=%zu\n", cases, d.size(), mismatch);
|
||||||
|
if (mismatch) return 5;
|
||||||
|
std::printf("PASS CUDA_F16_MMA architecture=gfx1201 cases=%zu exact_f16=1\n", cases);
|
||||||
|
return 0;
|
||||||
|
} catch (const std::exception& error) {
|
||||||
|
std::fprintf(stderr, "FAIL CUDA_F16_MMA %s\n", error.what());
|
||||||
|
loaded_modules();
|
||||||
|
return 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
"""Compare logical M inputs/outputs between private temporal capture sequences."""
|
||||||
|
import argparse
|
||||||
|
import pathlib
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from m_replay_validate import me, LAYERS
|
||||||
|
|
||||||
|
|
||||||
|
def logical(parameters, pointer, count, output=False, directory=None):
|
||||||
|
base, allocation = parameters.alloc_of(pointer)
|
||||||
|
if output:
|
||||||
|
identifier = next(i for i, (b, _) in parameters.allocs.items() if b == base)
|
||||||
|
allocation = np.fromfile(directory / 'gpu-output' / f'alloc-{identifier}.bin', np.uint8)
|
||||||
|
return allocation[pointer - base:pointer - base + count]
|
||||||
|
|
||||||
|
|
||||||
|
def compare(reference, actual):
|
||||||
|
paths = []
|
||||||
|
order = ['enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1']
|
||||||
|
for path in reference.glob('replay-*-rrlite_*_4x4'):
|
||||||
|
layer = path.name.split('rrlite_', 1)[1][:-4]
|
||||||
|
ordinal = int(path.name.split('-')[1])
|
||||||
|
frame = (ordinal - 1) // (6 if layer == 'enc3_tube' else 1)
|
||||||
|
paths.append((frame, order.index(layer), ordinal, path, layer))
|
||||||
|
for frame, _, ordinal, path, layer in sorted(paths):
|
||||||
|
other = actual / path.name
|
||||||
|
a, b = me.Params(str(path)), me.Params(str(other))
|
||||||
|
c, _, merged, cin = LAYERS[layer]
|
||||||
|
if (a.W, a.H, a.sx, a.sy, a.grid) != (b.W, b.H, b.sx, b.sy, b.grid):
|
||||||
|
raise RuntimeError(f'Launch geometry differs: {path.name}')
|
||||||
|
fields = [('input', 'inp', (a.W // 2) * (a.H // 2) * cin if cin else a.W * a.H * c, False)]
|
||||||
|
if cin:
|
||||||
|
fields.append(('skip', 'p32', a.W * a.H * c, False))
|
||||||
|
fields.append(('output', 'out', a.W * a.H * c, True))
|
||||||
|
if merged:
|
||||||
|
fields.append(('merge', 'p48', (a.W // 2) * (a.H // 2) * merged, True))
|
||||||
|
for label, field, count, output in fields:
|
||||||
|
x = logical(a, getattr(a, field), count, output, path)
|
||||||
|
y = logical(b, getattr(b, field), count, output, other)
|
||||||
|
if x.size != count or y.size != count:
|
||||||
|
raise RuntimeError(f'Truncated {path.name} {label}')
|
||||||
|
differing = np.flatnonzero(x != y)
|
||||||
|
print(f'M_CAPTURE_COMPARE frame={frame} layer={layer} ordinal={ordinal} field={label} '
|
||||||
|
f'codes={count} differing={differing.size} '
|
||||||
|
f'first={int(differing[0]) if differing.size else -1}', flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--reference', type=pathlib.Path, required=True)
|
||||||
|
parser.add_argument('--actual', type=pathlib.Path, required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
compare(args.reference, args.actual)
|
||||||
@@ -18,7 +18,10 @@ def validate(args):
|
|||||||
raise RuntimeError('Capture kernel does not match requested M layer')
|
raise RuntimeError('Capture kernel does not match requested M layer')
|
||||||
native = {i: np.fromfile(args.native_dir / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs}
|
native = {i: np.fromfile(args.native_dir / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs}
|
||||||
candidates = []
|
candidates = []
|
||||||
for bx, by in blocks(parameters.grid, 12):
|
coordinates = args.block or blocks(parameters.grid, args.candidate_blocks)
|
||||||
|
for bx, by in coordinates:
|
||||||
|
if not (0 <= bx < parameters.grid[0] and 0 <= by < parameters.grid[1]):
|
||||||
|
raise RuntimeError(f'Block {bx},{by} is outside the capture grid')
|
||||||
model = model_block(parameters, args.layer, bx, by)
|
model = model_block(parameters, args.layer, bx, by)
|
||||||
expected, actual = stored(parameters, args.layer, bx, by, native, model)
|
expected, actual = stored(parameters, args.layer, bx, by, native, model)
|
||||||
difference = float(np.abs(expected - actual).max())
|
difference = float(np.abs(expected - actual).max())
|
||||||
@@ -52,8 +55,12 @@ if __name__ == '__main__':
|
|||||||
parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
|
parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
|
||||||
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
|
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
|
||||||
parser.add_argument('--blocks', type=int, default=2)
|
parser.add_argument('--blocks', type=int, default=2)
|
||||||
|
parser.add_argument('--candidate-blocks', type=int, default=12,
|
||||||
|
help='NumPy scan size used to choose the worst blocks for independent PTX')
|
||||||
|
parser.add_argument('--block', type=lambda value: tuple(map(int, value.split(','))),
|
||||||
|
action='append', help='Explicit x,y regression block (repeatable)')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.blocks < 1:
|
if args.blocks < 1 or args.candidate_blocks < 1:
|
||||||
parser.error('--blocks must be positive')
|
parser.error('--blocks must be positive')
|
||||||
try:
|
try:
|
||||||
validate(args)
|
validate(args)
|
||||||
|
|||||||
@@ -142,7 +142,7 @@ def metrics(expected, actual, label):
|
|||||||
return maximum, peak, psnr, steps
|
return maximum, peak, psnr, steps
|
||||||
|
|
||||||
|
|
||||||
def validate(layer, directory, output, block_count, real_capture):
|
def validate(layer, directory, output, block_count, real_capture, exact=False):
|
||||||
parameters = me.Params(str(directory))
|
parameters = me.Params(str(directory))
|
||||||
if parameters.kernel != f'rrlite_{layer}_4x4':
|
if parameters.kernel != f'rrlite_{layer}_4x4':
|
||||||
raise RuntimeError('Replay kernel does not match requested M layer')
|
raise RuntimeError('Replay kernel does not match requested M layer')
|
||||||
@@ -151,7 +151,7 @@ def validate(layer, directory, output, block_count, real_capture):
|
|||||||
for bx, by in blocks(parameters.grid, block_count)]
|
for bx, by in blocks(parameters.grid, block_count)]
|
||||||
expected, actual = (np.concatenate(values) for values in zip(*pairs))
|
expected, actual = (np.concatenate(values) for values in zip(*pairs))
|
||||||
maximum, peak, psnr, steps = metrics(expected, actual, f'M_REPLAY_REFERENCE kernel={layer}')
|
maximum, peak, psnr, steps = metrics(expected, actual, f'M_REPLAY_REFERENCE kernel={layer}')
|
||||||
if (not real_capture and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)):
|
if ((exact or not real_capture) and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)):
|
||||||
raise RuntimeError(f'M mismatch: exact synthetic / <=1 FP8 step and >=50 dB real gate; peak={peak}')
|
raise RuntimeError(f'M mismatch: exact synthetic / <=1 FP8 step and >=50 dB real gate; peak={peak}')
|
||||||
print(f'PASS M_REPLAY_REFERENCE kernel={layer} blocks={len(pairs)} architecture=gfx1201 nan_inf=0')
|
print(f'PASS M_REPLAY_REFERENCE kernel={layer} blocks={len(pairs)} architecture=gfx1201 nan_inf=0')
|
||||||
|
|
||||||
@@ -165,6 +165,7 @@ if __name__ == '__main__':
|
|||||||
parser.add_argument('--size', type=int, choices=[8, 16, 24, 32], default=16)
|
parser.add_argument('--size', type=int, choices=[8, 16, 24, 32], default=16)
|
||||||
parser.add_argument('--blocks', type=int, default=12)
|
parser.add_argument('--blocks', type=int, default=12)
|
||||||
parser.add_argument('--real-capture', action='store_true')
|
parser.add_argument('--real-capture', action='store_true')
|
||||||
|
parser.add_argument('--exact', action='store_true', help='Require every stored real FP8 value to match')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
try:
|
try:
|
||||||
if args.mode == 'generate':
|
if args.mode == 'generate':
|
||||||
@@ -172,7 +173,7 @@ if __name__ == '__main__':
|
|||||||
else:
|
else:
|
||||||
if not args.output_dir or args.blocks < 1:
|
if not args.output_dir or args.blocks < 1:
|
||||||
parser.error('validate requires --output-dir and positive --blocks')
|
parser.error('validate requires --output-dir and positive --blocks')
|
||||||
validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture)
|
validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture, args.exact)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(f'FAIL M_REPLAY {error}', file=sys.stderr)
|
print(f'FAIL M_REPLAY {error}', file=sys.stderr)
|
||||||
raise SystemExit(5)
|
raise SystemExit(5)
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
"""Validate every captured M launch and spatial block; all allocations stay private."""
|
||||||
|
import argparse
|
||||||
|
import contextlib
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
from m_replay_validate import LAYERS, validate
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
summary = {'passed': False, 'captured_outputs': True, 'exact': args.exact, 'launches': []}
|
||||||
|
status = 1
|
||||||
|
try:
|
||||||
|
captures = sorted(args.capture_dir.glob('replay-*-rrlite_*_4x4'))
|
||||||
|
if not captures:
|
||||||
|
raise RuntimeError('No audited M captures were found')
|
||||||
|
for capture in captures:
|
||||||
|
layer = capture.name.split('rrlite_', 1)[1][:-4]
|
||||||
|
if layer not in LAYERS:
|
||||||
|
raise RuntimeError(f'Unknown captured M layer: {layer}')
|
||||||
|
logfile = args.output_dir / (capture.name + '.log')
|
||||||
|
with logfile.open('w', encoding='utf-8') as output, contextlib.redirect_stdout(output):
|
||||||
|
validate(layer, capture, capture / 'gpu-output', args.blocks, True, args.exact)
|
||||||
|
print(capture.name + ' ' + logfile.read_text().splitlines()[0], flush=True)
|
||||||
|
summary['launches'].append({'capture': capture.name, 'layer': layer, 'passed': True})
|
||||||
|
present = {item['layer'] for item in summary['launches']}
|
||||||
|
if present != set(LAYERS):
|
||||||
|
raise RuntimeError('Capture set does not contain all five M layers')
|
||||||
|
summary['passed'] = True
|
||||||
|
print(f'PASS M_TEMPORAL_REFERENCE launches={len(captures)} blocks={args.blocks} exact={int(args.exact)}', flush=True)
|
||||||
|
status = 0
|
||||||
|
except Exception as error:
|
||||||
|
summary['error'] = str(error)
|
||||||
|
summary['failed_capture'] = capture.name if 'capture' in locals() else None
|
||||||
|
traceback.print_exc()
|
||||||
|
finally:
|
||||||
|
(args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--capture-dir', type=pathlib.Path, required=True)
|
||||||
|
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
|
||||||
|
parser.add_argument('--blocks', type=int, default=10000)
|
||||||
|
parser.add_argument('--exact', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.blocks < 1:
|
||||||
|
parser.error('--blocks must be positive')
|
||||||
|
sys.exit(main(args))
|
||||||
@@ -82,6 +82,14 @@ int main(int argc, char** argv) {
|
|||||||
hipModule_t module = nullptr;
|
hipModule_t module = nullptr;
|
||||||
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(replay gfx1201)");
|
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(replay gfx1201)");
|
||||||
struct Cleanup { HipApi& hip; hipModule_t module; ~Cleanup() { (void)hip.hipModuleUnload(module); } } cleanup{hip, module};
|
struct Cleanup { HipApi& hip; hipModule_t module; ~Cleanup() { (void)hip.hipModuleUnload(module); } } cleanup{hip, module};
|
||||||
|
if (const char* debugBlock = std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
|
||||||
|
int coordinates[2]{};
|
||||||
|
if (std::sscanf(debugBlock, "%d,%d", &coordinates[0], &coordinates[1]) != 2) throw std::runtime_error("D4R_SWIN_DEBUG_BLOCK must be x,y");
|
||||||
|
hipDeviceptr_t address = nullptr; size_t size = 0;
|
||||||
|
hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_block"), "Find Swin diagnostic block");
|
||||||
|
if (size != sizeof(coordinates)) throw std::runtime_error("Swin diagnostic coordinates ABI");
|
||||||
|
hip.check(hip.hipMemcpy(address, coordinates, size, hipMemcpyHostToDevice), "Set Swin diagnostic block");
|
||||||
|
}
|
||||||
auto scalar = [&](const char* name, unsigned fallback) {
|
auto scalar = [&](const char* name, unsigned fallback) {
|
||||||
hipDeviceptr_t address = nullptr; size_t size = 0;
|
hipDeviceptr_t address = nullptr; size_t size = 0;
|
||||||
if (hip.hipModuleGetGlobal(&address, &size, module, name) != hipSuccess) return fallback;
|
if (hip.hipModuleGetGlobal(&address, &size, module, name) != hipSuccess) return fallback;
|
||||||
@@ -108,6 +116,15 @@ int main(int argc, char** argv) {
|
|||||||
nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)");
|
nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)");
|
||||||
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)");
|
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)");
|
||||||
std::filesystem::create_directories(output);
|
std::filesystem::create_directories(output);
|
||||||
|
if (std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
|
||||||
|
hipDeviceptr_t address = nullptr; size_t size = 0;
|
||||||
|
hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_values"), "Find Swin diagnostic stages");
|
||||||
|
std::vector<uint8_t> bytes(size);
|
||||||
|
hip.check(hip.hipMemcpy(bytes.data(), address, size, hipMemcpyDeviceToHost), "Download Swin diagnostic stages");
|
||||||
|
std::ofstream file(output / "swin-stages.f16", std::ios::binary);
|
||||||
|
file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
|
||||||
|
if (!file) throw std::runtime_error("Cannot save Swin diagnostic stages");
|
||||||
|
}
|
||||||
for (const auto& allocation : allocations) {
|
for (const auto& allocation : allocations) {
|
||||||
std::vector<uint8_t> bytes(allocation->size);
|
std::vector<uint8_t> bytes(allocation->size);
|
||||||
hip.check(hip.hipMemcpy(bytes.data(), allocation->device, bytes.size(), hipMemcpyDeviceToHost), "hipMemcpy(replay result)");
|
hip.check(hip.hipMemcpy(bytes.data(), allocation->device, bytes.size(), hipMemcpyDeviceToHost), "hipMemcpy(replay result)");
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
"""Compare compile-time Swin diagnostic snapshots with the NumPy intermediate values."""
|
||||||
|
import argparse
|
||||||
|
import numpy as np
|
||||||
|
from m_replay_validate import me, sm, LAYERS, metrics
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--capture-dir', required=True)
|
||||||
|
parser.add_argument('--stages', required=True)
|
||||||
|
parser.add_argument('--block', default='0,4')
|
||||||
|
parser.add_argument('--layer', choices=list(LAYERS), default='enc1')
|
||||||
|
args = parser.parse_args()
|
||||||
|
p = me.Params(args.capture_dir)
|
||||||
|
c, heads, _, cin = LAYERS[args.layer]
|
||||||
|
bx, by = map(int, args.block.split(','))
|
||||||
|
pre = 4 * c * cin + 8 * c if cin else 0
|
||||||
|
x0 = sm.decoder_input(p, bx, by, c, cin, heads, p.p32) if cin else sm.load_tokens(p, bx, by, c // 32)
|
||||||
|
stages = {}
|
||||||
|
sm.swin_block(p, x0, c, heads, pre, stages=stages)
|
||||||
|
actual = np.fromfile(args.stages, np.float16).reshape(4, 64, 128).astype(np.float64)
|
||||||
|
for index, name in enumerate(('h1', 'x1', 'h2', 'O')):
|
||||||
|
expected = stages[name]
|
||||||
|
values = actual[index, :, :expected.shape[1]]
|
||||||
|
metrics(expected, values, f'SWIN_STAGE {name}')
|
||||||
|
different = np.argwhere(expected != values)
|
||||||
|
for row, channel in different[:8]:
|
||||||
|
print(f'DIFFERENT stage={name} token={row} channel={channel} reference={expected[row, channel]} native={values[row, channel]}')
|
||||||
Reference in new issue
Block a user