diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b7a158..1ae40fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,7 +50,7 @@ foreach(family k m) list(APPEND native_flags -DD4R_K_FP16_BASELINE) endif() if(family STREQUAL "m" AND D4R_M_FP16_BASELINE) - list(APPEND native_flags -DSWIN_EXACT) + list(APPEND native_flags -DSWIN_EXACT -DSWIN_EXACT_PV) endif() if(flags_line) separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}") @@ -68,7 +68,7 @@ foreach(family k m) endforeach() endforeach() add_custom_target(d4r_native_gfx1201_compile ALL DEPENDS ${native_objects}) -foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe) +foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_mma_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe) add_executable(d4r_${probe} tools/windows/${probe}.cpp) target_compile_definitions(d4r_${probe} PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__) target_include_directories(d4r_${probe} PRIVATE "${D4R_HIP_ROOT}/include") @@ -147,6 +147,7 @@ install(FILES tools/windows/k_ptx_validate.py DESTINATION .) install(FILES tools/windows/frame_compare.py DESTINATION .) install(FILES tools/windows/m_replay_validate.py DESTINATION .) install(FILES tools/windows/m_ptx_validate.py DESTINATION .) +install(FILES tools/windows/m_temporal_validate.py tools/windows/m_capture_compare.py DESTINATION .) install(FILES scripts/windows/test-m-replay.ps1 DESTINATION .) install(FILES scripts/windows/test-m-captures.ps1 DESTINATION .) install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py @@ -182,6 +183,11 @@ if(EXISTS "${D4R_ZLUDA_ROOT}/nvcuda.dll") set_tests_properties(cuda_${context} PROPERTIES TIMEOUT 180 LABELS "hardware;zluda") endforeach() if(EXISTS "${D4R_ZLUDA_ROOT}/build-info.json") + add_test(NAME cuda_f16_mma_reference COMMAND d4r_cuda_mma_probe + --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll") + set_tests_properties(cuda_f16_mma_reference PROPERTIES TIMEOUT 120 + LABELS "hardware;zluda;mma;reference" + ENVIRONMENT "D4R_ZLUDA_WMMA=1;D4R_ZLUDA_WMMA_F16_REFERENCE=1") add_test(NAME cuda_images COMMAND d4r_cuda_image_probe --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32) set_tests_properties(cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors") diff --git a/docs/windows-rdna4-port.md b/docs/windows-rdna4-port.md index 3bd3587..22ad744 100644 --- a/docs/windows-rdna4-port.md +++ b/docs/windows-rdna4-port.md @@ -13,8 +13,12 @@ frames on Windows. M now passes all five synthetic and real-weight layers agains NumPy and independent PTX, and four native CUDA frames. The standalone D3D12 harness now produces four K and four M frames using imported VRAM and shared fences, with bit-exact RGB agreement against the corresponding CUDA harnesses. -Game command-list/queue integration remains in progress. The fully translated M -comparison is being rerun with the correct WMMA options and a writable JIT cache. +Game command-list/queue integration remains in progress. A full temporal M scan +found rare attention P/V accumulation mismatches that the earlier twelve-block +sampling missed. `SWIN_EXACT_PV` fixes the native baseline; ZLUDA patch `0013` +adds the corresponding opt-in f16 reference accumulation. All spatial blocks in +40 actual native M launches now exactly match NumPy. Four complete native M RGB +frames exactly match fully translated M with the corrected reference option. Do not integrate NGX until the integer PTX workload is stable on the real GPU. | Gate | Status | @@ -25,11 +29,70 @@ Do not integrate NGX until the integer PTX workload is stable on the real GPU. | M3: Windows NGX initialization and simple DLSS path | Init, SR capabilities, Create/Evaluate PASS; E, K and M CUDA harnesses execute | | M4: D3D12 / HIP external memory and fence round trip | PASS with TheRock; stable 7.2 has mapped-view leak | | M5: independently validated gfx12 WMMA backend | PASS: raw + legacy adapter + upstream layout, max abs/relative error 0 | -| M6: K layers, full transformer and image validation | CUDA harness PASS: 11 real-weight layers, native launches for all layers, four finite frames, 81.85–92.91 dB versus translated K; D3D12 integration pending | -| M7: M FP16-equivalent baseline and full transformer | CUDA harness PASS: 5 real-weight layers exactly match NumPy/PTX, 4 finite frames, 40 native Swin launches; D3D12 and full translated M comparison pending | +| M6: K layers, full transformer and image validation | PASS: 11 real-weight layers, 44 native launches, four D3D12 frames bit-exact against native CUDA, 81.85–92.91 dB versus translated K | +| M7: M FP16-equivalent baseline and full transformer | PASS: all blocks of 40 native launches exactly match NumPy; targeted independent PTX regression matches; four D3D12 frames bit-exact against corrected translated M | | M8: standalone Windows D3D12 NGX harness | PASS at explicit submitted queue boundaries: K/M, four frames each, no shim CPU image copies, bit-exact RGB against CUDA harness | | M9: OptiScaler integration, profiling, installation | Pending | +## M temporal precision regression (2026-09-30) + +The earlier finite-frame and sampled-layer results did not cover every spatial +block. The second enc1 launch has a cancellation-sensitive P/V dot in block +`0,4`: the old native f32 WMMA route differed by up to 21 e4m3 code steps. +NumPy and the independent PTX interpreter agree exactly on that block. +`SWIN_EXACT_PV` stores each wave's sixteen P rows and V rows in LDS, sums sixteen +f16 products in f64, rounds directly to f16, then applies the existing e4m3 +quantisation. Other GEMMs retain the gfx12 WMMA layout and FP16 widening. +The flag is enabled by `D4R_M_FP16_BASELINE`; upstream fast/Linux defaults are +preserved when that flag is absent. Intermediate snapshots are compiled only +with `D4R_SWIN_DIAGNOSTICS`; they are absent from packaged kernels. + +The translated reference also differed: first frame, fourth enc3_tube launch, +block `0,0`, thirty stored codes, maximum fifteen e4m3 steps. Its inputs match +the corrected native launch exactly. Patch `0013` adds +`D4R_ZLUDA_WMMA_F16_REFERENCE=1` for gfx12 f16-accumulator MMA helpers, retaining +the established fragment gathers, pair fusion and conversions. It uses f64 +sums and integer f16 RNE encoding, including subnormal bits. FP8 GEMMs continue +using the existing native WMMA widening. The default upstream helper behavior +remains available with this option unset or zero. Windows diagnostics select +the reference option by default. All d4r codegen options enter the JIT cache key. + +This is a match to the independent mathematical NumPy/PTX baseline. NVIDIA's +[PTX floating-point MMA specification](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-instructions-mma) +leaves accumulation order, rounding and subnormal handling unspecified; these +checks do not establish bit-exact agreement with NVIDIA hardware. + +Recorded results on the RX 9070 XT / TheRock runtime: + +* Public `cuda_mma_probe`: 512 synthetic matrix cases, 65,536 f16 outputs, + zero differing bits with reference accumulation. The old fast WMMA route + differs in 23,308 outputs on the same cancellation/subnormal stress fixture. +* `m-native-exact-pv-temporal-reference`: all spatial blocks of 40 actual native + launches, every stored e4m3 value identical to NumPy, max abs/relative error 0. +* `m-exact-pv-enc1-regression-ptx`: original problem block `0,4`, independent + PTX versus native and versus NumPy both max absolute/relative error 0. +* `m-exact-pv-complete-f16-reference.log`: four complete RGB frames against + corrected fully translated M, max absolute/relative error 0, PSNR infinite. +* `m-translated-f16-temporal-reference`: all blocks of all forty translated + M launches exactly match NumPy; no sampled-window exemption. +* `m-temporal-final-ctest.log`: 11/11 native Windows hardware tests PASS. + Pinned fresh-source application of all twelve ZLUDA patches PASS. +* `d3d12-m-exact-pv-final.zip`: four finite frames, forty native launches, + no shim CPU image copies, frame age zero. This remains the explicit queue + boundary harness; open game command-list support is the next gate. + +All captures, extracted PTX and real weights remain in ignored private result +directories. To repeat the full temporal scan without editing sources: + +```powershell +$env:PYTHONPATH="$PWD/.tools/python/vendor" +python tools/windows/m_temporal_validate.py --capture-dir test-results/ngx-m-exact-pv-capture/captures --output-dir test-results/m-temporal-reference --exact +``` + +The game selected for the next integration test is Silent Hill 2 at +`D:\Games\SILENT HILL 2\SHProto\Binaries\Win64\SHProto-Win64-Shipping.exe`. +No game files have been modified at this milestone. + ## Source baseline (checked 2026-09-29) * d4r: `ed1ab60`, main, 14 commits; https://github.com/countervolts/d4r. diff --git a/kernels/m/swin_block.h b/kernels/m/swin_block.h index 148f0a6..0e33892 100644 --- a/kernels/m/swin_block.h +++ b/kernels/m/swin_block.h @@ -5,6 +5,20 @@ // CIN patch-expand input channels (0 = none): decoder input = f16(q8(expand(low-res CIN)) + skip) #pragma once #include "swin_common.h" +#ifdef D4R_SWIN_DIAGNOSTICS +extern "C" { +__device__ int d4r_swin_debug_block[2] = {0, 4}; +__device__ half_t d4r_swin_debug_values[4 * 64 * 128]; +} +template __device__ __forceinline__ void swin_snapshot( + const half_t* activation, int stage, int bx, int by, int tid, bool pair_order) { + if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1] && tid < 64) + for (int c = 0; c < C; ++c) + d4r_swin_debug_values[(stage * 64 + tid) * 128 + c] = + activation[tid * AST + (pair_order ? gperm(c) : apos(c))]; + __syncthreads(); +} +#endif // Immediate lane masks need only a compile-time integer, not the host C++ // standard library (which is absent from device-only Windows HIP builds). @@ -79,7 +93,12 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara static_assert(NW % NH == 0 && NW / NH <= 4, "waves per head"); constexpr int NPL = L::NPL, NT = L::NT, INNER = L::INNER; constexpr int AST = (C > INNER ? C : INNER) + 8; - constexpr int GST = 40, QST = 40, PST = 24; + constexpr int GST = 40, PST = 24; +#ifdef SWIN_EXACT_PV + constexpr int QST = 56; +#else + constexpr int QST = 40; +#endif #ifndef SWIN_MLP_R #define SWIN_MLP_R(NW) ((NW) / 2) #endif @@ -294,6 +313,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara __syncthreads(); // ------------------------------------------------ stage 2: Q/V projection + attention (wave = head) +#ifdef D4R_SWIN_DIAGNOSTICS + swin_snapshot(A, 0, bx, by, tid, false); +#endif { constexpr int MW = 4 / (NW / NH); // windows per wave const int h = NW == NH ? wv : wv % NH, m0 = NW == NH ? 0 : MW * (wv / NH); @@ -400,6 +422,13 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara Qs[mrow(2 * k + 1) * PST + l16] = pw[1]; } wave_sync(); +#ifdef SWIN_EXACT_PV + half_t* Vs = Qs + 16 * PST; + for (int nt = 0; nt < 2; ++nt) + for (int i = 0; i < 8; ++i) + Vs[mrow(i) * 32 + 16 * nt + l16] = vp[m][nt][i / 2][i & 1]; + wave_sync(); +#endif const sop pa = lds16(Qs + l16 * PST); #pragma unroll for (int nt = 0; nt < 2; ++nt) @@ -422,7 +451,19 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara } const h16 vb = __builtin_bit_cast(h16, vw); #endif - const f8v o = wmma(pa, vb, splat(0.0f)); + f8v o = wmma(pa, vb, splat(0.0f)); +#ifdef SWIN_EXACT_PV + // FP16 P/V products need more than 24 significant bits in a + // cancellation-heavy dot. An f32 WMMA can round to the wrong + // FP16 neighbour at a midpoint, then flip the FP8 output code. + // This baseline sums the 16 FP16 products before one FP16 round. + for (int i = 0; i < 8; ++i) { + double dot = 0.0; + for (int k = 0; k < 16; ++k) + dot += (double)Qs[mrow(i) * PST + k] * (double)Vs[k * 32 + 16 * nt + l16]; + o[i] = (float)(half_t)dot; + } +#endif const int ocol = apos(32 * h + ginv(16 * nt + l16)); #pragma unroll for (int i = 0; i < 8; i += 2) @@ -438,6 +479,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara __syncthreads(); // ------------------------------------------------ stage 3: output projection + residual +#ifdef D4R_SWIN_DIAGNOSTICS + swin_snapshot(A, 3, bx, by, tid, false); +#endif #pragma unroll for (int kc = 0; kc < NH; ++kc) { @@ -471,6 +515,11 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara if (tid < 64) { // the row as 128-bit LDS loads: element 8 j + c of n8 tile j +#ifdef D4R_SWIN_DIAGNOSTICS + if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1]) + for (int c = 0; c < C; ++c) + d4r_swin_debug_values[(64 + tid) * 128 + c] = A[tid * AST + gperm(c)]; +#endif h8 row[L::NT8]; #pragma unroll for (int j = 0; j < L::NT8; ++j) @@ -529,6 +578,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara __syncthreads(); // ------------------------------------------------ stage 4: MLP, R chunks per round +#ifdef D4R_SWIN_DIAGNOSTICS + swin_snapshot(A, 2, bx, by, tid, false); +#endif { constexpr int WPC = NW / R, MPW = 4 / WPC; // waves per chunk, fc1 m tiles per wave const int cw = wv / WPC, mb = MPW * (wv % WPC); diff --git a/kernels/tools/swin_model.py b/kernels/tools/swin_model.py index b04f014..29726c8 100644 --- a/kernels/tools/swin_model.py +++ b/kernels/tools/swin_model.py @@ -98,7 +98,7 @@ def layout(C, heads, pre=0): L['end'] = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1) return L -def swin_block(P, x0, C, heads, pre=0): +def swin_block(P, x0, C, heads, pre=0, stages=None): """x0: 64 x C natural (f16 values). Returns (x2 pair order, out natural fp8 values).""" fp8 = lambda T: E4M3[P.wbuf[T]] L = layout(C, heads, pre) @@ -116,6 +116,8 @@ def swin_block(P, x0, C, heads, pre=0): Wo[32 * s:32 * s + 32] = woff_table(L['head'][s] + 64 * C + 512, 0, 512, 32, C) x1 = mma_chain(pair_to_nat(q8(O)), fp8(Wo), C0=f16(nat_to_pair(x0) + P.f16vec(L['bo'], C)[None, :])) h2 = pair_to_nat(rms_from_sum(x1, tree_norm2(x1), P.f16vec(L['g2'], C))) + if stages is not None: + stages.update(h1=h1, x1=pair_to_nat(x1), h2=h2, O=pair_to_nat(q8(O))) x2 = f16(x1 + P.f16vec(L['b2'], C)[None, :]) idx = gp(np.arange(32)) for c, B1 in enumerate(L['b1']): diff --git a/patches/zluda/0013-gfx12-f16-reference-accumulation.patch b/patches/zluda/0013-gfx12-f16-reference-accumulation.patch new file mode 100644 index 0000000..4fa149f --- /dev/null +++ b/patches/zluda/0013-gfx12-f16-reference-accumulation.patch @@ -0,0 +1,151 @@ +diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs +index 5a54417..90c0034 100644 +--- a/llvm_zluda/src/compile.rs ++++ b/llvm_zluda/src/compile.rs +@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option { + let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 }; + if family == 12 { + flags |= 2; ++ // Explicit diagnostic/correctness mode; retain upstream's fast default. ++ if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") { ++ flags |= 8; ++ } + // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM) + if flags & 1 != 0 && wmma_fp8_native_requested() { + flags |= 4; +diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp +index 07238db..d262fc6 100644 +--- a/llvm_zluda/src/lib.cpp ++++ b/llvm_zluda/src/lib.cpp +@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M) + // Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1). + // Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers. + // Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers. ++// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs. + // The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here. + unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags) + { +@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags) + const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_"; + const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch; + const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_"; ++ const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16"); + Helpers H; + H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2); + H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4); +- H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2); ++ H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2); + H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2); + H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2); + H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2); +- H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2); ++ H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2); + { + Type *F = Type::getFloatTy(Context); + Type *V4F = FixedVectorType::get(F, 4); +diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp +index 4ff25dd..36bd6b1 100644 +--- a/ptx/lib/zluda_ptx_impl.cpp ++++ b/ptx/lib/zluda_ptx_impl.cpp +@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8 + #endif + } + ++// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may ++// reach the other f16 neighbour around cancellation-heavy rounding midpoints. ++// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and ++// round directly to f16 once, matching d4r's independent NumPy/PTX reference. ++// The e4m3 and f32-shadow paths continue using native WMMA. ++__device__ static double wmma12_half_value(uint32_t bits) ++{ ++ const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10; ++ const double value = exponent == 0 ? double(magnitude) * 0x1p-24 : ++ double(std::bit_cast(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) : ++ ((magnitude << 13) + 0x38000000u))); ++ return bits & 0x8000u ? -value : value; ++} ++ ++// Preserve f16 subnormal result bits even when the target's default f16 ++// arithmetic mode flushes them. No intermediate f32 rounding is involved. ++__device__ static uint32_t wmma12_half_round(double value) ++{ ++ const uint64_t bits = std::bit_cast(value); ++ const uint32_t sign = uint32_t(bits >> 48) & 0x8000u; ++ const uint32_t exponent = uint32_t((bits >> 52) & 2047u); ++ const uint64_t fraction = bits & 0xfffffffffffffu; ++ if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u); ++ const int power = int(exponent) - 1023; ++ if (power < -25) return sign; ++ if (power > 15) return sign | 0x7c00u; ++ const unsigned shift = power < -14 ? unsigned(28 - power) : 42u; ++ const uint64_t significand = fraction | 0x10000000000000ull; ++ uint32_t code = uint32_t(significand >> shift); ++ const uint64_t tail = significand & ((uint64_t(1) << shift) - 1); ++ const uint64_t midpoint = uint64_t(1) << (shift - 1); ++ code += tail > midpoint || (tail == midpoint && (code & 1u)); ++ if (power >= -14) code += uint32_t(power + 14) << 10; ++ return sign | code; ++} ++ ++__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired) ++{ ++ const bool high = wmma12_half() != 0; ++ const uint2::Native_vec_ own = paired && high ? c1 : c0; ++ const uint2::Native_vec_ send = paired && !high ? c1 : c0; ++ const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)}; ++ const uint2::Native_vec_ even = paired && high ? other : own; ++ const uint2::Native_vec_ odd = paired && high ? own : other; ++ return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u), ++ (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)}; ++} ++ ++__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c) ++{ ++ // x/y: packed even rows, z/w: packed odd rows, in this half-wave. ++ wmma_uint4 result = {}; ++ const uint32_t half = wmma12_half(); ++ for (uint32_t i = 0; i < 8; ++i) ++ { ++ const uint32_t row = i + 8 * half; ++ double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu); ++ for (uint32_t khalf = 0; khalf < 2; ++khalf) ++ for (uint32_t j = 0; j < 4; ++j) ++ { ++ const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]); ++ const uint32_t other = wmma12_other_half(b[j]); ++ const uint32_t bv = half == khalf ? b[j] : other; ++ sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu); ++ sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16); ++ } ++ // Do not round through f32: that can reintroduce the midpoint error. ++ result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1)); ++ } ++ return result; ++} ++ + // f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8 + // e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling + __device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c) +@@ -1959,6 +2031,22 @@ extern "C" + return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t))); + } + ++ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t) ++ { ++ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false)); ++ const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)}; ++ return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y}; ++ } ++ ++ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t) ++ { ++ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true)); ++ const bool high = wmma12_half() != 0; ++ const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z); ++ const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w); ++ return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1}; ++ } ++ + wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down) + { + return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down; diff --git a/scripts/windows/stage-native-m.ps1 b/scripts/windows/stage-native-m.ps1 index 2f75986..f348c28 100644 --- a/scripts/windows/stage-native-m.ps1 +++ b/scripts/windows/stage-native-m.ps1 @@ -16,7 +16,7 @@ foreach ($layer in @('enc1','enc2','enc3_tube','dec2','dec1')) { } & python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll if ($LASTEXITCODE) { throw "Native manifest generation failed ($LASTEXITCODE)" } -@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; FP16 widening; native FP8 disabled'; +@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; SWIN_EXACT_PV; FP16 widening; native FP8 disabled'; dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash; validation='synthetic layers passed; actual DLSS capture/reference validation required'; modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} | diff --git a/scripts/windows/test-windows-rdna4.ps1 b/scripts/windows/test-windows-rdna4.ps1 index 4139f91..a056d02 100644 --- a/scripts/windows/test-windows-rdna4.ps1 +++ b/scripts/windows/test-windows-rdna4.ps1 @@ -53,7 +53,7 @@ $originalPythonPath = $env:PYTHONPATH $originalVerbose = $env:D4R_ZLUDA_VERBOSE $originalCache = $env:ZLUDA_CACHE_DIR $originalCodegen = @{} -foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) { +foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) { $originalCodegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process') } $exitStatus = 1 @@ -103,14 +103,14 @@ function Invoke-Probe([string]$Name, [string]$Exe, [string[]]$Arguments) { try { if (!$env:ZLUDA_CACHE_DIR) { $env:ZLUDA_CACHE_DIR = Join-Path $repoRoot 'build/zluda-cache-windows' } $summary.zludaCacheDirectory = $env:ZLUDA_CACHE_DIR - foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) { + foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) { if ($null -eq $originalCodegen[$setting]) { [Environment]::SetEnvironmentVariable($setting, $(if ($setting -eq 'D4R_ZLUDA_WMMA_FP8_NATIVE') { '0' } else { '1' }), 'Process') } } $summary.codegen = @{} foreach ($setting in $originalCodegen.Keys) { $summary.codegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process') } - Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE" + Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE f16 reference=$env:D4R_ZLUDA_WMMA_F16_REFERENCE" if ($NgxOnly -and (!$NgxCore -or !$DlssDll)) { throw '-NgxOnly requires both locally supplied NVIDIA DLL paths.' } if ($RequireNativeNetwork) { if ($NgxMode -notin @('evaluate','d3d12') -or $Preset -notin @(11,13) -or !$env:D4R_ZLUDA_NATIVE_DIR) { @@ -197,6 +197,14 @@ try { if ($cudaOk -and ($NgxMode -eq 'evaluate' -or (Test-Path (Join-Path $ZludaRoot 'build-info.json')))) { $cudaOk = Invoke-Probe 'cuda-images' (Join-Path $bin 'd4r_cuda_image_probe.exe') @( '--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'), '--iterations', "$Iterations") + if ($cudaOk) { + $previousF16 = $env:D4R_ZLUDA_WMMA_F16_REFERENCE + try { + $env:D4R_ZLUDA_WMMA_F16_REFERENCE = '1' + $cudaOk = Invoke-Probe 'cuda-f16-mma-reference' (Join-Path $bin 'd4r_cuda_mma_probe.exe') @( + '--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll')) + } finally { $env:D4R_ZLUDA_WMMA_F16_REFERENCE = $previousF16 } + } } } if ($cudaOk) { diff --git a/tools/windows/cuda_mma_probe.cpp b/tools/windows/cuda_mma_probe.cpp new file mode 100644 index 0000000..79875bb --- /dev/null +++ b/tools/windows/cuda_mma_probe.cpp @@ -0,0 +1,165 @@ +#include "hip_api.h" +#include "cuda_api.h" +#include +#include +#include + +// Public synthetic matrix fragments, independent of NVIDIA DLLs and weights. +static constexpr char source[] = R"PTX( +.version 7.0 +.target sm_75 +.address_size 64 +.visible .entry d4r_f16_mma_reference( + .param .u64 A, .param .u64 B, .param .u64 C, .param .u64 D) +{ + .reg .b32 %r<24>; + .reg .b64 %rd<12>; + .reg .b16 %h<4>; + ld.param.u64 %rd0, [A]; + ld.param.u64 %rd1, [B]; + ld.param.u64 %rd2, [C]; + ld.param.u64 %rd3, [D]; + mov.u32 %r0, %ctaid.x; + mov.u32 %r1, %tid.x; + shr.u32 %r2, %r1, 2; + and.b32 %r3, %r1, 3; + shl.b32 %r3, %r3, 1; + mul.wide.u32 %rd4, %r0, 512; + add.u64 %rd0, %rd0, %rd4; + mul.wide.u32 %rd4, %r0, 256; + add.u64 %rd1, %rd1, %rd4; + add.u64 %rd2, %rd2, %rd4; + add.u64 %rd3, %rd3, %rd4; + mad.lo.u32 %r4, %r2, 16, %r3; + mul.wide.u32 %rd4, %r4, 2; + add.u64 %rd5, %rd0, %rd4; + ld.global.u32 %r8, [%rd5]; + ld.global.u32 %r9, [%rd5+256]; + ld.global.u32 %r10, [%rd5+16]; + ld.global.u32 %r11, [%rd5+272]; + mad.lo.u32 %r4, %r3, 8, %r2; + mul.wide.u32 %rd4, %r4, 2; + add.u64 %rd5, %rd1, %rd4; + ld.global.b16 %h0, [%rd5]; + ld.global.b16 %h1, [%rd5+16]; + mov.b32 %r12, {%h0,%h1}; + ld.global.b16 %h2, [%rd5+128]; + ld.global.b16 %h3, [%rd5+144]; + mov.b32 %r13, {%h2,%h3}; + mad.lo.u32 %r4, %r2, 8, %r3; + mul.wide.u32 %rd4, %r4, 2; + add.u64 %rd5, %rd2, %rd4; + ld.global.u32 %r14, [%rd5]; + ld.global.u32 %r15, [%rd5+128]; + mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16 + {%r16,%r17}, {%r8,%r9,%r10,%r11}, {%r12,%r13}, {%r14,%r15}; + add.u64 %rd5, %rd3, %rd4; + st.global.u32 [%rd5], %r16; + st.global.u32 [%rd5+128], %r17; + ret; +} +)PTX"; + +static uint16_t half_bits(double value) { + _Float16 half = static_cast<_Float16>(value); + uint16_t bits; + std::memcpy(&bits, &half, sizeof(bits)); + return bits; +} +static double half_value(uint16_t bits) { + _Float16 half; + std::memcpy(&half, &bits, sizeof(bits)); + return static_cast(half); +} + +int main(int argc, char** argv) { + using namespace d4r::diag; + start(); + try { + Args args(argc, argv); + HipApi hip(args.hip_root); + hipDeviceProp_t props{}; + hip.select_gfx1201(args.device, props); + CudaApi cuda(args.cuda_dll); + cuda.check(cuda.cuInit(0), "cuInit"); + CUdevice device = -1; + int count = 0; + cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount"); + for (int i = 0; i < count; ++i) { + CUdevice candidate; + int bus, pci, domain; + cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet"); + cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus"); + cuda.check(cuda.cuDeviceGetAttribute(&pci, 34, candidate), "PCI device"); + cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain"); + if (bus == props.pciBusID && pci == props.pciDeviceID && domain == props.pciDomainID) device = candidate; + } + if (device < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch"); + CUcontext context = nullptr; + cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, device), "primary context"); + struct Context { CudaApi& api; CUdevice device; ~Context() { api.cuCtxSetCurrent(nullptr); api.cuDevicePrimaryCtxRelease_v2(device); } } context_owner{cuda, device}; + cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent"); + CUmodule module = nullptr; + cuda.check(cuda.cuModuleLoadData(&module, source), "MMA PTX JIT"); + struct Module { CudaApi& api; CUmodule module; ~Module() { api.cuModuleUnload(module); } } module_owner{cuda, module}; + CUfunction kernel = nullptr; + cuda.check(cuda.cuModuleGetFunction(&kernel, module, "d4r_f16_mma_reference"), "MMA kernel"); + constexpr size_t cases = 512; + std::vector a(cases * 256), b(cases * 128), c(cases * 128), d(cases * 128); + uint32_t random = 0x192a321u; + auto next = [&]() { random = random * 1664525u + 1013904223u; return random; }; + for (size_t n = 0; n < cases; ++n) { + for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = uint16_t(0x2800u + (next() >> 20)); + for (size_t i = 0; i < 128; ++i) { + b[n * 128 + i] = uint16_t((next() & 0x8000u) | (0x3000u + (next() >> 20))); + c[n * 128 + i] = half_bits(n % 3 ? 0. : 0.015625); + } + // Sweep large cancellation terms and a tiny surviving product + // across K positions; this exposes loss hidden by integer fixtures. + if (n < 256) { + for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = half_bits(1.); + std::fill(b.begin() + n * 128, b.begin() + (n + 1) * 128, half_bits(0.)); + for (size_t col = 0; col < 8; ++col) { + b[n * 128 + (n % 16) * 8 + col] = half_bits(32768.); + b[n * 128 + ((n % 16 + 1) % 16) * 8 + col] = half_bits(-32768.); + b[n * 128 + ((n % 16 + 2 + n / 16 % 14) % 16) * 8 + col] = half_bits(0.0009765625); + } + } + } + struct Allocation { + CudaApi& api; CUdeviceptr pointer = 0; + Allocation(CudaApi& api, size_t bytes) : api(api) { api.check(api.cuMemAlloc_v2(&pointer, bytes), "MMA allocate"); } + ~Allocation() { if (pointer) api.cuMemFree_v2(pointer); } + } da(cuda, a.size() * 2), db(cuda, b.size() * 2), dc(cuda, c.size() * 2), dd(cuda, d.size() * 2); + cuda.check(cuda.cuMemcpyHtoD_v2(da.pointer, a.data(), a.size() * 2), "MMA A upload"); + cuda.check(cuda.cuMemcpyHtoD_v2(db.pointer, b.data(), b.size() * 2), "MMA B upload"); + cuda.check(cuda.cuMemcpyHtoD_v2(dc.pointer, c.data(), c.size() * 2), "MMA C upload"); + std::fill(d.begin(), d.end(), uint16_t(0x7e00)); + cuda.check(cuda.cuMemcpyHtoD_v2(dd.pointer, d.data(), d.size() * 2), "MMA NaN sentinel"); + void* parameters[] = {&da.pointer, &db.pointer, &dc.pointer, &dd.pointer}; + cuda.check(cuda.cuLaunchKernel(kernel, cases, 1, 1, 32, 1, 1, 0, nullptr, parameters, nullptr), "MMA launch"); + cuda.check(cuda.cuCtxSynchronize(), "MMA synchronize"); + cuda.check(cuda.cuMemcpyDtoH_v2(d.data(), dd.pointer, d.size() * 2), "MMA download"); + size_t mismatch = 0; + for (size_t n = 0; n < cases; ++n) + for (size_t row = 0; row < 16; ++row) + for (size_t col = 0; col < 8; ++col) { + double sum = half_value(c[n * 128 + row * 8 + col]); + for (size_t k = 0; k < 16; ++k) + sum += half_value(a[n * 256 + row * 16 + k]) * half_value(b[n * 128 + k * 8 + col]); + const uint16_t expected = half_bits(sum); + const uint16_t actual = d[n * 128 + row * 8 + col]; + if (expected != actual) { + if (mismatch++ < 8) std::fprintf(stderr, "MMA_DIFFERENT case=%zu row=%zu column=%zu expected=%04x actual=%04x\n", n, row, col, expected, actual); + } + } + std::printf("MMA_REFERENCE cases=%zu elements=%zu differing=%zu\n", cases, d.size(), mismatch); + if (mismatch) return 5; + std::printf("PASS CUDA_F16_MMA architecture=gfx1201 cases=%zu exact_f16=1\n", cases); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL CUDA_F16_MMA %s\n", error.what()); + loaded_modules(); + return 4; + } +} diff --git a/tools/windows/m_capture_compare.py b/tools/windows/m_capture_compare.py new file mode 100644 index 0000000..6b21ee3 --- /dev/null +++ b/tools/windows/m_capture_compare.py @@ -0,0 +1,54 @@ +"""Compare logical M inputs/outputs between private temporal capture sequences.""" +import argparse +import pathlib + +import numpy as np + +from m_replay_validate import me, LAYERS + + +def logical(parameters, pointer, count, output=False, directory=None): + base, allocation = parameters.alloc_of(pointer) + if output: + identifier = next(i for i, (b, _) in parameters.allocs.items() if b == base) + allocation = np.fromfile(directory / 'gpu-output' / f'alloc-{identifier}.bin', np.uint8) + return allocation[pointer - base:pointer - base + count] + + +def compare(reference, actual): + paths = [] + order = ['enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1'] + for path in reference.glob('replay-*-rrlite_*_4x4'): + layer = path.name.split('rrlite_', 1)[1][:-4] + ordinal = int(path.name.split('-')[1]) + frame = (ordinal - 1) // (6 if layer == 'enc3_tube' else 1) + paths.append((frame, order.index(layer), ordinal, path, layer)) + for frame, _, ordinal, path, layer in sorted(paths): + other = actual / path.name + a, b = me.Params(str(path)), me.Params(str(other)) + c, _, merged, cin = LAYERS[layer] + if (a.W, a.H, a.sx, a.sy, a.grid) != (b.W, b.H, b.sx, b.sy, b.grid): + raise RuntimeError(f'Launch geometry differs: {path.name}') + fields = [('input', 'inp', (a.W // 2) * (a.H // 2) * cin if cin else a.W * a.H * c, False)] + if cin: + fields.append(('skip', 'p32', a.W * a.H * c, False)) + fields.append(('output', 'out', a.W * a.H * c, True)) + if merged: + fields.append(('merge', 'p48', (a.W // 2) * (a.H // 2) * merged, True)) + for label, field, count, output in fields: + x = logical(a, getattr(a, field), count, output, path) + y = logical(b, getattr(b, field), count, output, other) + if x.size != count or y.size != count: + raise RuntimeError(f'Truncated {path.name} {label}') + differing = np.flatnonzero(x != y) + print(f'M_CAPTURE_COMPARE frame={frame} layer={layer} ordinal={ordinal} field={label} ' + f'codes={count} differing={differing.size} ' + f'first={int(differing[0]) if differing.size else -1}', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--reference', type=pathlib.Path, required=True) + parser.add_argument('--actual', type=pathlib.Path, required=True) + args = parser.parse_args() + compare(args.reference, args.actual) diff --git a/tools/windows/m_ptx_validate.py b/tools/windows/m_ptx_validate.py index d4c373e..90237df 100644 --- a/tools/windows/m_ptx_validate.py +++ b/tools/windows/m_ptx_validate.py @@ -18,7 +18,10 @@ def validate(args): raise RuntimeError('Capture kernel does not match requested M layer') native = {i: np.fromfile(args.native_dir / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs} candidates = [] - for bx, by in blocks(parameters.grid, 12): + coordinates = args.block or blocks(parameters.grid, args.candidate_blocks) + for bx, by in coordinates: + if not (0 <= bx < parameters.grid[0] and 0 <= by < parameters.grid[1]): + raise RuntimeError(f'Block {bx},{by} is outside the capture grid') model = model_block(parameters, args.layer, bx, by) expected, actual = stored(parameters, args.layer, bx, by, native, model) difference = float(np.abs(expected - actual).max()) @@ -52,8 +55,12 @@ if __name__ == '__main__': parser.add_argument('--dlss-dll', type=pathlib.Path, required=True) parser.add_argument('--output-dir', type=pathlib.Path, required=True) parser.add_argument('--blocks', type=int, default=2) + parser.add_argument('--candidate-blocks', type=int, default=12, + help='NumPy scan size used to choose the worst blocks for independent PTX') + parser.add_argument('--block', type=lambda value: tuple(map(int, value.split(','))), + action='append', help='Explicit x,y regression block (repeatable)') args = parser.parse_args() - if args.blocks < 1: + if args.blocks < 1 or args.candidate_blocks < 1: parser.error('--blocks must be positive') try: validate(args) diff --git a/tools/windows/m_replay_validate.py b/tools/windows/m_replay_validate.py index 504e807..adc54cd 100644 --- a/tools/windows/m_replay_validate.py +++ b/tools/windows/m_replay_validate.py @@ -142,7 +142,7 @@ def metrics(expected, actual, label): return maximum, peak, psnr, steps -def validate(layer, directory, output, block_count, real_capture): +def validate(layer, directory, output, block_count, real_capture, exact=False): parameters = me.Params(str(directory)) if parameters.kernel != f'rrlite_{layer}_4x4': raise RuntimeError('Replay kernel does not match requested M layer') @@ -151,7 +151,7 @@ def validate(layer, directory, output, block_count, real_capture): for bx, by in blocks(parameters.grid, block_count)] expected, actual = (np.concatenate(values) for values in zip(*pairs)) maximum, peak, psnr, steps = metrics(expected, actual, f'M_REPLAY_REFERENCE kernel={layer}') - if (not real_capture and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)): + if ((exact or not real_capture) and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)): raise RuntimeError(f'M mismatch: exact synthetic / <=1 FP8 step and >=50 dB real gate; peak={peak}') print(f'PASS M_REPLAY_REFERENCE kernel={layer} blocks={len(pairs)} architecture=gfx1201 nan_inf=0') @@ -165,6 +165,7 @@ if __name__ == '__main__': parser.add_argument('--size', type=int, choices=[8, 16, 24, 32], default=16) parser.add_argument('--blocks', type=int, default=12) parser.add_argument('--real-capture', action='store_true') + parser.add_argument('--exact', action='store_true', help='Require every stored real FP8 value to match') args = parser.parse_args() try: if args.mode == 'generate': @@ -172,7 +173,7 @@ if __name__ == '__main__': else: if not args.output_dir or args.blocks < 1: parser.error('validate requires --output-dir and positive --blocks') - validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture) + validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture, args.exact) except Exception as error: print(f'FAIL M_REPLAY {error}', file=sys.stderr) raise SystemExit(5) diff --git a/tools/windows/m_temporal_validate.py b/tools/windows/m_temporal_validate.py new file mode 100644 index 0000000..f46660c --- /dev/null +++ b/tools/windows/m_temporal_validate.py @@ -0,0 +1,53 @@ +"""Validate every captured M launch and spatial block; all allocations stay private.""" +import argparse +import contextlib +import json +import pathlib +import sys +import traceback + +from m_replay_validate import LAYERS, validate + + +def main(args): + args.output_dir.mkdir(parents=True, exist_ok=True) + summary = {'passed': False, 'captured_outputs': True, 'exact': args.exact, 'launches': []} + status = 1 + try: + captures = sorted(args.capture_dir.glob('replay-*-rrlite_*_4x4')) + if not captures: + raise RuntimeError('No audited M captures were found') + for capture in captures: + layer = capture.name.split('rrlite_', 1)[1][:-4] + if layer not in LAYERS: + raise RuntimeError(f'Unknown captured M layer: {layer}') + logfile = args.output_dir / (capture.name + '.log') + with logfile.open('w', encoding='utf-8') as output, contextlib.redirect_stdout(output): + validate(layer, capture, capture / 'gpu-output', args.blocks, True, args.exact) + print(capture.name + ' ' + logfile.read_text().splitlines()[0], flush=True) + summary['launches'].append({'capture': capture.name, 'layer': layer, 'passed': True}) + present = {item['layer'] for item in summary['launches']} + if present != set(LAYERS): + raise RuntimeError('Capture set does not contain all five M layers') + summary['passed'] = True + print(f'PASS M_TEMPORAL_REFERENCE launches={len(captures)} blocks={args.blocks} exact={int(args.exact)}', flush=True) + status = 0 + except Exception as error: + summary['error'] = str(error) + summary['failed_capture'] = capture.name if 'capture' in locals() else None + traceback.print_exc() + finally: + (args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8') + return status + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--capture-dir', type=pathlib.Path, required=True) + parser.add_argument('--output-dir', type=pathlib.Path, required=True) + parser.add_argument('--blocks', type=int, default=10000) + parser.add_argument('--exact', action='store_true') + args = parser.parse_args() + if args.blocks < 1: + parser.error('--blocks must be positive') + sys.exit(main(args)) diff --git a/tools/windows/native_replay_probe.cpp b/tools/windows/native_replay_probe.cpp index 183b56c..92075e3 100644 --- a/tools/windows/native_replay_probe.cpp +++ b/tools/windows/native_replay_probe.cpp @@ -82,6 +82,14 @@ int main(int argc, char** argv) { hipModule_t module = nullptr; hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(replay gfx1201)"); struct Cleanup { HipApi& hip; hipModule_t module; ~Cleanup() { (void)hip.hipModuleUnload(module); } } cleanup{hip, module}; + if (const char* debugBlock = std::getenv("D4R_SWIN_DEBUG_BLOCK")) { + int coordinates[2]{}; + if (std::sscanf(debugBlock, "%d,%d", &coordinates[0], &coordinates[1]) != 2) throw std::runtime_error("D4R_SWIN_DEBUG_BLOCK must be x,y"); + hipDeviceptr_t address = nullptr; size_t size = 0; + hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_block"), "Find Swin diagnostic block"); + if (size != sizeof(coordinates)) throw std::runtime_error("Swin diagnostic coordinates ABI"); + hip.check(hip.hipMemcpy(address, coordinates, size, hipMemcpyHostToDevice), "Set Swin diagnostic block"); + } auto scalar = [&](const char* name, unsigned fallback) { hipDeviceptr_t address = nullptr; size_t size = 0; if (hip.hipModuleGetGlobal(&address, &size, module, name) != hipSuccess) return fallback; @@ -108,6 +116,15 @@ int main(int argc, char** argv) { nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)"); hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)"); std::filesystem::create_directories(output); + if (std::getenv("D4R_SWIN_DEBUG_BLOCK")) { + hipDeviceptr_t address = nullptr; size_t size = 0; + hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_values"), "Find Swin diagnostic stages"); + std::vector bytes(size); + hip.check(hip.hipMemcpy(bytes.data(), address, size, hipMemcpyDeviceToHost), "Download Swin diagnostic stages"); + std::ofstream file(output / "swin-stages.f16", std::ios::binary); + file.write(reinterpret_cast(bytes.data()), std::streamsize(bytes.size())); + if (!file) throw std::runtime_error("Cannot save Swin diagnostic stages"); + } for (const auto& allocation : allocations) { std::vector bytes(allocation->size); hip.check(hip.hipMemcpy(bytes.data(), allocation->device, bytes.size(), hipMemcpyDeviceToHost), "hipMemcpy(replay result)"); diff --git a/tools/windows/swin_debug_compare.py b/tools/windows/swin_debug_compare.py new file mode 100644 index 0000000..51297a4 --- /dev/null +++ b/tools/windows/swin_debug_compare.py @@ -0,0 +1,26 @@ +"""Compare compile-time Swin diagnostic snapshots with the NumPy intermediate values.""" +import argparse +import numpy as np +from m_replay_validate import me, sm, LAYERS, metrics + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--capture-dir', required=True) +parser.add_argument('--stages', required=True) +parser.add_argument('--block', default='0,4') +parser.add_argument('--layer', choices=list(LAYERS), default='enc1') +args = parser.parse_args() +p = me.Params(args.capture_dir) +c, heads, _, cin = LAYERS[args.layer] +bx, by = map(int, args.block.split(',')) +pre = 4 * c * cin + 8 * c if cin else 0 +x0 = sm.decoder_input(p, bx, by, c, cin, heads, p.p32) if cin else sm.load_tokens(p, bx, by, c // 32) +stages = {} +sm.swin_block(p, x0, c, heads, pre, stages=stages) +actual = np.fromfile(args.stages, np.float16).reshape(4, 64, 128).astype(np.float64) +for index, name in enumerate(('h1', 'x1', 'h2', 'O')): + expected = stages[name] + values = actual[index, :, :expected.shape[1]] + metrics(expected, values, f'SWIN_STAGE {name}') + different = np.argwhere(expected != values) + for row, channel in different[:8]: + print(f'DIFFERENT stage={name} token={row} channel={channel} reference={expected[row, channel]} native={values[row, channel]}')