Correct M attention precision and validate every temporal transformer block

CMakeLists.txt

@@ -50,7 +50,7 @@ foreach(family k m)
       list(APPEND native_flags -DD4R_K_FP16_BASELINE)
     endif()
     if(family STREQUAL "m" AND D4R_M_FP16_BASELINE)
-      list(APPEND native_flags -DSWIN_EXACT)
+      list(APPEND native_flags -DSWIN_EXACT -DSWIN_EXACT_PV)
     endif()
     if(flags_line)
       separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}")
@@ -68,7 +68,7 @@ foreach(family k m)
   endforeach()
 endforeach()
 add_custom_target(d4r_native_gfx1201_compile ALL DEPENDS ${native_objects})
-foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe)
+foreach(probe hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_mma_probe cuda_image_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe)
   add_executable(d4r_${probe} tools/windows/${probe}.cpp)
   target_compile_definitions(d4r_${probe} PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
   target_include_directories(d4r_${probe} PRIVATE "${D4R_HIP_ROOT}/include")
@@ -147,6 +147,7 @@ install(FILES tools/windows/k_ptx_validate.py DESTINATION .)
 install(FILES tools/windows/frame_compare.py DESTINATION .)
 install(FILES tools/windows/m_replay_validate.py DESTINATION .)
 install(FILES tools/windows/m_ptx_validate.py DESTINATION .)
+install(FILES tools/windows/m_temporal_validate.py tools/windows/m_capture_compare.py DESTINATION .)
 install(FILES scripts/windows/test-m-replay.ps1 DESTINATION .)
 install(FILES scripts/windows/test-m-captures.ps1 DESTINATION .)
 install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py
@@ -182,6 +183,11 @@ if(EXISTS "${D4R_ZLUDA_ROOT}/nvcuda.dll")
     set_tests_properties(cuda_${context} PROPERTIES TIMEOUT 180 LABELS "hardware;zluda")
   endforeach()
   if(EXISTS "${D4R_ZLUDA_ROOT}/build-info.json")
+    add_test(NAME cuda_f16_mma_reference COMMAND d4r_cuda_mma_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll")
+    set_tests_properties(cuda_f16_mma_reference PROPERTIES TIMEOUT 120
+      LABELS "hardware;zluda;mma;reference"
+      ENVIRONMENT "D4R_ZLUDA_WMMA=1;D4R_ZLUDA_WMMA_F16_REFERENCE=1")
     add_test(NAME cuda_images COMMAND d4r_cuda_image_probe
       --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32)
     set_tests_properties(cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors")

docs/windows-rdna4-port.md

@@ -13,8 +13,12 @@ frames on Windows. M now passes all five synthetic and real-weight layers agains
 NumPy and independent PTX, and four native CUDA frames. The standalone D3D12
 harness now produces four K and four M frames using imported VRAM and shared
 fences, with bit-exact RGB agreement against the corresponding CUDA harnesses.
-Game command-list/queue integration remains in progress. The fully translated M
+Game command-list/queue integration remains in progress. A full temporal M scan
-comparison is being rerun with the correct WMMA options and a writable JIT cache.
+found rare attention P/V accumulation mismatches that the earlier twelve-block
+sampling missed. `SWIN_EXACT_PV` fixes the native baseline; ZLUDA patch `0013`
+adds the corresponding opt-in f16 reference accumulation. All spatial blocks in
+40 actual native M launches now exactly match NumPy. Four complete native M RGB
+frames exactly match fully translated M with the corrected reference option.
 Do not integrate NGX until the integer PTX workload is stable on the real GPU.
 
 | Gate | Status |
@@ -25,11 +29,70 @@ Do not integrate NGX until the integer PTX workload is stable on the real GPU.
 | M3: Windows NGX initialization and simple DLSS path | Init, SR capabilities, Create/Evaluate PASS; E, K and M CUDA harnesses execute |
 | M4: D3D12 / HIP external memory and fence round trip | PASS with TheRock; stable 7.2 has mapped-view leak |
 | M5: independently validated gfx12 WMMA backend | PASS: raw + legacy adapter + upstream layout, max abs/relative error 0 |
-| M6: K layers, full transformer and image validation | CUDA harness PASS: 11 real-weight layers, native launches for all layers, four finite frames, 81.85–92.91 dB versus translated K; D3D12 integration pending |
+| M6: K layers, full transformer and image validation | PASS: 11 real-weight layers, 44 native launches, four D3D12 frames bit-exact against native CUDA, 81.85–92.91 dB versus translated K |
-| M7: M FP16-equivalent baseline and full transformer | CUDA harness PASS: 5 real-weight layers exactly match NumPy/PTX, 4 finite frames, 40 native Swin launches; D3D12 and full translated M comparison pending |
+| M7: M FP16-equivalent baseline and full transformer | PASS: all blocks of 40 native launches exactly match NumPy; targeted independent PTX regression matches; four D3D12 frames bit-exact against corrected translated M |
 | M8: standalone Windows D3D12 NGX harness | PASS at explicit submitted queue boundaries: K/M, four frames each, no shim CPU image copies, bit-exact RGB against CUDA harness |
 | M9: OptiScaler integration, profiling, installation | Pending |
 
+## M temporal precision regression (2026-09-30)
+
+The earlier finite-frame and sampled-layer results did not cover every spatial
+block. The second enc1 launch has a cancellation-sensitive P/V dot in block
+`0,4`: the old native f32 WMMA route differed by up to 21 e4m3 code steps.
+NumPy and the independent PTX interpreter agree exactly on that block.
+`SWIN_EXACT_PV` stores each wave's sixteen P rows and V rows in LDS, sums sixteen
+f16 products in f64, rounds directly to f16, then applies the existing e4m3
+quantisation. Other GEMMs retain the gfx12 WMMA layout and FP16 widening.
+The flag is enabled by `D4R_M_FP16_BASELINE`; upstream fast/Linux defaults are
+preserved when that flag is absent. Intermediate snapshots are compiled only
+with `D4R_SWIN_DIAGNOSTICS`; they are absent from packaged kernels.
+
+The translated reference also differed: first frame, fourth enc3_tube launch,
+block `0,0`, thirty stored codes, maximum fifteen e4m3 steps. Its inputs match
+the corrected native launch exactly. Patch `0013` adds
+`D4R_ZLUDA_WMMA_F16_REFERENCE=1` for gfx12 f16-accumulator MMA helpers, retaining
+the established fragment gathers, pair fusion and conversions. It uses f64
+sums and integer f16 RNE encoding, including subnormal bits. FP8 GEMMs continue
+using the existing native WMMA widening. The default upstream helper behavior
+remains available with this option unset or zero. Windows diagnostics select
+the reference option by default. All d4r codegen options enter the JIT cache key.
+
+This is a match to the independent mathematical NumPy/PTX baseline. NVIDIA's
+[PTX floating-point MMA specification](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-instructions-mma)
+leaves accumulation order, rounding and subnormal handling unspecified; these
+checks do not establish bit-exact agreement with NVIDIA hardware.
+
+Recorded results on the RX 9070 XT / TheRock runtime:
+
+* Public `cuda_mma_probe`: 512 synthetic matrix cases, 65,536 f16 outputs,
+  zero differing bits with reference accumulation. The old fast WMMA route
+  differs in 23,308 outputs on the same cancellation/subnormal stress fixture.
+* `m-native-exact-pv-temporal-reference`: all spatial blocks of 40 actual native
+  launches, every stored e4m3 value identical to NumPy, max abs/relative error 0.
+* `m-exact-pv-enc1-regression-ptx`: original problem block `0,4`, independent
+  PTX versus native and versus NumPy both max absolute/relative error 0.
+* `m-exact-pv-complete-f16-reference.log`: four complete RGB frames against
+  corrected fully translated M, max absolute/relative error 0, PSNR infinite.
+* `m-translated-f16-temporal-reference`: all blocks of all forty translated
+  M launches exactly match NumPy; no sampled-window exemption.
+* `m-temporal-final-ctest.log`: 11/11 native Windows hardware tests PASS.
+  Pinned fresh-source application of all twelve ZLUDA patches PASS.
+* `d3d12-m-exact-pv-final.zip`: four finite frames, forty native launches,
+  no shim CPU image copies, frame age zero. This remains the explicit queue
+  boundary harness; open game command-list support is the next gate.
+
+All captures, extracted PTX and real weights remain in ignored private result
+directories. To repeat the full temporal scan without editing sources:
+
+```powershell
+$env:PYTHONPATH="$PWD/.tools/python/vendor"
+python tools/windows/m_temporal_validate.py --capture-dir test-results/ngx-m-exact-pv-capture/captures --output-dir test-results/m-temporal-reference --exact
+```
+
+The game selected for the next integration test is Silent Hill 2 at
+`D:\Games\SILENT HILL 2\SHProto\Binaries\Win64\SHProto-Win64-Shipping.exe`.
+No game files have been modified at this milestone.
+
 ## Source baseline (checked 2026-09-29)
 
 * d4r: `ed1ab60`, main, 14 commits; https://github.com/countervolts/d4r.

kernels/m/swin_block.h

@@ -5,6 +5,20 @@
 //   CIN  patch-expand input channels (0 = none): decoder input = f16(q8(expand(low-res CIN)) + skip)
 #pragma once
 #include "swin_common.h"
+#ifdef D4R_SWIN_DIAGNOSTICS
+extern "C" {
+__device__ int d4r_swin_debug_block[2] = {0, 4};
+__device__ half_t d4r_swin_debug_values[4 * 64 * 128];
+}
+template <int C, int AST> __device__ __forceinline__ void swin_snapshot(
+    const half_t* activation, int stage, int bx, int by, int tid, bool pair_order) {
+    if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1] && tid < 64)
+        for (int c = 0; c < C; ++c)
+            d4r_swin_debug_values[(stage * 64 + tid) * 128 + c] =
+                activation[tid * AST + (pair_order ? gperm(c) : apos(c))];
+    __syncthreads();
+}
+#endif
 
 // Immediate lane masks need only a compile-time integer, not the host C++
 // standard library (which is absent from device-only Windows HIP builds).
@@ -79,7 +93,12 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     static_assert(NW % NH == 0 && NW / NH <= 4, "waves per head");
     constexpr int NPL = L::NPL, NT = L::NT, INNER = L::INNER;
     constexpr int AST = (C > INNER ? C : INNER) + 8;
-    constexpr int GST = 40, QST = 40, PST = 24;
+    constexpr int GST = 40, PST = 24;
+#ifdef SWIN_EXACT_PV
+    constexpr int QST = 56;
+#else
+    constexpr int QST = 40;
+#endif
 #ifndef SWIN_MLP_R
 #define SWIN_MLP_R(NW) ((NW) / 2)
 #endif
@@ -294,6 +313,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     __syncthreads();
 
     // ------------------------------------------------ stage 2: Q/V projection + attention (wave = head)
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<C, AST>(A, 0, bx, by, tid, false);
+#endif
     {
         constexpr int MW = 4 / (NW / NH); // windows per wave
         const int h = NW == NH ? wv : wv % NH, m0 = NW == NH ? 0 : MW * (wv / NH);
@@ -400,6 +422,13 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
                 Qs[mrow(2 * k + 1) * PST + l16] = pw[1];
             }
             wave_sync();
+#ifdef SWIN_EXACT_PV
+            half_t* Vs = Qs + 16 * PST;
+            for (int nt = 0; nt < 2; ++nt)
+                for (int i = 0; i < 8; ++i)
+                    Vs[mrow(i) * 32 + 16 * nt + l16] = vp[m][nt][i / 2][i & 1];
+            wave_sync();
+#endif
             const sop pa = lds16(Qs + l16 * PST);
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
@@ -422,7 +451,19 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
                 }
                 const h16 vb = __builtin_bit_cast(h16, vw);
 #endif
-                const f8v o = wmma(pa, vb, splat(0.0f));
+                f8v o = wmma(pa, vb, splat(0.0f));
+#ifdef SWIN_EXACT_PV
+                // FP16 P/V products need more than 24 significant bits in a
+                // cancellation-heavy dot. An f32 WMMA can round to the wrong
+                // FP16 neighbour at a midpoint, then flip the FP8 output code.
+                // This baseline sums the 16 FP16 products before one FP16 round.
+                for (int i = 0; i < 8; ++i) {
+                    double dot = 0.0;
+                    for (int k = 0; k < 16; ++k)
+                        dot += (double)Qs[mrow(i) * PST + k] * (double)Vs[k * 32 + 16 * nt + l16];
+                    o[i] = (float)(half_t)dot;
+                }
+#endif
                 const int ocol = apos(32 * h + ginv(16 * nt + l16));
 #pragma unroll
                 for (int i = 0; i < 8; i += 2)
@@ -438,6 +479,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     __syncthreads();
 
     // ------------------------------------------------ stage 3: output projection + residual
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<INNER, AST>(A, 3, bx, by, tid, false);
+#endif
 #pragma unroll
     for (int kc = 0; kc < NH; ++kc)
     {
@@ -471,6 +515,11 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     if (tid < 64)
     {
         // the row as 128-bit LDS loads: element 8 j + c of n8 tile j
+#ifdef D4R_SWIN_DIAGNOSTICS
+        if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1])
+            for (int c = 0; c < C; ++c)
+                d4r_swin_debug_values[(64 + tid) * 128 + c] = A[tid * AST + gperm(c)];
+#endif
         h8 row[L::NT8];
 #pragma unroll
         for (int j = 0; j < L::NT8; ++j)
@@ -529,6 +578,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     __syncthreads();
 
     // ------------------------------------------------ stage 4: MLP, R chunks per round
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<C, AST>(A, 2, bx, by, tid, false);
+#endif
     {
         constexpr int WPC = NW / R, MPW = 4 / WPC; // waves per chunk, fc1 m tiles per wave
         const int cw = wv / WPC, mb = MPW * (wv % WPC);

kernels/tools/swin_model.py

@@ -98,7 +98,7 @@ def layout(C, heads, pre=0):
     L['end'] = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1)
     return L
 
-def swin_block(P, x0, C, heads, pre=0):
+def swin_block(P, x0, C, heads, pre=0, stages=None):
     """x0: 64 x C natural (f16 values). Returns (x2 pair order, out natural fp8 values)."""
     fp8 = lambda T: E4M3[P.wbuf[T]]
     L = layout(C, heads, pre)
@@ -116,6 +116,8 @@ def swin_block(P, x0, C, heads, pre=0):
         Wo[32 * s:32 * s + 32] = woff_table(L['head'][s] + 64 * C + 512, 0, 512, 32, C)
     x1 = mma_chain(pair_to_nat(q8(O)), fp8(Wo), C0=f16(nat_to_pair(x0) + P.f16vec(L['bo'], C)[None, :]))
     h2 = pair_to_nat(rms_from_sum(x1, tree_norm2(x1), P.f16vec(L['g2'], C)))
+    if stages is not None:
+        stages.update(h1=h1, x1=pair_to_nat(x1), h2=h2, O=pair_to_nat(q8(O)))
     x2 = f16(x1 + P.f16vec(L['b2'], C)[None, :])
     idx = gp(np.arange(32))
     for c, B1 in enumerate(L['b1']):

patches/zluda/0013-gfx12-f16-reference-accumulation.patch

@@ -0,0 +1,151 @@
+diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
+index 5a54417..90c0034 100644
+--- a/llvm_zluda/src/compile.rs
++++ b/llvm_zluda/src/compile.rs
+@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
+     let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+     if family == 12 {
+         flags |= 2;
++        // Explicit diagnostic/correctness mode; retain upstream's fast default.
++        if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") {
++            flags |= 8;
++        }
+         // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
+         if flags & 1 != 0 && wmma_fp8_native_requested() {
+             flags |= 4;
+diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
+index 07238db..d262fc6 100644
+--- a/llvm_zluda/src/lib.cpp
++++ b/llvm_zluda/src/lib.cpp
+@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M)
+ // Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
+ // Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
+ // Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
++// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs.
+ // The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
+ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+ {
+@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+     const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
+     const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
+     const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
++    const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16");
+     Helpers H;
+     H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
+     H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
+-    H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
++    H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2);
+     H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
+     H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
+     H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
+-    H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
++    H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
+     {
+         Type *F = Type::getFloatTy(Context);
+         Type *V4F = FixedVectorType::get(F, 4);
+diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
+index 4ff25dd..36bd6b1 100644
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8
+ #endif
+ }
+ 
++// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may
++// reach the other f16 neighbour around cancellation-heavy rounding midpoints.
++// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and
++// round directly to f16 once, matching d4r's independent NumPy/PTX reference.
++// The e4m3 and f32-shadow paths continue using native WMMA.
++__device__ static double wmma12_half_value(uint32_t bits)
++{
++    const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10;
++    const double value = exponent == 0 ? double(magnitude) * 0x1p-24 :
++        double(std::bit_cast<float>(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) :
++                                     ((magnitude << 13) + 0x38000000u)));
++    return bits & 0x8000u ? -value : value;
++}
++
++// Preserve f16 subnormal result bits even when the target's default f16
++// arithmetic mode flushes them. No intermediate f32 rounding is involved.
++__device__ static uint32_t wmma12_half_round(double value)
++{
++    const uint64_t bits = std::bit_cast<uint64_t>(value);
++    const uint32_t sign = uint32_t(bits >> 48) & 0x8000u;
++    const uint32_t exponent = uint32_t((bits >> 52) & 2047u);
++    const uint64_t fraction = bits & 0xfffffffffffffu;
++    if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u);
++    const int power = int(exponent) - 1023;
++    if (power < -25) return sign;
++    if (power > 15) return sign | 0x7c00u;
++    const unsigned shift = power < -14 ? unsigned(28 - power) : 42u;
++    const uint64_t significand = fraction | 0x10000000000000ull;
++    uint32_t code = uint32_t(significand >> shift);
++    const uint64_t tail = significand & ((uint64_t(1) << shift) - 1);
++    const uint64_t midpoint = uint64_t(1) << (shift - 1);
++    code += tail > midpoint || (tail == midpoint && (code & 1u));
++    if (power >= -14) code += uint32_t(power + 14) << 10;
++    return sign | code;
++}
++
++__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired)
++{
++    const bool high = wmma12_half() != 0;
++    const uint2::Native_vec_ own = paired && high ? c1 : c0;
++    const uint2::Native_vec_ send = paired && !high ? c1 : c0;
++    const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
++    const uint2::Native_vec_ even = paired && high ? other : own;
++    const uint2::Native_vec_ odd = paired && high ? own : other;
++    return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u),
++            (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)};
++}
++
++__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c)
++{
++    // x/y: packed even rows, z/w: packed odd rows, in this half-wave.
++    wmma_uint4 result = {};
++    const uint32_t half = wmma12_half();
++    for (uint32_t i = 0; i < 8; ++i)
++    {
++        const uint32_t row = i + 8 * half;
++        double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu);
++        for (uint32_t khalf = 0; khalf < 2; ++khalf)
++            for (uint32_t j = 0; j < 4; ++j)
++            {
++                const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]);
++                const uint32_t other = wmma12_other_half(b[j]);
++                const uint32_t bv = half == khalf ? b[j] : other;
++                sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu);
++                sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16);
++            }
++        // Do not round through f32: that can reintroduce the midpoint error.
++        result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1));
++    }
++    return result;
++}
++
+ // f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
+ // e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
+ __device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+@@ -1959,6 +2031,22 @@ extern "C"
+         return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
+     }
+ 
++    uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
++    {
++        const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false));
++        const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)};
++        return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y};
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
++    {
++        const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true));
++        const bool high = wmma12_half() != 0;
++        const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z);
++        const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w);
++        return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1};
++    }
++
+     wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
+     {
+         return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;

scripts/windows/stage-native-m.ps1

@@ -16,7 +16,7 @@ foreach ($layer in @('enc1','enc2','enc3_tube','dec2','dec1')) {
 }
 & python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll
 if ($LASTEXITCODE) { throw "Native manifest generation failed ($LASTEXITCODE)" }
-@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; FP16 widening; native FP8 disabled';
+@{architecture='gfx1201'; family='M'; numerics='SWIN_EXACT; SWIN_EXACT_PV; FP16 widening; native FP8 disabled';
     dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash;
     validation='synthetic layers passed; actual DLSS capture/reference validation required';
     modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} |

scripts/windows/test-windows-rdna4.ps1

@@ -53,7 +53,7 @@ $originalPythonPath = $env:PYTHONPATH
 $originalVerbose = $env:D4R_ZLUDA_VERBOSE
 $originalCache = $env:ZLUDA_CACHE_DIR
 $originalCodegen = @{}
-foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) {
+foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) {
     $originalCodegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process')
 }
 $exitStatus = 1
@@ -103,14 +103,14 @@ function Invoke-Probe([string]$Name, [string]$Exe, [string[]]$Arguments) {
 try {
     if (!$env:ZLUDA_CACHE_DIR) { $env:ZLUDA_CACHE_DIR = Join-Path $repoRoot 'build/zluda-cache-windows' }
     $summary.zludaCacheDirectory = $env:ZLUDA_CACHE_DIR
-    foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE')) {
+    foreach ($setting in @('D4R_ZLUDA_WMMA','D4R_ZLUDA_WMMA_FP8','D4R_ZLUDA_WMMA_FP8_NATIVE','D4R_ZLUDA_WMMA_F16_REFERENCE')) {
         if ($null -eq $originalCodegen[$setting]) {
             [Environment]::SetEnvironmentVariable($setting, $(if ($setting -eq 'D4R_ZLUDA_WMMA_FP8_NATIVE') { '0' } else { '1' }), 'Process')
         }
     }
     $summary.codegen = @{}
     foreach ($setting in $originalCodegen.Keys) { $summary.codegen[$setting] = [Environment]::GetEnvironmentVariable($setting,'Process') }
-    Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE"
+    Write-Host "ZLUDA codegen: WMMA=$env:D4R_ZLUDA_WMMA FP8 widening=$env:D4R_ZLUDA_WMMA_FP8 native FP8=$env:D4R_ZLUDA_WMMA_FP8_NATIVE f16 reference=$env:D4R_ZLUDA_WMMA_F16_REFERENCE"
     if ($NgxOnly -and (!$NgxCore -or !$DlssDll)) { throw '-NgxOnly requires both locally supplied NVIDIA DLL paths.' }
     if ($RequireNativeNetwork) {
         if ($NgxMode -notin @('evaluate','d3d12') -or $Preset -notin @(11,13) -or !$env:D4R_ZLUDA_NATIVE_DIR) {
@@ -197,6 +197,14 @@ try {
         if ($cudaOk -and ($NgxMode -eq 'evaluate' -or (Test-Path (Join-Path $ZludaRoot 'build-info.json')))) {
             $cudaOk = Invoke-Probe 'cuda-images' (Join-Path $bin 'd4r_cuda_image_probe.exe') @(
                 '--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'), '--iterations', "$Iterations")
+            if ($cudaOk) {
+                $previousF16 = $env:D4R_ZLUDA_WMMA_F16_REFERENCE
+                try {
+                    $env:D4R_ZLUDA_WMMA_F16_REFERENCE = '1'
+                    $cudaOk = Invoke-Probe 'cuda-f16-mma-reference' (Join-Path $bin 'd4r_cuda_mma_probe.exe') @(
+                        '--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'))
+                } finally { $env:D4R_ZLUDA_WMMA_F16_REFERENCE = $previousF16 }
+            }
         }
         }
         if ($cudaOk) {

tools/windows/cuda_mma_probe.cpp

@@ -0,0 +1,165 @@
+#include "hip_api.h"
+#include "cuda_api.h"
+#include <array>
+#include <cmath>
+#include <cstring>
+
+// Public synthetic matrix fragments, independent of NVIDIA DLLs and weights.
+static constexpr char source[] = R"PTX(
+.version 7.0
+.target sm_75
+.address_size 64
+.visible .entry d4r_f16_mma_reference(
+ .param .u64 A, .param .u64 B, .param .u64 C, .param .u64 D)
+{
+ .reg .b32 %r<24>;
+ .reg .b64 %rd<12>;
+ .reg .b16 %h<4>;
+ ld.param.u64 %rd0, [A];
+ ld.param.u64 %rd1, [B];
+ ld.param.u64 %rd2, [C];
+ ld.param.u64 %rd3, [D];
+ mov.u32 %r0, %ctaid.x;
+ mov.u32 %r1, %tid.x;
+ shr.u32 %r2, %r1, 2;
+ and.b32 %r3, %r1, 3;
+ shl.b32 %r3, %r3, 1;
+ mul.wide.u32 %rd4, %r0, 512;
+ add.u64 %rd0, %rd0, %rd4;
+ mul.wide.u32 %rd4, %r0, 256;
+ add.u64 %rd1, %rd1, %rd4;
+ add.u64 %rd2, %rd2, %rd4;
+ add.u64 %rd3, %rd3, %rd4;
+ mad.lo.u32 %r4, %r2, 16, %r3;
+ mul.wide.u32 %rd4, %r4, 2;
+ add.u64 %rd5, %rd0, %rd4;
+ ld.global.u32 %r8, [%rd5];
+ ld.global.u32 %r9, [%rd5+256];
+ ld.global.u32 %r10, [%rd5+16];
+ ld.global.u32 %r11, [%rd5+272];
+ mad.lo.u32 %r4, %r3, 8, %r2;
+ mul.wide.u32 %rd4, %r4, 2;
+ add.u64 %rd5, %rd1, %rd4;
+ ld.global.b16 %h0, [%rd5];
+ ld.global.b16 %h1, [%rd5+16];
+ mov.b32 %r12, {%h0,%h1};
+ ld.global.b16 %h2, [%rd5+128];
+ ld.global.b16 %h3, [%rd5+144];
+ mov.b32 %r13, {%h2,%h3};
+ mad.lo.u32 %r4, %r2, 8, %r3;
+ mul.wide.u32 %rd4, %r4, 2;
+ add.u64 %rd5, %rd2, %rd4;
+ ld.global.u32 %r14, [%rd5];
+ ld.global.u32 %r15, [%rd5+128];
+ mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16
+  {%r16,%r17}, {%r8,%r9,%r10,%r11}, {%r12,%r13}, {%r14,%r15};
+ add.u64 %rd5, %rd3, %rd4;
+ st.global.u32 [%rd5], %r16;
+ st.global.u32 [%rd5+128], %r17;
+ ret;
+}
+)PTX";
+
+static uint16_t half_bits(double value) {
+    _Float16 half = static_cast<_Float16>(value);
+    uint16_t bits;
+    std::memcpy(&bits, &half, sizeof(bits));
+    return bits;
+}
+static double half_value(uint16_t bits) {
+    _Float16 half;
+    std::memcpy(&half, &bits, sizeof(bits));
+    return static_cast<double>(half);
+}
+
+int main(int argc, char** argv) {
+    using namespace d4r::diag;
+    start();
+    try {
+        Args args(argc, argv);
+        HipApi hip(args.hip_root);
+        hipDeviceProp_t props{};
+        hip.select_gfx1201(args.device, props);
+        CudaApi cuda(args.cuda_dll);
+        cuda.check(cuda.cuInit(0), "cuInit");
+        CUdevice device = -1;
+        int count = 0;
+        cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
+        for (int i = 0; i < count; ++i) {
+            CUdevice candidate;
+            int bus, pci, domain;
+            cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
+            cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
+            cuda.check(cuda.cuDeviceGetAttribute(&pci, 34, candidate), "PCI device");
+            cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
+            if (bus == props.pciBusID && pci == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
+        }
+        if (device < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
+        CUcontext context = nullptr;
+        cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, device), "primary context");
+        struct Context { CudaApi& api; CUdevice device; ~Context() { api.cuCtxSetCurrent(nullptr); api.cuDevicePrimaryCtxRelease_v2(device); } } context_owner{cuda, device};
+        cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent");
+        CUmodule module = nullptr;
+        cuda.check(cuda.cuModuleLoadData(&module, source), "MMA PTX JIT");
+        struct Module { CudaApi& api; CUmodule module; ~Module() { api.cuModuleUnload(module); } } module_owner{cuda, module};
+        CUfunction kernel = nullptr;
+        cuda.check(cuda.cuModuleGetFunction(&kernel, module, "d4r_f16_mma_reference"), "MMA kernel");
+        constexpr size_t cases = 512;
+        std::vector<uint16_t> a(cases * 256), b(cases * 128), c(cases * 128), d(cases * 128);
+        uint32_t random = 0x192a321u;
+        auto next = [&]() { random = random * 1664525u + 1013904223u; return random; };
+        for (size_t n = 0; n < cases; ++n) {
+            for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = uint16_t(0x2800u + (next() >> 20));
+            for (size_t i = 0; i < 128; ++i) {
+                b[n * 128 + i] = uint16_t((next() & 0x8000u) | (0x3000u + (next() >> 20)));
+                c[n * 128 + i] = half_bits(n % 3 ? 0. : 0.015625);
+            }
+            // Sweep large cancellation terms and a tiny surviving product
+            // across K positions; this exposes loss hidden by integer fixtures.
+            if (n < 256) {
+                for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = half_bits(1.);
+                std::fill(b.begin() + n * 128, b.begin() + (n + 1) * 128, half_bits(0.));
+                for (size_t col = 0; col < 8; ++col) {
+                    b[n * 128 + (n % 16) * 8 + col] = half_bits(32768.);
+                    b[n * 128 + ((n % 16 + 1) % 16) * 8 + col] = half_bits(-32768.);
+                    b[n * 128 + ((n % 16 + 2 + n / 16 % 14) % 16) * 8 + col] = half_bits(0.0009765625);
+                }
+            }
+        }
+        struct Allocation {
+            CudaApi& api; CUdeviceptr pointer = 0;
+            Allocation(CudaApi& api, size_t bytes) : api(api) { api.check(api.cuMemAlloc_v2(&pointer, bytes), "MMA allocate"); }
+            ~Allocation() { if (pointer) api.cuMemFree_v2(pointer); }
+        } da(cuda, a.size() * 2), db(cuda, b.size() * 2), dc(cuda, c.size() * 2), dd(cuda, d.size() * 2);
+        cuda.check(cuda.cuMemcpyHtoD_v2(da.pointer, a.data(), a.size() * 2), "MMA A upload");
+        cuda.check(cuda.cuMemcpyHtoD_v2(db.pointer, b.data(), b.size() * 2), "MMA B upload");
+        cuda.check(cuda.cuMemcpyHtoD_v2(dc.pointer, c.data(), c.size() * 2), "MMA C upload");
+        std::fill(d.begin(), d.end(), uint16_t(0x7e00));
+        cuda.check(cuda.cuMemcpyHtoD_v2(dd.pointer, d.data(), d.size() * 2), "MMA NaN sentinel");
+        void* parameters[] = {&da.pointer, &db.pointer, &dc.pointer, &dd.pointer};
+        cuda.check(cuda.cuLaunchKernel(kernel, cases, 1, 1, 32, 1, 1, 0, nullptr, parameters, nullptr), "MMA launch");
+        cuda.check(cuda.cuCtxSynchronize(), "MMA synchronize");
+        cuda.check(cuda.cuMemcpyDtoH_v2(d.data(), dd.pointer, d.size() * 2), "MMA download");
+        size_t mismatch = 0;
+        for (size_t n = 0; n < cases; ++n)
+            for (size_t row = 0; row < 16; ++row)
+                for (size_t col = 0; col < 8; ++col) {
+                    double sum = half_value(c[n * 128 + row * 8 + col]);
+                    for (size_t k = 0; k < 16; ++k)
+                        sum += half_value(a[n * 256 + row * 16 + k]) * half_value(b[n * 128 + k * 8 + col]);
+                    const uint16_t expected = half_bits(sum);
+                    const uint16_t actual = d[n * 128 + row * 8 + col];
+                    if (expected != actual) {
+                        if (mismatch++ < 8) std::fprintf(stderr, "MMA_DIFFERENT case=%zu row=%zu column=%zu expected=%04x actual=%04x\n", n, row, col, expected, actual);
+                    }
+                }
+        std::printf("MMA_REFERENCE cases=%zu elements=%zu differing=%zu\n", cases, d.size(), mismatch);
+        if (mismatch) return 5;
+        std::printf("PASS CUDA_F16_MMA architecture=gfx1201 cases=%zu exact_f16=1\n", cases);
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "FAIL CUDA_F16_MMA %s\n", error.what());
+        loaded_modules();
+        return 4;
+    }
+}

tools/windows/m_capture_compare.py

@@ -0,0 +1,54 @@
+"""Compare logical M inputs/outputs between private temporal capture sequences."""
+import argparse
+import pathlib
+
+import numpy as np
+
+from m_replay_validate import me, LAYERS
+
+
+def logical(parameters, pointer, count, output=False, directory=None):
+    base, allocation = parameters.alloc_of(pointer)
+    if output:
+        identifier = next(i for i, (b, _) in parameters.allocs.items() if b == base)
+        allocation = np.fromfile(directory / 'gpu-output' / f'alloc-{identifier}.bin', np.uint8)
+    return allocation[pointer - base:pointer - base + count]
+
+
+def compare(reference, actual):
+    paths = []
+    order = ['enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1']
+    for path in reference.glob('replay-*-rrlite_*_4x4'):
+        layer = path.name.split('rrlite_', 1)[1][:-4]
+        ordinal = int(path.name.split('-')[1])
+        frame = (ordinal - 1) // (6 if layer == 'enc3_tube' else 1)
+        paths.append((frame, order.index(layer), ordinal, path, layer))
+    for frame, _, ordinal, path, layer in sorted(paths):
+        other = actual / path.name
+        a, b = me.Params(str(path)), me.Params(str(other))
+        c, _, merged, cin = LAYERS[layer]
+        if (a.W, a.H, a.sx, a.sy, a.grid) != (b.W, b.H, b.sx, b.sy, b.grid):
+            raise RuntimeError(f'Launch geometry differs: {path.name}')
+        fields = [('input', 'inp', (a.W // 2) * (a.H // 2) * cin if cin else a.W * a.H * c, False)]
+        if cin:
+            fields.append(('skip', 'p32', a.W * a.H * c, False))
+        fields.append(('output', 'out', a.W * a.H * c, True))
+        if merged:
+            fields.append(('merge', 'p48', (a.W // 2) * (a.H // 2) * merged, True))
+        for label, field, count, output in fields:
+            x = logical(a, getattr(a, field), count, output, path)
+            y = logical(b, getattr(b, field), count, output, other)
+            if x.size != count or y.size != count:
+                raise RuntimeError(f'Truncated {path.name} {label}')
+            differing = np.flatnonzero(x != y)
+            print(f'M_CAPTURE_COMPARE frame={frame} layer={layer} ordinal={ordinal} field={label} '
+                  f'codes={count} differing={differing.size} '
+                  f'first={int(differing[0]) if differing.size else -1}', flush=True)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--reference', type=pathlib.Path, required=True)
+    parser.add_argument('--actual', type=pathlib.Path, required=True)
+    args = parser.parse_args()
+    compare(args.reference, args.actual)

tools/windows/m_ptx_validate.py

@@ -18,7 +18,10 @@ def validate(args):
         raise RuntimeError('Capture kernel does not match requested M layer')
     native = {i: np.fromfile(args.native_dir / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs}
     candidates = []
-    for bx, by in blocks(parameters.grid, 12):
+    coordinates = args.block or blocks(parameters.grid, args.candidate_blocks)
+    for bx, by in coordinates:
+        if not (0 <= bx < parameters.grid[0] and 0 <= by < parameters.grid[1]):
+            raise RuntimeError(f'Block {bx},{by} is outside the capture grid')
         model = model_block(parameters, args.layer, bx, by)
         expected, actual = stored(parameters, args.layer, bx, by, native, model)
         difference = float(np.abs(expected - actual).max())
@@ -52,8 +55,12 @@ if __name__ == '__main__':
     parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
     parser.add_argument('--output-dir', type=pathlib.Path, required=True)
     parser.add_argument('--blocks', type=int, default=2)
+    parser.add_argument('--candidate-blocks', type=int, default=12,
+                        help='NumPy scan size used to choose the worst blocks for independent PTX')
+    parser.add_argument('--block', type=lambda value: tuple(map(int, value.split(','))),
+                        action='append', help='Explicit x,y regression block (repeatable)')
     args = parser.parse_args()
-    if args.blocks < 1:
+    if args.blocks < 1 or args.candidate_blocks < 1:
         parser.error('--blocks must be positive')
     try:
         validate(args)

tools/windows/m_replay_validate.py

@@ -142,7 +142,7 @@ def metrics(expected, actual, label):
     return maximum, peak, psnr, steps
 
 
-def validate(layer, directory, output, block_count, real_capture):
+def validate(layer, directory, output, block_count, real_capture, exact=False):
     parameters = me.Params(str(directory))
     if parameters.kernel != f'rrlite_{layer}_4x4':
         raise RuntimeError('Replay kernel does not match requested M layer')
@@ -151,7 +151,7 @@ def validate(layer, directory, output, block_count, real_capture):
              for bx, by in blocks(parameters.grid, block_count)]
     expected, actual = (np.concatenate(values) for values in zip(*pairs))
     maximum, peak, psnr, steps = metrics(expected, actual, f'M_REPLAY_REFERENCE kernel={layer}')
-    if (not real_capture and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)):
+    if ((exact or not real_capture) and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)):
         raise RuntimeError(f'M mismatch: exact synthetic / <=1 FP8 step and >=50 dB real gate; peak={peak}')
     print(f'PASS M_REPLAY_REFERENCE kernel={layer} blocks={len(pairs)} architecture=gfx1201 nan_inf=0')
 
@@ -165,6 +165,7 @@ if __name__ == '__main__':
     parser.add_argument('--size', type=int, choices=[8, 16, 24, 32], default=16)
     parser.add_argument('--blocks', type=int, default=12)
     parser.add_argument('--real-capture', action='store_true')
+    parser.add_argument('--exact', action='store_true', help='Require every stored real FP8 value to match')
     args = parser.parse_args()
     try:
         if args.mode == 'generate':
@@ -172,7 +173,7 @@ if __name__ == '__main__':
         else:
             if not args.output_dir or args.blocks < 1:
                 parser.error('validate requires --output-dir and positive --blocks')
-            validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture)
+            validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture, args.exact)
     except Exception as error:
         print(f'FAIL M_REPLAY {error}', file=sys.stderr)
         raise SystemExit(5)

tools/windows/m_temporal_validate.py

@@ -0,0 +1,53 @@
+"""Validate every captured M launch and spatial block; all allocations stay private."""
+import argparse
+import contextlib
+import json
+import pathlib
+import sys
+import traceback
+
+from m_replay_validate import LAYERS, validate
+
+
+def main(args):
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    summary = {'passed': False, 'captured_outputs': True, 'exact': args.exact, 'launches': []}
+    status = 1
+    try:
+        captures = sorted(args.capture_dir.glob('replay-*-rrlite_*_4x4'))
+        if not captures:
+            raise RuntimeError('No audited M captures were found')
+        for capture in captures:
+            layer = capture.name.split('rrlite_', 1)[1][:-4]
+            if layer not in LAYERS:
+                raise RuntimeError(f'Unknown captured M layer: {layer}')
+            logfile = args.output_dir / (capture.name + '.log')
+            with logfile.open('w', encoding='utf-8') as output, contextlib.redirect_stdout(output):
+                validate(layer, capture, capture / 'gpu-output', args.blocks, True, args.exact)
+            print(capture.name + ' ' + logfile.read_text().splitlines()[0], flush=True)
+            summary['launches'].append({'capture': capture.name, 'layer': layer, 'passed': True})
+        present = {item['layer'] for item in summary['launches']}
+        if present != set(LAYERS):
+            raise RuntimeError('Capture set does not contain all five M layers')
+        summary['passed'] = True
+        print(f'PASS M_TEMPORAL_REFERENCE launches={len(captures)} blocks={args.blocks} exact={int(args.exact)}', flush=True)
+        status = 0
+    except Exception as error:
+        summary['error'] = str(error)
+        summary['failed_capture'] = capture.name if 'capture' in locals() else None
+        traceback.print_exc()
+    finally:
+        (args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
+    return status
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--capture-dir', type=pathlib.Path, required=True)
+    parser.add_argument('--output-dir', type=pathlib.Path, required=True)
+    parser.add_argument('--blocks', type=int, default=10000)
+    parser.add_argument('--exact', action='store_true')
+    args = parser.parse_args()
+    if args.blocks < 1:
+        parser.error('--blocks must be positive')
+    sys.exit(main(args))

tools/windows/native_replay_probe.cpp

@@ -82,6 +82,14 @@ int main(int argc, char** argv) {
         hipModule_t module = nullptr;
         hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(replay gfx1201)");
         struct Cleanup { HipApi& hip; hipModule_t module; ~Cleanup() { (void)hip.hipModuleUnload(module); } } cleanup{hip, module};
+        if (const char* debugBlock = std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
+            int coordinates[2]{};
+            if (std::sscanf(debugBlock, "%d,%d", &coordinates[0], &coordinates[1]) != 2) throw std::runtime_error("D4R_SWIN_DEBUG_BLOCK must be x,y");
+            hipDeviceptr_t address = nullptr; size_t size = 0;
+            hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_block"), "Find Swin diagnostic block");
+            if (size != sizeof(coordinates)) throw std::runtime_error("Swin diagnostic coordinates ABI");
+            hip.check(hip.hipMemcpy(address, coordinates, size, hipMemcpyHostToDevice), "Set Swin diagnostic block");
+        }
         auto scalar = [&](const char* name, unsigned fallback) {
             hipDeviceptr_t address = nullptr; size_t size = 0;
             if (hip.hipModuleGetGlobal(&address, &size, module, name) != hipSuccess) return fallback;
@@ -108,6 +116,15 @@ int main(int argc, char** argv) {
                 nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)");
         hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)");
         std::filesystem::create_directories(output);
+        if (std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
+            hipDeviceptr_t address = nullptr; size_t size = 0;
+            hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_values"), "Find Swin diagnostic stages");
+            std::vector<uint8_t> bytes(size);
+            hip.check(hip.hipMemcpy(bytes.data(), address, size, hipMemcpyDeviceToHost), "Download Swin diagnostic stages");
+            std::ofstream file(output / "swin-stages.f16", std::ios::binary);
+            file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
+            if (!file) throw std::runtime_error("Cannot save Swin diagnostic stages");
+        }
         for (const auto& allocation : allocations) {
             std::vector<uint8_t> bytes(allocation->size);
             hip.check(hip.hipMemcpy(bytes.data(), allocation->device, bytes.size(), hipMemcpyDeviceToHost), "hipMemcpy(replay result)");

tools/windows/swin_debug_compare.py

@@ -0,0 +1,26 @@
+"""Compare compile-time Swin diagnostic snapshots with the NumPy intermediate values."""
+import argparse
+import numpy as np
+from m_replay_validate import me, sm, LAYERS, metrics
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('--capture-dir', required=True)
+parser.add_argument('--stages', required=True)
+parser.add_argument('--block', default='0,4')
+parser.add_argument('--layer', choices=list(LAYERS), default='enc1')
+args = parser.parse_args()
+p = me.Params(args.capture_dir)
+c, heads, _, cin = LAYERS[args.layer]
+bx, by = map(int, args.block.split(','))
+pre = 4 * c * cin + 8 * c if cin else 0
+x0 = sm.decoder_input(p, bx, by, c, cin, heads, p.p32) if cin else sm.load_tokens(p, bx, by, c // 32)
+stages = {}
+sm.swin_block(p, x0, c, heads, pre, stages=stages)
+actual = np.fromfile(args.stages, np.float16).reshape(4, 64, 128).astype(np.float64)
+for index, name in enumerate(('h1', 'x1', 'h2', 'O')):
+    expected = stages[name]
+    values = actual[index, :, :expected.shape[1]]
+    metrics(expected, values, f'SWIN_STAGE {name}')
+    different = np.argwhere(expected != values)
+    for row, channel in different[:8]:
+        print(f'DIFFERENT stage={name} token={row} channel={channel} reference={expected[row, channel]} native={values[row, channel]}')