diff --git a/CMakeLists.txt b/CMakeLists.txt index 55a9211..676e738 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ add_custom_target(d4r_wmma_code_object ALL DEPENDS "${wmma_object}") # directory, until each layer and then the complete network pass replay. # M uses the existing FP16 widening baseline; native FP8 is not enabled here. set(native_objects) +option(D4R_K_FP16_BASELINE "Round every K16 accumulation step like NVIDIA f16 WMMA" ON) foreach(family k m) file(GLOB native_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${family}/*.hip") file(GLOB native_headers CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${family}/*.h") @@ -44,8 +45,12 @@ foreach(family k m) file(READ "${source}" kernel_source) string(REGEX MATCH "// d4r-build-flags: ([^\r\n]+)" flags_line "${kernel_source}") set(native_flags) + if(family STREQUAL "k" AND D4R_K_FP16_BASELINE) + list(APPEND native_flags -DD4R_K_FP16_BASELINE) + endif() if(flags_line) - separate_arguments(native_flags NATIVE_COMMAND "${CMAKE_MATCH_1}") + separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}") + list(APPEND native_flags ${source_flags}) endif() add_custom_command(OUTPUT "${native_object}" COMMAND "${D4R_HIP_CLANG}" -x hip -std=c++17 --offload-arch=gfx1201 @@ -116,9 +121,14 @@ install(FILES "${probe_object}" DESTINATION bin) install(FILES "${wmma_object}" DESTINATION bin) install(FILES scripts/windows/test-windows-rdna4.ps1 DESTINATION .) install(FILES scripts/windows/test-k-replay.ps1 DESTINATION .) +install(FILES scripts/windows/test-k-captures.ps1 DESTINATION .) +install(FILES scripts/windows/test-native-overrides.ps1 DESTINATION .) install(FILES tools/windows/k_layer_validate.py DESTINATION .) install(FILES tools/windows/k_replay_validate.py DESTINATION .) -install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py DESTINATION reference) +install(FILES tools/windows/k_ptx_validate.py DESTINATION .) +install(FILES tools/windows/frame_compare.py DESTINATION .) +install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py + kernels/tools/extract_dlss_ptx.py kernels/tools/kernel_manifest.py DESTINATION reference) install(FILES docs/windows-rdna4-port.md DESTINATION docs) include(CTest) add_test(NAME hip_gfx1201 COMMAND d4r_hip_gfx1201_probe diff --git a/docs/windows-rdna4-port.md b/docs/windows-rdna4-port.md index d59b49d..b24eba4 100644 --- a/docs/windows-rdna4-port.md +++ b/docs/windows-rdna4-port.md @@ -7,8 +7,9 @@ This is a work log for an implementation in progress, not a claim of DLSS suppor ## Milestone and gates Current milestone: M1/M2/M4/M5 passed; native NGX Create/Evaluate on the simple -requested E path passes with the user's DLLs. M6 is in progress: all 11 K layers -run on gfx1201 and pass synthetic nonzero NumPy reference checks. +requested E path passes with the user's DLLs. All 11 K layers now pass synthetic +and real-weight NumPy/PTX replay checks; native K Evaluate produces four finite +frames on Windows. D3D12 shim integration and M remain in progress. Do not integrate NGX until the integer PTX workload is stable on the real GPU. | Gate | Status | @@ -19,7 +20,7 @@ Do not integrate NGX until the integer PTX workload is stable on the real GPU. | M3: Windows NGX initialization and simple DLSS path | Init, SR capabilities, Create/Evaluate PASS; four finite synthetic frames; K/M pending | | M4: D3D12 / HIP external memory and fence round trip | PASS with TheRock; stable 7.2 has mapped-view leak | | M5: independently validated gfx12 WMMA backend | PASS: raw + legacy adapter + upstream layout, max abs/relative error 0 | -| M6: K layers, full transformer and image validation | All 11 synthetic layer replays PASS; real weights, full network and K image validation pending | +| M6: K layers, full transformer and image validation | CUDA harness PASS: 11 real-weight layers, native launches for all layers, four finite frames, 81.85–92.91 dB versus translated K; D3D12 integration pending | | M7: M FP16-equivalent baseline and full transformer | Pending | | M8: standalone Windows D3D12 NGX harness | Pending | | M9: OptiScaler integration, profiling, installation | Pending | @@ -123,6 +124,75 @@ FP8/performance work. No complete DLSS or OptiScaler game result is claimed. ## Native Windows ZLUDA build and Evaluate work (2026-09-30) +### K real-weight native Evaluate and correctness baseline + +The locally supplied DLSS 310.9.1.0 selects all eleven `dltss_pwin_*` functions +with preset 11 (K). `0010-native-identity-replay-windows-cache.patch` adds: + +* FNV-1a PTX identity checking against upstream `d4r-kernels.txt`, with an + explicit native miss/translated fallback log. A mismatched version cannot + receive an override by function name alone. +* Owned native modules/images, released with their CUDA function/module, and + direct native loading for fully replaced single-entry modules with only LDS + declarations. Modules with exported globals or other functions retain JIT. +* Windows loaded-DLL size/mtime in the translator cache identity. Rebuilding + an uncommitted translator no longer reuses its older cached code. +* Opt-in native Windows K replay capture, before and after a launch, using + HIP allocation range queries. Captures are synchronous diagnostics, can + contain proprietary weights, and are never enabled in the game fast path. + `D4R_ZLUDA_VERBOSE=1` logs actual native/translated launches. + +The Windows minimal `__syncthreads` previously omitted HIP's release/acquire +memory fences. Real, multi-window captures exposed nondeterministic LDS data +and intermediate NaNs, despite the one-window synthetic tests passing. The +shim now matches public HIP barrier semantics. Those earlier captures are +invalid for correctness claims. + +Windows K defaults to `D4R_K_FP16_BASELINE=ON`: every K16 accumulator step and +the L2 reduction tree round to FP16. The norm grouping was independently +recovered and checked with the existing CPU PTX interpreter for C=32/64/96/128/160. +The reference model supports `NORM_FRAGMENT_ORDER=True`; its legacy default +remains unchanged. The original float norm approximation itself differed from +PTX by 0.13965 on one real decoder window, so it cannot serve as a bit-exact +oracle. Strict references and the independent PTX gate are both required. + +Validated on RX 9070 XT with isolated TheRock: + +* `test-results/k-strict-synthetic-final.zip`: 11/11, 32 launches per layer, + **max absolute and relative error 0** with strict references. +* `test-results/k-real-strict-dual-reference`: all eleven real-weight layers + pass. Up to 12 NumPy windows/layer, plus the two worst NumPy difference + windows checked independently through PTX. NumPy PSNR 75.40–103.98 dB; + PTX PSNR 67.11–122.39 dB, all selected output elements finite. Max relative + error is reported with a 0.001 denominator floor; near-zero values can have + a large relative error. Gates require PSNR >=60 dB and max absolute error + <=max(0.001, 0.005*reference peak). +* `test-results/ngx-k-strict-norm-final.zip`: four K frames, all 44 native + transformer launches logged, no NaN/Inf or unwritten output. +* Complete-frame comparison with translated WMMA K: PSNR 92.91, 81.85, 84.45, + 86.10 dB, max absolute error 0.01514. Differences are localized: at most + 21 RGB elements exceed 0.002, and at most 0.00294% exceed 1% of peak. + These are approximate numerical results, not bit-exact NVIDIA hardware + verification. The frame gate requires >=60 dB, max <=2.5% of peak, and + no more than 0.01% of RGB elements above 1% of peak. An initial stricter + 1% single-element gate failed on these isolated temporal differences; + the observed metrics and current bounds are recorded explicitly. +* `test-results/native-identity-final`: wrong hash rejects the override and + runs translated PTX correctly; matching hash executes 32 native launches. +* CTest 9/9 and fresh-source application of all nine ZLUDA patches PASS. + +Reproduce the actual K layer gate without editing sources: + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts/windows/test-k-captures.ps1 ` + -CaptureDirectory C:\Users\Administrator\d4r\test-results\ngx-k-fp16-fenced-capture\replay ` + -DlssDll C:\Users\Administrator\d4r\nvngx_dlss.dll +``` + +This is CUDA NGX validation with synthetic image inputs and actual local DLSS +weights. It does not claim a completed Windows D3D12 shim, OptiScaler game test, +dynamic scene validation, or M support. These are the next gates. + ### All K layers: native Windows synthetic replay (2026-09-30) `native_replay_probe` reads upstream `manifest.txt`/`args.bin`/`alloc-N.bin`, diff --git a/kernels/common/hip_device_minimal.h b/kernels/common/hip_device_minimal.h index 781d1b8..04b291e 100644 --- a/kernels/common/hip_device_minimal.h +++ b/kernels/common/hip_device_minimal.h @@ -35,5 +35,9 @@ __device__ __forceinline__ d4r_device_dim3 d4r_grid_dim() { #define blockIdx d4r_block_idx() #define gridDim d4r_grid_dim() __device__ __attribute__((convergent, always_inline)) inline void __syncthreads() { + // Match HIP __work_group_barrier: execution rendezvous alone does not + // publish LDS stores or prevent loads moving across the barrier. + __builtin_amdgcn_fence(__ATOMIC_RELEASE, "workgroup"); __builtin_amdgcn_s_barrier(); + __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup"); } diff --git a/kernels/k/pwin_common.h b/kernels/k/pwin_common.h index a40ebba..20b65bc 100644 --- a/kernels/k/pwin_common.h +++ b/kernels/k/pwin_common.h @@ -30,12 +30,55 @@ __device__ __forceinline__ uint32_t lane_id() typedef wm_op op_t; // WMMA operand: u8v (gfx11) or u4v (gfx12) +// The CUDA fragment reduction rounds products and every tree node to f16. +// A float sum of the squared halves changes rsqrt and can amplify differences +// through attention. The strict baseline preserves its channel grouping. +template +__device__ __forceinline__ half_t l2_sum(const half_t* row) +{ +#ifdef D4R_K_FP16_BASELINE + static_assert(C % 32 == 0 && C <= 160, "audited K channel counts"); + hv2 partial[4]; +#pragma unroll + for (int t = 0; t < 4; ++t) + { + hv2 halves[2]; +#pragma unroll + for (int parity = 0; parity < 2; ++parity) + { + hv2 pairs[C / 32]; +#pragma unroll + for (int group = 0; group < C / 32; ++group) + { + const hv2 a = *(const hv2*)(row + 2 * t + 8 * parity + 32 * group); + const hv2 b = *(const hv2*)(row + 2 * t + 8 * parity + 32 * group + 16); + pairs[group] = (a * a) + (b * b); + } + hv2 value = pairs[0]; + if constexpr (C >= 64) value = value + pairs[1]; + if constexpr (C == 96) value = value + pairs[2]; + if constexpr (C >= 128) value = value + (pairs[2] + pairs[3]); + if constexpr (C == 160) value = value + pairs[4]; + halves[parity] = value; + } + partial[t] = halves[0] + halves[1]; + } + const hv2 sum = (partial[0] + partial[1]) + (partial[2] + partial[3]); + return sum[0] + sum[1]; +#else + float sum = 0; +#pragma unroll + for (int c = 0; c < C; ++c) sum += (float)(half_t)(row[c] * row[c]); + return (half_t)sum; +#endif +} + // one k16 step with the f16 accumulator of NVIDIA's f16 wmma (rounded after the step) // PWIN_F32ACC: keep the accumulator in f32 through the chain (rounded to f16 where the values are used) __device__ __forceinline__ f8v mma16(const op_t& a, const op_t& b, f8v c) { f8v d = wm_mma(a, b, c); -#ifndef PWIN_F32ACC +#if !defined(PWIN_F32ACC) || defined(D4R_K_FP16_BASELINE) #pragma unroll for (int i = 0; i < 8; ++i) d[i] = (float)(half_t)d[i]; diff --git a/kernels/k/pwin_layer.h b/kernels/k/pwin_layer.h index f0ad301..2d6e559 100644 --- a/kernels/k/pwin_layer.h +++ b/kernels/k/pwin_layer.h @@ -259,17 +259,7 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons STAMP(1); // ---- L2 norm: h1 = x0 * (rsqrt(sum x0^2) * gamma1) -> hb - float ss = 0.0f; - for (int kt = 0; kt < KT; ++kt) - { - const u8v xv = lds_row16(arow + 16 * kt); -#pragma unroll - for (int c = 0; c < 16; ++c) - { - const half_t v = op_get(xv, c); - ss += (float)(half_t)(v * v); - } - } + const half_t ss = l2_sum(arow); const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss); for (int kt = (int)hf; kt < KT; kt += 2) { diff --git a/kernels/k/pwin_pos.h b/kernels/k/pwin_pos.h index 9159f21..2b38161 100644 --- a/kernels/k/pwin_pos.h +++ b/kernels/k/pwin_pos.h @@ -19,18 +19,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const { const half_t* arow = act[16 * (wv * MT + mi) + m]; half_t* hrow = hb[16 * (wv * MT + mi) + m]; - float ss = 0.0f; -#pragma unroll - for (int kt = 0; kt < KT; ++kt) - { - const u8v xv = lds_row16(arow + 16 * kt); -#pragma unroll - for (int c = 0; c < 16; ++c) - { - const half_t v = op_get(xv, c); - ss += (float)(half_t)(v * v); - } - } + const half_t ss = l2_sum(arow); const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss); for (int kt = (int)hf; kt < KT; kt += 2) { diff --git a/kernels/k/pwin_wide.h b/kernels/k/pwin_wide.h index 298fd48..8dc75cf 100644 --- a/kernels/k/pwin_wide.h +++ b/kernels/k/pwin_wide.h @@ -38,17 +38,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons half_t* hrow = hb[tok]; // ---- L2 norm: h1 = x0 * (rsqrt(sum x0^2) * gamma1) -> hb (all groups of a token tile compute the sum) - float ss = 0.0f; - for (int kt = 0; kt < KT; ++kt) - { - const u8v xv = lds_row16(arow + 16 * kt); -#pragma unroll - for (int c = 0; c < 16; ++c) - { - const half_t v = op_get(xv, c); - ss += (float)(half_t)(v * v); - } - } + const half_t ss = l2_sum(arow); const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss); for (int kt = 2 * g + (int)hf; kt < KT; kt += 2 * NG) { diff --git a/kernels/tools/pwin_model.py b/kernels/tools/pwin_model.py index af4bc02..4f56f21 100644 --- a/kernels/tools/pwin_model.py +++ b/kernels/tools/pwin_model.py @@ -6,6 +6,38 @@ import numpy as np F16, F32, F64 = np.float16, np.float32, np.float64 +# Legacy replay reports retain their original approximate norm by default. +# Windows strict K validation selects the PTX fragment's rounded reduction. +NORM_FRAGMENT_ORDER = False + + +def norm_rows(x): + if not NORM_FRAGMENT_ORDER: + return f16(np.sum(x.astype(F32).astype(F64) ** 2, axis=1)) + tokens, channels = x.shape + assert channels in (32, 64, 96, 128, 160) + square = f16(x.astype(F64) ** 2) + partial = [] + for lane in range(4): + halves = [] + for parity in range(2): + pairs = [add16(square[:, 2 * lane + 8 * parity + 32 * g:2 * lane + 8 * parity + 32 * g + 2], + square[:, 2 * lane + 8 * parity + 32 * g + 16:2 * lane + 8 * parity + 32 * g + 18]) + for g in range(channels // 32)] + value = pairs[0] + if channels >= 64: + value = add16(value, pairs[1]) + if channels == 96: + value = add16(value, pairs[2]) + if channels >= 128: + value = add16(value, add16(pairs[2], pairs[3])) + if channels == 160: + value = add16(value, pairs[4]) + halves.append(value) + partial.append(add16(halves[0], halves[1])) + total = add16(add16(partial[0], partial[1]), add16(partial[2], partial[3])) + return add16(total[:, 0], total[:, 1]) + class Dump: def __init__(self, d): @@ -181,7 +213,7 @@ def pwin_enc2(P, bx, by, pm_order=((0, 0), (0, 1), (1, 0), (1, 1)), b2_in_c0=Tru """logical enc2 layer (H 2, C 64 -> merged 96) for one window; returns (y [64 x 64], merged [16 x 96])""" C, H, Dh = 64, 2, 32 x0 = load_window(P, bx, by, C) - s = f16(np.sum(x0.astype(F32).astype(F64) ** 2, axis=1)) + s = norm_rows(x0) r = rsqrt16(s) h1 = mul16(x0, mul16(r[:, None], vec(P, 0, C)[None, :])) Os = [] @@ -248,7 +280,7 @@ def swin_core(P, L, x0, dbg=None, posattn=False): """norm -> attention -> Wo + residual -> MLP (+ residual): the block output y [64 x C]""" C, H = L.C, L.H NCH = C // 32 - s = f16(np.sum(x0.astype(F32).astype(F64) ** 2, axis=1)) + s = norm_rows(x0) r = rsqrt16(s) h1 = mul16(x0, mul16(r[:, None], vec(P, L.G1, C)[None, :])) Os = [] diff --git a/patches/zluda/0010-native-identity-replay-windows-cache.patch b/patches/zluda/0010-native-identity-replay-windows-cache.patch new file mode 100644 index 0000000..44586e0 --- /dev/null +++ b/patches/zluda/0010-native-identity-replay-windows-cache.patch @@ -0,0 +1,440 @@ +Subject: [PATCH] Guard and own native overrides; capture K replay on Windows + +Native replacements require the upstream PTX identity manifest. Preserve native +module/image ownership, bypass redundant JIT for completely replaced modules, +include the loaded Windows DLL fingerprint in cache identity, and expose opt-in +private real-weight replay captures with explicit native/translated launch logs. + +diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs +new file mode 100644 +index 0000000..875b2e3 +--- /dev/null ++++ b/zluda/src/impl/d4r_replay.rs +@@ -0,0 +1,109 @@ ++//! Opt-in, synchronous diagnostic captures. Never enabled in the game fast path. ++//! Captures can contain proprietary network weights; keep them out of releases. ++use hip_runtime_sys::*; ++use std::{collections::HashMap, ffi::{c_void, CStr}, fs, path::PathBuf, sync::{Mutex, OnceLock}}; ++ ++struct Config { directory: PathBuf, filter: String, limit: u32 } ++static CONFIG: OnceLock> = OnceLock::new(); ++static COUNTS: OnceLock>> = OnceLock::new(); ++ ++pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> } ++ ++fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t { ++ eprintln!("[d4r-replay] capture failed: {error}"); ++ hipErrorCode_t::r#Unknown ++} ++ ++impl Capture { ++ fn save(&self, destination: &std::path::Path) -> hipError_t { ++ fs::create_dir_all(destination).map_err(io_error)?; ++ for (id, &(base, size)) in self.allocations.iter().enumerate() { ++ let mut bytes = vec![0u8; size]; ++ unsafe { hipMemcpyDtoH(bytes.as_mut_ptr().cast(), hipDeviceptr_t(base as *mut c_void), size) }?; ++ fs::write(destination.join(format!("alloc-{id}.bin")), bytes).map_err(io_error)?; ++ } ++ Ok(()) ++ } ++ ++ pub(crate) fn finish(self, stream: hipStream_t) -> hipError_t { ++ unsafe { hipStreamSynchronize(stream) }?; ++ self.save(&self.directory.join("gpu-output"))?; ++ eprintln!("[d4r-replay] complete path={}", self.directory.display()); ++ Ok(()) ++ } ++} ++ ++pub(crate) unsafe fn begin( ++ name: &CStr, stream: hipStream_t, parameters: *mut *mut c_void, ++ extra: *mut *mut c_void, launch: [u32; 7], ++) -> Result, hipErrorCode_t> { ++ let Some(config) = CONFIG.get_or_init(|| std::env::var_os("D4R_CUDA_REPLAY_DUMP_DIR").map(|directory| Config { ++ directory: directory.into(), ++ filter: std::env::var("D4R_CUDA_REPLAY_DUMP_FILTER").unwrap_or_default(), ++ limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1), ++ })) else { return Ok(None) }; ++ let name = name.to_str().map_err(io_error)?; ++ // Only the audited, single 176-byte K argument ABI is captured here. ++ if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) { ++ return Ok(None); ++ } ++ let mut block = std::ptr::null::(); ++ let mut bytes = None; ++ if !extra.is_null() { ++ for i in (0..6).step_by(2) { ++ match *extra.add(i) as usize { ++ 1 => block = (*extra.add(i + 1)).cast(), ++ 2 => bytes = Some(*(*extra.add(i + 1) as *const usize)), ++ 0 => break, ++ _ => return Err(hipErrorCode_t::r#InvalidValue), ++ } ++ } ++ } else if !parameters.is_null() { ++ block = (*parameters).cast(); ++ bytes = Some(176); ++ } ++ if block.is_null() || bytes != Some(176) { ++ return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}"))); ++ } ++ let sequence = { ++ let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?; ++ let count = counts.entry(name.to_owned()).or_default(); ++ if *count >= config.limit { return Ok(None); } ++ *count += 1; ++ *count ++ }; ++ hipStreamSynchronize(stream)?; ++ let arguments = std::slice::from_raw_parts(block, 176); ++ let directory = config.directory.join(format!("replay-{sequence:06}-{name}")); ++ fs::create_dir_all(&directory).map_err(io_error)?; ++ fs::write(directory.join("args.bin"), arguments).map_err(io_error)?; ++ let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n", ++ launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]); ++ let mut allocations = Vec::<(usize, usize)>::new(); ++ for offset in (0..176).step_by(8) { ++ let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize; ++ if pointer == 0 { continue; } ++ let mut base = hipDeviceptr_t(std::ptr::null_mut()); ++ let mut size = 0; ++ if hipMemGetAddressRange(&mut base, &mut size, hipDeviceptr_t(pointer as *mut c_void)).is_err() { continue; } ++ let base = base.0 as usize; ++ if pointer < base || pointer - base >= size { continue; } ++ if size > 512 * 1024 * 1024 { return Err(io_error("allocation exceeds diagnostic 512 MiB limit")); } ++ let id = match allocations.iter().position(|&(b, _)| b == base) { ++ Some(id) => id, ++ None => { ++ let id = allocations.len(); ++ allocations.push((base, size)); ++ manifest.push_str(&format!("alloc {id} 0x{base:x} {size}\n")); ++ id ++ } ++ }; ++ manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base)); ++ } ++ if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); } ++ fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?; ++ let capture = Capture { directory, allocations }; ++ capture.save(&capture.directory)?; ++ eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display()); ++ Ok(Some(capture)) ++} +diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs +index 033d9a0..7a7470a 100644 +--- a/zluda/src/impl/function.rs ++++ b/zluda/src/impl/function.rs +@@ -7,6 +7,8 @@ use hip_runtime_sys::*; + use std::mem; + + pub(crate) struct Function { ++ pub(crate) name: std::ffi::CString, ++ pub(crate) native_owner: Option, + pub(crate) base: hipFunction_t, + pub(crate) sm_version: u32, + pub(crate) explicit_args_size_align: Option>, +@@ -144,6 +146,8 @@ pub(crate) fn launch_kernel( + } + translated_extra_storage.as_mut_ptr() + }; ++ let replay = unsafe { super::d4r_replay::begin(&f.name, stream, kernel_params, extra, ++ [grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y, block_dim_z, shared_mem_bytes]) }?; + // the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer + let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) { + let mut block: *const u8 = std::ptr::null(); +@@ -206,7 +210,11 @@ pub(crate) fn launch_kernel( + } + let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z }; + let (grid_dim_x, grid_dim_y, grid_dim_z) = if f.grid_x != 0 { (f.grid_x, 1, 1) } else { (grid_dim_x, grid_dim_y, grid_dim_z) }; +- unsafe { ++ if std::env::var_os("D4R_ZLUDA_VERBOSE").is_some() { ++ eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}", ++ f.name, if f.native_owner.is_some() { "native" } else { "translated" }); ++ } ++ let result = unsafe { + hipModuleLaunchKernel( + f.base, + grid_dim_x, +@@ -220,7 +228,10 @@ pub(crate) fn launch_kernel( + kernel_params, + extra_for_hip, + ) +- } ++ }; ++ result?; ++ if let Some(replay) = replay { replay.finish(stream)?; } ++ Ok(()) + } + + pub(crate) unsafe fn set_attribute( +diff --git a/zluda/src/impl/library.rs b/zluda/src/impl/library.rs +index 98be33e..1319792 100644 +--- a/zluda/src/impl/library.rs ++++ b/zluda/src/impl/library.rs +@@ -21,8 +21,8 @@ impl Library { + let module_lock = self.modules.get(device).ok_or(CUerror::INVALID_DEVICE)?; + *module_lock.get_or_init(|| match self.data { + LibraryData::Lazy(lib) => { +- let (module, sm_version, meta32) = module::load_hip_module(lib)?; +- Ok(Module::new(module, sm_version, meta32).wrap()) ++ let (module, sm_version, meta32, ptx_hash, native_image) = module::load_hip_module(lib)?; ++ Ok(Module::new(module, sm_version, meta32, ptx_hash, native_image).wrap()) + } + LibraryData::Eager(()) => Err(CUerror::NOT_SUPPORTED), + }) +diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs +index 5329d32..14e0aee 100644 +--- a/zluda/src/impl/mod.rs ++++ b/zluda/src/impl/mod.rs +@@ -5,6 +5,7 @@ pub(super) mod array; + pub(super) mod context; + pub(super) mod device; + pub(super) mod driver; ++pub(super) mod d4r_replay; + pub(super) mod event; + pub(super) mod function; + pub(super) mod graph; +diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs +index 9f72f2f..5a79368 100644 +--- a/zluda/src/impl/module.rs ++++ b/zluda/src/impl/module.rs +@@ -19,6 +19,8 @@ pub(crate) struct Module { + pub(crate) base: hipModule_t, + pub(crate) sm_version: u32, + pub(crate) bit32: Option, ++ ptx_hash: Option, ++ _native_base_image: Option>, + mutable: Mutex, + } + +@@ -33,12 +35,14 @@ impl ModuleMutable { + sm_version: u32, + name: CString, + explicit_args_size_align: Option>, ++ ptx_hash: Option, + ) -> Result<&Function, CUerror> { + Ok(match self.functions.entry(name) { + hash_map::Entry::Occupied(entry) => &*entry.into_mut(), + hash_map::Entry::Vacant(entry) => { +- let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) { +- Some(native) => native, ++ let mut native_owner = None; ++ let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key(), ptx_hash) { ++ Some((native, owner)) => { native_owner = Some(owner); native }, + None => { + let mut func_handle = unsafe { std::mem::zeroed() }; + unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?; +@@ -46,6 +50,8 @@ impl ModuleMutable { + } + }; + let func = Box::new(Function { ++ name: entry.key().clone(), ++ native_owner, + base: func_handle, + sm_version, + explicit_args_size_align, +@@ -69,19 +75,48 @@ impl ModuleMutable { + /// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads. + /// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x` + /// (persistent kernels, which derive the original grid from their parameters) the grid. ++pub(crate) struct NativeModule { ++ module: hipModule_t, ++ _image: Vec, ++} ++ ++impl Drop for NativeModule { ++ fn drop(&mut self) { ++ if self.module.0.is_null() { return; } ++ if let Err(error) = unsafe { hipModuleUnload(self.module) } { ++ eprintln!("[d4r-native] unload failed: {error:?}"); ++ } ++ } ++} ++ + fn native_override( + name: &CStr, +-) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option, u32)> { ++ ptx_hash: Option, ++) -> Option<((hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option, u32), NativeModule)> { + let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?; ++ let manifest = std::path::Path::new(&dir).join("d4r-kernels.txt"); ++ let allowed = fs::read_to_string(&manifest).ok().is_some_and(|text| text.lines().any(|line| { ++ let mut fields = line.split_whitespace(); ++ fields.next() == name.to_str().ok() ++ && fields.next().and_then(|s| u64::from_str_radix(s, 16).ok()) == ptx_hash ++ && ptx_hash.is_some() ++ })); ++ if !allowed { ++ eprintln!("[d4r-native] miss kernel={name:?} reason=ptx_identity hash={ptx_hash:016x?} fallback=translated"); ++ return None; ++ } + let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?)); +- let image = fs::read(&path).ok()?; +- // HIP may reference the image after loading; the override lives for the whole process. +- let image: &'static [u8] = Box::leak(image.into_boxed_slice()); ++ let image = match fs::read(&path) { ++ Ok(image) => image, ++ Err(error) => { eprintln!("[d4r-native] miss kernel={name:?} reason=object_missing error={error} fallback=translated"); return None; } ++ }; + let mut native_module = unsafe { mem::zeroed() }; + if unsafe { hipModuleLoadData(&mut native_module, image.as_ptr().cast()) }.is_err() { + eprintln!("[d4r-native] failed to load {}", path.display()); + return None; + } ++ // Keep the image and module owned by the CUfunction; release at module unload. ++ let owner = NativeModule { module: native_module, _image: image }; + let mut func = unsafe { mem::zeroed() }; + if unsafe { hipModuleGetFunction(&mut func, native_module, name.as_ptr()) }.is_err() { + eprintln!("[d4r-native] {} has no kernel {:?}", path.display(), name); +@@ -138,7 +173,7 @@ fn native_override( + prep_key, + prep_slots + ); +- Some((func, prep, block_z, grid_x, prep_key, prep_slots)) ++ Some(((func, prep, block_z, grid_x, prep_key, prep_slots), owner)) + } + + impl ZludaObject for Module { +@@ -229,7 +264,7 @@ pub(crate) struct Global32Bit { + } + + impl Module { +- pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option) -> Self { ++ pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option, ptx_hash: Option, native_base_image: Option>) -> Self { + Self { + base, + sm_version, +@@ -237,6 +272,8 @@ impl Module { + functions: FxHashMap::default(), + }), + bit32, ++ ptx_hash, ++ _native_base_image: native_base_image, + } + } + +@@ -248,7 +285,7 @@ impl Module { + .map(Vec::clone) + }); + mutable +- .get_function(self.base, self.sm_version, name.to_owned(), explicit_args) ++ .get_function(self.base, self.sm_version, name.to_owned(), explicit_args, self.ptx_hash) + .map(|f| unsafe { (f as *const Function).as_ref().unwrap() }) + } + } +@@ -256,7 +293,7 @@ impl Module { + fn get_best_ptx_and_compile( + global_state: &GlobalState, + image: CodeLibraryRef<'_>, +-) -> Result<(hipModule_t, u32, Option), CUerror> { ++) -> Result<(hipModule_t, u32, Option, Option, Option>), CUerror> { + let mut ptx_modules = Vec::new(); + unsafe { + CodeLibraryRef::iterate_modules(image, |_, module| match module { +@@ -280,7 +317,7 @@ fn get_best_ptx_and_compile( + let dump_dir = std::path::PathBuf::from(dump_dir); + let _ = fs::create_dir_all(&dump_dir); + for (index, source) in ptx_modules.iter().enumerate() { +- let stem = format!("module-{index:04}"); ++ let stem = format!("module-{:016x}-{index:04}", ptx_identity(source)); + let _ = fs::write(dump_dir.join(format!("{stem}.ptx")), source.as_bytes()); + let diagnostic = match ptx_parser::parse_module_checked(source.as_ref()) { + Ok(ast) => format!( +@@ -345,6 +382,32 @@ fn get_best_ptx_and_compile( + ) + } + }; ++ let ptx_hash = text.map(|source| ptx_identity(source)); ++ // A fully replaced, single-entry module needs no translated binary. LDS ++ // declarations belong to the replaced kernel; exported globals do not. ++ let only_kernel_and_lds = module.directives.iter().all(|directive| match directive { ++ ptx_parser::Directive::Method(_, method) => method.func_directive.name.is_kernel(), ++ ptx_parser::Directive::Variable(_, variable) => matches!(variable.info.state_space, ptx_parser::StateSpace::Shared), ++ }); ++ let entries: Vec<_> = module.directives.iter().filter_map(|directive| match directive { ++ ptx_parser::Directive::Method(_, method) => Some(method), _ => None, ++ }).collect(); ++ if module.address_size == 64 && only_kernel_and_lds && entries.len() == 1 { ++ let method = entries[0]; ++ { ++ if method.func_directive.name.is_kernel() { ++ if let Ok(name) = CString::new(method.func_directive.name.text()) { ++ if let Some((_, mut owner)) = native_override(&name, ptx_hash) { ++ let base = owner.module; ++ owner.module = hipModule_t(std::ptr::null_mut()); ++ let image = mem::take(&mut owner._image); ++ eprintln!("[d4r-native] module_direct kernel={name:?} translated_jit=skipped"); ++ return Ok((base, module.sm_version, None, ptx_hash, Some(image))); ++ } ++ } ++ } ++ } ++ } + // TODO: get this information on initialization + let hip_properties = get_hip_properties()?; + let gcn_arch = get_gcn_arch(&hip_properties)?; +@@ -372,7 +435,12 @@ fn get_best_ptx_and_compile( + .or_else(|_| compile_and_cache(gcn_arch, attributes, module, &mut cache_with_key))?; + let mut hip_module = unsafe { mem::zeroed() }; + unsafe { hipModuleLoadData(&mut hip_module, elf_module.as_ptr().cast()) }?; +- Ok((hip_module, sm_version, zluda32)) ++ Ok((hip_module, sm_version, zluda32, ptx_hash, None)) ++} ++ ++fn ptx_identity(text: &str) -> u64 { ++ text.trim_end_matches('\0').bytes().fold(0xcbf29ce484222325u64, |hash, byte| ++ (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3)) + } + + fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option> { +@@ -384,7 +452,7 @@ fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option> { + + pub(crate) fn load_hip_module( + library: CodeLibraryRef, +-) -> Result<(hipModule_t, u32, Option), CUerror> { ++) -> Result<(hipModule_t, u32, Option, Option, Option>), CUerror> { + let global_state = driver::global_state()?; + get_best_ptx_and_compile(global_state, library) + } +@@ -434,7 +502,7 @@ fn zluda_build_version() -> &'static str { + key.starts_with("D4R_ZLUDA_") + && !matches!( + key.as_str(), +- "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" ++ "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" + | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes) + ) + }) +@@ -469,7 +537,10 @@ fn loaded_library_fingerprint() -> Option { + + #[cfg(windows)] + fn loaded_library_fingerprint() -> Option { +- None ++ let path = zluda_windows::get_module_path_for_function(zluda_build_version as *const () as usize)?; ++ let metadata = fs::metadata(path).ok()?; ++ let timestamp = metadata.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?; ++ Some(format!("{:x}.{:x}.{:x}", metadata.len(), timestamp.as_secs(), timestamp.subsec_nanos())) + } + + fn get_cache_key<'a, 'b>( +@@ -577,16 +648,16 @@ pub(crate) fn load(module: &mut CUmodule, fname: &CStr) -> CUresult { + image.push(0); + let library = unsafe { CodeLibraryRef::try_load(image.as_ptr() as *const std::ffi::c_void) } + .map_err(|_| CUerror::NO_BINARY_FOR_GPU)?; +- let (hip_module, sm_version, meta32) = load_hip_module(library)?; +- *module = Module::new(hip_module, sm_version, meta32).wrap(); ++ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?; ++ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap(); + Ok(()) + } + + pub(crate) fn load_data(module: &mut CUmodule, image: &std::ffi::c_void) -> CUresult { + let library = + unsafe { CodeLibraryRef::try_load(image) }.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?; +- let (hip_module, sm_version, meta32) = load_hip_module(library)?; +- *module = Module::new(hip_module, sm_version, meta32).wrap(); ++ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?; ++ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap(); + Ok(()) + } + diff --git a/scripts/windows/build-zluda-windows.ps1 b/scripts/windows/build-zluda-windows.ps1 index 15f2725..1aa98f9 100644 --- a/scripts/windows/build-zluda-windows.ps1 +++ b/scripts/windows/build-zluda-windows.ps1 @@ -66,6 +66,7 @@ try { Copy-Item -LiteralPath (Join-Path $release 'zluda_trace.dll') -Destination (Join-Path $InstallRoot 'trace/nvcuda.dll') -Force Copy-Item -LiteralPath (Join-Path $release 'examples/d4r_emit.exe') -Destination $InstallRoot -Force @{base='ee2f25a180099fa42f36b2346732e1f2470a03ad'; target='x86_64-pc-windows-gnu'; + sourceCommit=(& git rev-parse HEAD); workingTreeDirty=[bool](& git status --porcelain); rust='1.98.1'; llvm=(& (Join-Path $LlvmBuildRoot 'bin/llvm-config.exe') --version); files=@(Get-ChildItem -LiteralPath $InstallRoot -Filter '*.dll' | Get-FileHash -Algorithm SHA256)} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $InstallRoot 'build-info.json') -Encoding UTF8 diff --git a/scripts/windows/stage-native-k.ps1 b/scripts/windows/stage-native-k.ps1 new file mode 100644 index 0000000..3f7dc7b --- /dev/null +++ b/scripts/windows/stage-native-k.ps1 @@ -0,0 +1,24 @@ +[CmdletBinding()] +param([Parameter(Mandatory=$true)][string]$DlssDll, [string]$PackageRoot, [string]$OutputDirectory) +$ErrorActionPreference = 'Stop' +$repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +if (!$PackageRoot) { $PackageRoot = Join-Path $repo 'dist/windows-rdna4-therock' } +if (!$OutputDirectory) { $OutputDirectory = Join-Path $repo 'build/native-k-gfx1201' } +if (![IO.Path]::IsPathRooted($DlssDll) -or !(Test-Path -LiteralPath $DlssDll)) { + throw 'Supply the absolute path to your local nvngx_dlss.dll.' +} +New-Item -ItemType Directory -Force $OutputDirectory | Out-Null +$layers = @('enc0','enc1','enc2','enc3','enc4','dec5','dec4','dec3','dec2','dec1','dec0') +foreach ($layer in $layers) { + $name = "dltss_pwin_${layer}_layer" + $source = Join-Path $PackageRoot "experimental/k/${name}_gfx1201.hsaco" + if (!(Test-Path -LiteralPath $source)) { throw "Native K module missing: $source" } + Copy-Item -LiteralPath $source -Destination (Join-Path $OutputDirectory "$name.hsaco") -Force +} +& python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll +if ($LASTEXITCODE) { throw "Native manifest generation failed ($LASTEXITCODE)" } +@{architecture='gfx1201'; family='K'; dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash; + validation='synthetic layers passed; actual DLSS capture/reference validation required'; + modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} | + ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'build-info.json') -Encoding UTF8 +Write-Host "Experimental native K directory: $OutputDirectory" diff --git a/scripts/windows/test-k-captures.ps1 b/scripts/windows/test-k-captures.ps1 new file mode 100644 index 0000000..aa70b43 --- /dev/null +++ b/scripts/windows/test-k-captures.ps1 @@ -0,0 +1,52 @@ +[CmdletBinding()] +param([Parameter(Mandatory=$true)][string]$CaptureDirectory, [string]$HipRoot, + [string]$PackageRoot, [string]$OutputDirectory, [int]$Windows = 12, + [Parameter(Mandatory=$true)][string]$DlssDll, [int]$PtxWindows = 2) +$ErrorActionPreference = 'Stop' +$repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +if (!$HipRoot) { $HipRoot = Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' } +if (!$PackageRoot) { $PackageRoot = Join-Path $repo 'dist/windows-rdna4-therock' } +if (!$OutputDirectory) { $OutputDirectory = Join-Path $repo ('test-results/k-real-reference-' + (Get-Date -Format 'yyyyMMdd-HHmmss')) } +New-Item -ItemType Directory -Force $OutputDirectory | Out-Null +$previousPython = $env:PYTHONPATH +$previousDiag = $env:D4R_DIAG_DIR +$env:PYTHONPATH = "$(Join-Path $repo '.tools/python/vendor');$previousPython" +$summary = [ordered]@{utc=[DateTime]::UtcNow.ToString('o'); captureDirectory=$CaptureDirectory; layers=@(); passed=$false} +try { + foreach ($layer in @('enc0','enc1','enc2','enc3','enc4','dec5','dec4','dec3','dec2','dec1','dec0')) { + $name = "dltss_pwin_${layer}_layer" + $captures = @(Get-ChildItem -LiteralPath $CaptureDirectory -Directory | Where-Object Name -Match ("^replay-[0-9]+-" + $name + '$')) + if (!$captures.Count) { throw "Missing actual K capture: $name" } + $capture = $captures | Sort-Object Name | Select-Object -First 1 + $folder = Join-Path $OutputDirectory $layer + $output = Join-Path $folder 'output' + New-Item -ItemType Directory -Force $folder | Out-Null + $env:D4R_DIAG_DIR = $folder + & (Join-Path $PackageRoot 'bin/d4r_native_replay_probe.exe') --hip-root $HipRoot ` + --module (Join-Path $PackageRoot "experimental/k/${name}_gfx1201.hsaco") ` + --fixture-dir $capture.FullName --output-dir $output --iterations 1 ` + > (Join-Path $folder 'gpu.stdout.log') 2> (Join-Path $folder 'gpu.stderr.log') + if ($LASTEXITCODE) { throw "K $layer actual GPU replay failed: $folder" } + & python (Join-Path $PackageRoot 'k_replay_validate.py') validate --layer $layer ` + --fixture-dir $capture.FullName --output-dir $output --windows $Windows --real-capture ` + > (Join-Path $folder 'reference.stdout.log') 2> (Join-Path $folder 'reference.stderr.log') + if ($LASTEXITCODE) { throw "K $layer mismatch; stopping before the next layer: $folder" } + Get-Content -LiteralPath (Join-Path $folder 'reference.stdout.log') | Write-Host + $ptxValidator = Join-Path $PackageRoot 'k_ptx_validate.py' + if (!(Test-Path -LiteralPath $ptxValidator)) { $ptxValidator = Join-Path $repo 'tools/windows/k_ptx_validate.py' } + & python -u $ptxValidator --layer $layer ` + --capture-dir $capture.FullName --native-dir $output --dlss-dll $DlssDll ` + --output-dir (Join-Path $folder 'private-ptx') --windows $PtxWindows ` + > (Join-Path $folder 'ptx-reference.stdout.log') 2> (Join-Path $folder 'ptx-reference.stderr.log') + if ($LASTEXITCODE) { throw "K $layer independent PTX mismatch; stopping: $folder" } + Get-Content -LiteralPath (Join-Path $folder 'ptx-reference.stdout.log') | Write-Host + $summary.layers += @{layer=$layer; passed=$true} + } + $summary.passed = $true +} catch { $summary.error=$_.Exception.Message; Write-Warning $summary.error } +finally { + $env:PYTHONPATH = $previousPython; $env:D4R_DIAG_DIR = $previousDiag + $summary | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'summary.json') -Encoding UTF8 + Write-Host "Private real-weight replay results: $OutputDirectory (do not redistribute captures/allocations)" +} +if (!$summary.passed) { exit 1 } diff --git a/scripts/windows/test-native-overrides.ps1 b/scripts/windows/test-native-overrides.ps1 new file mode 100644 index 0000000..8279696 --- /dev/null +++ b/scripts/windows/test-native-overrides.ps1 @@ -0,0 +1,50 @@ +[CmdletBinding()] +param([string]$HipRoot, [string]$PackageRoot, [string]$ZludaRoot, [string]$OutputDirectory) +$ErrorActionPreference = 'Stop' +$repo = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..')) +if (!$HipRoot) { $HipRoot = Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' } +if (!$PackageRoot) { $PackageRoot = Join-Path $repo 'dist/windows-rdna4-therock' } +if (!$ZludaRoot) { $ZludaRoot = Join-Path $repo 'dist/zluda-windows-native' } +if (!$OutputDirectory) { $OutputDirectory = Join-Path $repo ('test-results/native-identity-' + (Get-Date -Format 'yyyyMMdd-HHmmss')) } +New-Item -ItemType Directory -Force $OutputDirectory | Out-Null +$previous = @{D4R_ZLUDA_NATIVE_DIR=$env:D4R_ZLUDA_NATIVE_DIR; D4R_ZLUDA_VERBOSE=$env:D4R_ZLUDA_VERBOSE; D4R_DIAG_DIR=$env:D4R_DIAG_DIR} +function Invoke-NativeProbe([string]$Name) { + $info = New-Object Diagnostics.ProcessStartInfo + $info.FileName = Join-Path $PackageRoot 'bin/d4r_cuda_driver_probe.exe' + $info.Arguments = '--hip-root "' + $HipRoot + '" --cuda-dll "' + (Join-Path $ZludaRoot 'nvcuda.dll') + '" --iterations 32' + $info.UseShellExecute=$false; $info.CreateNoWindow=$true; $info.RedirectStandardOutput=$true; $info.RedirectStandardError=$true + $process = New-Object Diagnostics.Process + $process.StartInfo=$info + if (!$process.Start()) { throw 'Cannot start native override probe' } + $stdout=$process.StandardOutput.ReadToEndAsync(); $stderr=$process.StandardError.ReadToEndAsync() + if (!$process.WaitForExit(120000)) { $process.Kill(); $process.WaitForExit(); throw 'Native override test timed out' } + [IO.File]::WriteAllText((Join-Path $OutputDirectory "$Name.stdout.log"),$stdout.GetAwaiter().GetResult()) + [IO.File]::WriteAllText((Join-Path $OutputDirectory "$Name.stderr.log"),$stderr.GetAwaiter().GetResult()) + $code=$process.ExitCode; $process.Dispose() + return $code +} +try { + $env:D4R_ZLUDA_VERBOSE = '1'; $env:D4R_DIAG_DIR = $OutputDirectory + $bad = Join-Path $OutputDirectory 'wrong-identity' + $good = Join-Path $OutputDirectory 'matching-identity' + New-Item -ItemType Directory -Force $bad,$good | Out-Null + foreach ($directory in @($bad,$good)) { + Copy-Item -LiteralPath (Join-Path $PackageRoot 'bin/probe_gfx1201.hsaco') -Destination (Join-Path $directory 'd4r_ptx_pattern.hsaco') -Force + } + 'd4r_ptx_pattern 0000000000000000' | Set-Content -LiteralPath (Join-Path $bad 'd4r-kernels.txt') -Encoding ASCII + $env:D4R_ZLUDA_NATIVE_DIR = $bad + if (Invoke-NativeProbe 'wrong') { throw 'Wrong identity must use the correct translated fallback' } + if (!(Select-String -LiteralPath (Join-Path $OutputDirectory 'wrong.stderr.log') -Pattern 'reason=ptx_identity.*fallback=translated' -Quiet)) { + throw 'Wrong identity was not explicitly rejected' + } + $identity = Select-String -LiteralPath (Join-Path $OutputDirectory 'wrong.stdout.log') -Pattern 'fnv1a64=([0-9a-f]{16})' + if (!$identity) { throw 'PTX probe did not report its identity' } + "d4r_ptx_pattern $($identity.Matches[0].Groups[1].Value)" | Set-Content -LiteralPath (Join-Path $good 'd4r-kernels.txt') -Encoding ASCII + $env:D4R_ZLUDA_NATIVE_DIR = $good + if (Invoke-NativeProbe 'matching') { throw 'Matching native override produced incorrect output' } + $hits = @(Select-String -LiteralPath (Join-Path $OutputDirectory 'matching.stderr.log') -Pattern '\[d4r-launch\].*backend=native') + if ($hits.Count -ne 32) { throw "Expected 32 native launches, got $($hits.Count)" } + Write-Host 'PASS NATIVE_IDENTITY wrong_hash=fallback matching_hash=native launches=32' +} finally { + foreach ($name in $previous.Keys) { [Environment]::SetEnvironmentVariable($name,$previous[$name],'Process') } +} diff --git a/scripts/windows/test-windows-rdna4.ps1 b/scripts/windows/test-windows-rdna4.ps1 index cb4b5ff..fe5c078 100644 --- a/scripts/windows/test-windows-rdna4.ps1 +++ b/scripts/windows/test-windows-rdna4.ps1 @@ -10,6 +10,8 @@ param( [ValidateSet('init', 'evaluate')][string]$NgxMode = 'init', [ValidateSet(5, 11, 13)][int]$Preset = 11, [switch]$NgxOnly, + [switch]$Trace, + [switch]$RequireNativeNetwork, [int]$Iterations = 32, [int]$TimeoutSeconds = 180 ) @@ -48,6 +50,7 @@ $originalLog = $env:ZLUDA_LOG_DIR $originalCuda = $env:ZLUDA_CUDA_LIB $originalNvapi = $env:D4R_NVAPI_BACKEND $originalPythonPath = $env:PYTHONPATH +$originalVerbose = $env:D4R_ZLUDA_VERBOSE $exitStatus = 1 $ngxRuntimeDirectory = $null $summary = [ordered]@{utc=[DateTime]::UtcNow.ToString('o'); profile=$RuntimeProfile; hipRoot=$HipRoot; zludaRoot=$ZludaRoot; tests=@()} @@ -72,13 +75,17 @@ function Invoke-Probe([string]$Name, [string]$Exe, [string[]]$Arguments) { $process.StartInfo = $info if (!$process.Start()) { throw "Process start failed: $Exe" } # Consume both streams concurrently; a verbose JIT trace must not block its child. - $outTask = $process.StandardOutput.ReadToEndAsync() - $errTask = $process.StandardError.ReadToEndAsync() + $outFile = [IO.File]::Open($stdout, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::Read) + $errFile = [IO.File]::Open($stderr, [IO.FileMode]::Create, [IO.FileAccess]::Write, [IO.FileShare]::Read) + $outTask = $process.StandardOutput.BaseStream.CopyToAsync($outFile) + $errTask = $process.StandardError.BaseStream.CopyToAsync($errFile) $finished = $process.WaitForExit($TimeoutSeconds * 1000) if (!$finished) { $process.Kill(); $process.WaitForExit() } $code = $process.ExitCode - [IO.File]::WriteAllText($stdout, $outTask.GetAwaiter().GetResult()) - [IO.File]::WriteAllText($stderr, $errTask.GetAwaiter().GetResult()) + try { + $null = $outTask.GetAwaiter().GetResult() + $null = $errTask.GetAwaiter().GetResult() + } finally { $outFile.Dispose(); $errFile.Dispose() } $process.Dispose() $codeHex = if ($null -eq $code) { 'unknown' } else { '0x{0:x8}' -f [BitConverter]::ToUInt32([BitConverter]::GetBytes([int]$code), 0) @@ -90,6 +97,12 @@ function Invoke-Probe([string]$Name, [string]$Exe, [string[]]$Arguments) { } try { if ($NgxOnly -and (!$NgxCore -or !$DlssDll)) { throw '-NgxOnly requires both locally supplied NVIDIA DLL paths.' } + if ($RequireNativeNetwork) { + if ($NgxMode -ne 'evaluate' -or $Preset -ne 11 -or !$env:D4R_ZLUDA_NATIVE_DIR) { + throw '-RequireNativeNetwork currently requires K Evaluate and D4R_ZLUDA_NATIVE_DIR.' + } + $env:D4R_ZLUDA_VERBOSE = '1' + } $summary.foundationDiagnosticsPerformed = !$NgxOnly if ($Iterations -lt 1 -or $Iterations -gt 10000) { throw 'Iterations must be 1..10000' } if ($TimeoutSeconds -lt 1) { throw 'TimeoutSeconds must be positive' } @@ -199,12 +212,27 @@ try { Copy-Item -LiteralPath (Join-Path $bin 'd4r_ngx_cuda_init_probe.exe') -Destination $ngxExe Copy-Item -LiteralPath $NgxCore -Destination $localCore Copy-Item -LiteralPath $DlssDll -Destination $localDlss - $ngxArguments = @('--hip-root', $HipRoot, '--cuda-dll', (Join-Path $ZludaRoot 'nvcuda.dll'), + $selectedCuda = if ($Trace) { Join-Path $ZludaRoot 'trace/nvcuda.dll' } else { Join-Path $ZludaRoot 'nvcuda.dll' } + if (!(Test-Path -LiteralPath $selectedCuda)) { throw "CUDA runtime missing: $selectedCuda" } + $summary.traceRequested = [bool]$Trace + $ngxArguments = @('--hip-root', $HipRoot, '--cuda-dll', $selectedCuda, '--ngx-core', $localCore, '--dlss-dll', $localDlss, '--nvapi-dll', (Join-Path $PackageRoot 'nvapi-compat/nvapi64.dll'), '--ngx-mode', $NgxMode, '--preset', "$Preset", '--iterations', "$Iterations") $ngxName = if ($NgxMode -eq 'evaluate') { "ngx-evaluate-preset-$Preset" } else { 'ngx-init' } $ngxOk = Invoke-Probe $ngxName $ngxExe $ngxArguments + if ($ngxOk -and $RequireNativeNetwork) { + $hits = @() + foreach ($layer in @('enc0','enc1','enc2','enc3','enc4','dec5','dec4','dec3','dec2','dec1','dec0')) { + $pattern = '\[d4r-launch\] kernel="dltss_pwin_' + $layer + '_layer" backend=native' + $count = @(Select-String -LiteralPath (Join-Path $OutputDirectory "$ngxName.stderr.log") -Pattern $pattern).Count + $hits += @{layer=$layer; nativeLaunches=$count; expectedFrames=$Iterations} + if ($count -lt $Iterations) { $ngxOk=$false } + } + $summary.nativeTransformer = $hits + $summary.nativeTransformerPassed = $ngxOk + Write-Host "Native K transformer launches validated=$ngxOk" + } if (!$ngxOk) { $exitStatus = 1 if (Test-Path (Join-Path $ZludaRoot 'trace/nvcuda.dll')) { @@ -240,6 +268,7 @@ try { $env:ZLUDA_CUDA_LIB = $originalCuda $env:D4R_NVAPI_BACKEND = $originalNvapi $env:PYTHONPATH = $originalPythonPath + $env:D4R_ZLUDA_VERBOSE = $originalVerbose $archive = "$OutputDirectory.zip" Compress-Archive -LiteralPath $OutputDirectory -DestinationPath $archive -Force Write-Host "Diagnostic bundle: $archive" diff --git a/tools/windows/cuda_driver_probe.cpp b/tools/windows/cuda_driver_probe.cpp index 9da1a01..2594c3d 100644 --- a/tools/windows/cuda_driver_probe.cpp +++ b/tools/windows/cuda_driver_probe.cpp @@ -38,6 +38,9 @@ int main(int argc, char** argv) start(); try { Args args(argc, argv); + uint64_t identity = 0xcbf29ce484222325ull; + for (const char* c = ptx; *c; ++c) identity = (identity ^ uint8_t(*c)) * 0x100000001b3ull; + std::printf("PTX_IDENTITY kernel=d4r_ptx_pattern fnv1a64=%016llx\n", static_cast(identity)); if (args.cuda_dll.empty()) throw std::runtime_error("--cuda-dll is required (absolute ZLUDA nvcuda.dll path)"); if (args.context != "primary" && args.context != "created") throw std::runtime_error("context must be primary or created"); // Require the real HIP architecture, then match the CUDA device by PCI identity. diff --git a/tools/windows/frame_compare.py b/tools/windows/frame_compare.py new file mode 100644 index 0000000..01f8bbe --- /dev/null +++ b/tools/windows/frame_compare.py @@ -0,0 +1,50 @@ +"""Validate every RGBA16F diagnostic frame against another complete pipeline.""" +import argparse +import math +import pathlib +import sys +import numpy as np + + +def main(args): + frames = sorted(args.reference.glob('output-*.rgba16f')) + if not frames: + raise RuntimeError('Reference contains no frames') + for path in frames: + actual = args.actual / path.name + if not actual.is_file(): + raise RuntimeError(f'Missing output frame {path.name}') + reference = np.fromfile(path, np.float16) + output = np.fromfile(actual, np.float16) + if reference.size != output.size or reference.size != args.width * args.height * 4: + raise RuntimeError(f'Output size mismatch for {path.name}') + if not np.isfinite(reference).all() or not np.isfinite(output).all(): + raise RuntimeError(f'NaN/Inf in {path.name}') + x = reference.reshape(-1, 4)[:, :3].astype(np.float64) + y = output.reshape(-1, 4)[:, :3].astype(np.float64) + error = np.abs(x - y) + rms = float(np.sqrt(np.mean(error ** 2))) + peak = max(1., float(np.abs(x).max())) + psnr = math.inf if rms == 0 else 20 * math.log10(peak / rms) + outliers = float(np.mean(error > .01 * peak)) + print(f'FRAME_REFERENCE name={path.name} max_abs={error.max():.9g} ' + f'max_rel={(error / np.maximum(np.abs(x), .001)).max():.9g} rms={rms:.9g} psnr_db={psnr:.6g} ' + f'fraction_over_1pct={outliers:.9g}') + if psnr < args.min_psnr or error.max() > .025 * peak or outliers > .0001: + raise RuntimeError(f'Complete pipeline mismatch in {path.name}') + print(f'PASS FRAME_REFERENCE frames={len(frames)} finite=1 min_psnr_db={args.min_psnr}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--reference', type=pathlib.Path, required=True) + parser.add_argument('--actual', type=pathlib.Path, required=True) + parser.add_argument('--width', type=int, default=512) + parser.add_argument('--height', type=int, default=288) + parser.add_argument('--min-psnr', type=float, default=60.) + args = parser.parse_args() + try: + main(args) + except Exception as error: + print(f'FAIL FRAME_REFERENCE {error}', file=sys.stderr) + raise SystemExit(5) diff --git a/tools/windows/k_ptx_validate.py b/tools/windows/k_ptx_validate.py new file mode 100644 index 0000000..bfbb2f6 --- /dev/null +++ b/tools/windows/k_ptx_validate.py @@ -0,0 +1,95 @@ +"""Compare real K captures with the upstream CPU PTX interpreter. + +The local DLL and captures remain private. Only numerical metrics are public. +Use this independent reference to audit the NumPy model's reduction uncertainty. +""" +import argparse +import math +import pathlib +import re +import sys + +import numpy as np + +ROOT = pathlib.Path(__file__).resolve().parents[2] +REFERENCE = pathlib.Path(__file__).resolve().parent / 'reference' +if not (REFERENCE / 'ptxsim.py').exists(): + REFERENCE = ROOT / 'kernels/tools' +sys.path.insert(0, str(REFERENCE)) +import pwin_model as pm +pm.NORM_FRAGMENT_ORDER = True +import pwin_check as pc +import ptxsim +from extract_dlss_ptx import fatbins, ptx_entries + + +def metrics(label, expected, actual): + expected = expected.astype(np.float64) + actual = actual.astype(np.float64) + if not np.isfinite(expected).all() or not np.isfinite(actual).all(): + raise RuntimeError(f'{label} contains NaN/Inf') + error = np.abs(expected - actual) + peak = max(1., float(np.abs(expected).max())) + maximum = float(error.max()) + relative = float((error / np.maximum(np.abs(expected), .001)).max()) + rms = float(np.sqrt(np.mean(error ** 2))) + psnr = math.inf if rms == 0 else 20 * math.log10(peak / rms) + print(f'{label} max_abs={maximum:.9g} max_rel={relative:.9g} rms={rms:.9g} ' + f'peak={peak:.9g} psnr_db={psnr:.6g}', flush=True) + return maximum, peak, psnr + + +def validate(args): + name = f'dltss_pwin_{args.layer}_layer' + dump = pm.Dump(str(args.capture_dir)) + if dump.kernel != name: + raise RuntimeError('Capture kernel does not match requested layer') + native = pc.Outputs(dump, str(args.native_dir)) + candidates = [] + for bx, by in pc.windows(dump.grid, 12): + model = pc.model_window(dump, args.layer, bx, by) + expected, actual = pc.stored(dump, native, args.layer, bx, by, *model) + difference = float(np.abs(expected.astype(np.float64) - actual.astype(np.float64)).max()) + candidates.append((difference, bx, by, model, expected, actual)) + selected = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))[:args.windows] + pattern = re.compile(rb'\.entry\s+' + name.encode() + rb'\s*\(') + sources = [source.rstrip(b'\0') for _, fatbin in fatbins(args.dlss_dll.read_bytes()) + for source in ptx_entries(fatbin) if pattern.search(source)] + if not sources: + raise RuntimeError('Local DLSS DLL does not contain the captured kernel') + args.output_dir.mkdir(parents=True, exist_ok=True) + path = args.output_dir / (name + '.ptx') + path.write_bytes(sources[-1]) + for _, bx, by, model, numpy_expected, actual in selected: + _, memory, steps = ptxsim.run_block(str(path), name, str(args.capture_dir), (bx, by, 0)) + reference_output = pc.Outputs.__new__(pc.Outputs) + reference_output.d = dump + reference_output.allocs = {identifier: (base, data) for identifier, base, data + in zip(memory.ids, memory.bases, memory.data)} + _, expected = pc.stored(dump, reference_output, args.layer, bx, by, *model) + print(f'PTX_REFERENCE layer={args.layer} window={bx},{by} statements={steps}', flush=True) + maximum, peak, psnr = metrics('PTX_VS_NATIVE', expected, actual) + metrics('PTX_VS_NUMPY', expected, numpy_expected) + # Half reductions and approximate reciprocal/rsqrt are not bit-exact + # across ISAs. Bound both the worst element and total error explicitly. + if maximum > max(.001, .005 * peak) or psnr < 60: + raise RuntimeError('Native layer differs from independent PTX reference') + print(f'PASS K_PTX_REFERENCE layer={args.layer} windows={len(selected)} nan_inf=0', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--layer', choices=list(pc.LAYERS), required=True) + parser.add_argument('--capture-dir', type=pathlib.Path, required=True) + parser.add_argument('--native-dir', type=pathlib.Path, required=True) + parser.add_argument('--dlss-dll', type=pathlib.Path, required=True) + parser.add_argument('--output-dir', type=pathlib.Path, required=True) + parser.add_argument('--windows', type=int, default=2) + args = parser.parse_args() + if args.windows < 1: + parser.error('--windows must be positive') + try: + validate(args) + except Exception as error: + print(f'FAIL K_PTX_REFERENCE {error}', file=sys.stderr) + raise SystemExit(5) diff --git a/tools/windows/k_replay_validate.py b/tools/windows/k_replay_validate.py index 8c8b3c4..348ff17 100644 --- a/tools/windows/k_replay_validate.py +++ b/tools/windows/k_replay_validate.py @@ -18,7 +18,8 @@ if not (REFERENCE / 'pwin_check.py').exists(): REFERENCE = ROOT / 'kernels/tools' sys.path.insert(0, str(REFERENCE)) import pwin_model as pm -from pwin_check import LAYERS, model_window, Outputs, stored +pm.NORM_FRAGMENT_ORDER = True +from pwin_check import LAYERS, model_window, Outputs, stored, windows def generate(layer, directory): @@ -113,10 +114,14 @@ def generate(layer, directory): print(f'PASS K_FIXTURE kernel={layer} weights={weight_bytes} public_synthetic=1') -def validate(layer, directory, output): +def validate(layer, directory, output, window_count=1, real_capture=False): dump = pm.Dump(str(directory)) - model, extra = model_window(dump, layer, 0, 0) - expected, actual = stored(dump, Outputs(dump, str(output)), layer, 0, 0, model, extra) + outputs = Outputs(dump, str(output)) + pairs = [] + for bx, by in windows(dump.grid, window_count): + model, extra = model_window(dump, layer, bx, by) + pairs.append(stored(dump, outputs, layer, bx, by, model, extra)) + expected, actual = (np.concatenate(values) for values in zip(*pairs)) if not np.isfinite(expected).all() or not np.isfinite(actual).all(): raise RuntimeError('K replay contains NaN/Inf or unwritten output') difference = np.abs(actual.astype(np.float64) - expected.astype(np.float64)) @@ -127,7 +132,11 @@ def validate(layer, directory, output): psnr = math.inf if not rms else 20 * math.log10(peak / rms) print(f'K_REPLAY_REFERENCE kernel={layer} elements={actual.size} max_abs={maximum:.9g} ' f'max_rel={relative:.9g} rms={rms:.9g} psnr_db={psnr:.6g}') - if maximum > .001 or psnr < 60: + # Bound ISA approximate reciprocal/rsqrt and matrix accumulation differences; + # require the independent PTX reference gate for every real-weight layer. + absolute_limit = max(.001, .005 * peak) if real_capture else .001 + print(f'K_REPLAY_LIMIT windows={len(pairs)} max_abs_limit={absolute_limit:.9g} min_psnr_db=60 real_capture={int(real_capture)}') + if maximum > absolute_limit or psnr < 60: worst = int(difference.argmax()) raise RuntimeError(f'K mismatch at {worst}: reference={expected[worst]}, gpu={actual[worst]}') print(f'PASS K_REPLAY_REFERENCE kernel={layer} architecture=gfx1201 nan_inf=0') @@ -139,6 +148,8 @@ if __name__ == '__main__': parser.add_argument('--layer', choices=list(LAYERS), required=True) parser.add_argument('--fixture-dir', type=pathlib.Path, required=True) parser.add_argument('--output-dir', type=pathlib.Path) + parser.add_argument('--windows', type=int, default=1) + parser.add_argument('--real-capture', action='store_true') args = parser.parse_args() try: if args.mode == 'generate': @@ -146,7 +157,9 @@ if __name__ == '__main__': else: if not args.output_dir: parser.error('validate requires --output-dir') - validate(args.layer, args.fixture_dir, args.output_dir) + if args.windows < 1: + parser.error('--windows must be positive') + validate(args.layer, args.fixture_dir, args.output_dir, args.windows, args.real_capture) except Exception as error: print(f'FAIL K_REPLAY {error}', file=sys.stderr) raise SystemExit(5) diff --git a/tools/windows/ngx_cuda_init_probe.cpp b/tools/windows/ngx_cuda_init_probe.cpp index 758523a..22628a5 100644 --- a/tools/windows/ngx_cuda_init_probe.cpp +++ b/tools/windows/ngx_cuda_init_probe.cpp @@ -121,7 +121,7 @@ int main(int argc, char** argv) cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize"); cuda.check(cuda.cuCtxSetCurrent(nullptr), "cuCtxSetCurrent(NULL)"); cuda.check(cuda.cuDevicePrimaryCtxRelease_v2(device), "cuDevicePrimaryCtxRelease_v2"); context = nullptr; - std::printf("PASS NGX_INIT architecture=gfx1201 sr_available=1 transformer_executed=0\n"); + std::printf("PASS NGX_INIT architecture=gfx1201 sr_available=1 transformer_validation=pending\n"); return 0; } catch (const std::exception& e) { std::fprintf(stderr, "FAIL NGX_INIT %s\n", e.what()); diff --git a/tools/windows/probe_kernel.hip b/tools/windows/probe_kernel.hip index 250d25f..bac23fa 100644 --- a/tools/windows/probe_kernel.hip +++ b/tools/windows/probe_kernel.hip @@ -1,4 +1,13 @@ // No host SDK, CRT, or device library dependency. Compiled only for gfx1201. +extern "C" __attribute__((global)) void d4r_ptx_pattern( + unsigned int* output, unsigned int count, unsigned int seed) +{ + const unsigned int i = __builtin_amdgcn_workgroup_id_x() * 128u + + __builtin_amdgcn_workitem_id_x(); + if (i < count) + output[i] = (i * 1664525u + seed) ^ 0xa5a55a5au; +} + extern "C" __attribute__((global)) void d4r_probe_pattern( unsigned int* output, unsigned int count, unsigned int seed) {