From 5461c161f3e33c78c4535ccec30b6121563fe4b7 Mon Sep 17 00:00:00 2001 From: woutermans Date: Sat, 3 Oct 2026 23:21:46 +0200 Subject: [PATCH] Keep RR guides and separate alpha in VRAM --- docs/architecture.md | 10 +- docs/building.md | 45 ++ docs/performance.md | 25 +- packaging/RELEASE_NOTES.md | 19 +- scripts/run_rr_guide_compare.sh | 144 ++++++ tests/test_rr_output.py | 89 ++++ tools/compare_rr_output.py | 244 ++++++++++ tools/d3d12_dlss_harness.cpp | 455 +++++++++++++++++- tools/d4r_nvngx_shim.cpp | 825 ++++++++++++++++++++++++++------ tools/wine_nvcuda_bridge.c | 10 +- 10 files changed, 1697 insertions(+), 169 deletions(-) create mode 100755 scripts/run_rr_guide_compare.sh create mode 100644 tests/test_rr_output.py create mode 100755 tools/compare_rr_output.py diff --git a/docs/architecture.md b/docs/architecture.md index bc9897e..1d41454 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,9 +44,15 @@ Feature 13 uses the NGX core's CUDA lifecycle, not the denoiser's native D3D12 b Feature requirements may be queried before NGX initialization. The shim loads the providers without initializing NGX with a provisional application identity; when the core declines feature-13 discovery, it asks the CUDA-capable denoiser's D3D12 requirements export for the adapter's eligibility. This discovery-only call does not select the native D3D12 rendering backend. Initialized capability checks and feature creation still require the core's successful denoiser initialization. -The shim snapshots denoiser guide bindings, scalar settings, camera matrices, and subrect origins per submitted frame. Normals and roughness use the SDK's `GBuffer.Normals` and `GBuffer.Roughness` resource keys; diffuse and specular albedo bindings are translated to the CUDA denoiser's `DLSS.Input.*Albedo` keys. Guide readback and CUDA upload follow the main input marker, and the backing resources remain held until that frame retires. Uploads use per-slot host staging because direct ROCm image copies from Vulkan-mapped memory truncated guide rows on gfx1201. CUDA parameters point at stable texture-handle storage, not D3D12 resource pointers. Normals preserve the caller's component values rather than being renormalized during format conversion. +The shim snapshots denoiser guide bindings, scalar settings, camera matrices, and subrect origins per submitted frame. Normals and roughness use the SDK's `GBuffer.Normals` and `GBuffer.Roughness` keys; albedo bindings are translated to the CUDA denoiser's `DLSS.Input.*Albedo` keys. CUDA parameters point at stable texture-handle storage, not D3D12 resource pointers. Normals preserve the caller's component values rather than being renormalized during conversion. -Separate alpha output follows the same produced-frame slot as RGB, including its output resource and subrect. Requesting separate alpha cuts over through the existing drained host-staging path instead of publishing mismatched VRAM RGB and alpha results. +With VRAM interop, guides copy from Vulkan images into per-slot shared device buffers, then into the CUDA arrays sampled by the denoiser. Half, float, UNORM, and SNORM layouts copy verbatim where possible. BGRA and packed RGB formats are converted to RGBA32F by a Vulkan blit. Raw integer guides retain the host conversion because integer-to-float blits do not reproduce its semantics. CUDA import happens on the worker after allocation, outside the guide-binding lock. The input marker orders both routes, and resources remain held until their frame retires. + +`D4R_SHIM_RR_VRAM_GUIDES=0` selects the host guide route for A/B checks without disabling RGB VRAM interop. `D4R_SHIM_VRAM_VERIFY=1` also stages the guides and compares the actual CUDA arrays against the CPU conversion, failing the frame on mismatches. Verification preserves pitch-linear input row alignment; diagnostics must not replace GPU geometry with a tightly packed host stride. + +Separate alpha no longer forces RGB through host staging. R32F alpha copies from the denoiser's CUDA array into a shared result buffer and into the caller's subrect; half and UNORM destinations use a GPU blit. Raw integer destinations keep their existing host conversion, independently of the RGB route. Blit rounding is hardware-dependent and can differ from CPU conversion by one destination quantization step. + +Split frames reserve the current frame's alpha slot before its producer runs. Previous-frame presentation matches alpha to RGB's produced frame and validates its format and subrect. Pixels outside the evaluated rectangle remain untouched. `D4R_SHIM_ALPHA_VERIFY=1` compares the array region with its shared buffer; combined with output capture, it also saves that region as `*.rgba16f.alpha.f32`. ## Why each piece exists diff --git a/docs/building.md b/docs/building.md index d554de4..4470c64 100644 --- a/docs/building.md +++ b/docs/building.md @@ -104,6 +104,25 @@ scripts/d4r_play.sh In the game, pick DLSS as the upscaler (OptiScaler intercepts it). Every setting in the file can also be given as an environment variable, which takes precedence; `D4R_NO_CONFIG=1` ignores the file. +### Cyberpunk 2077 RR launch notes + +The optimized RR route was exercised in a loaded Cyberpunk 2077 2.31 save on RX 9070 XT (gfx1201), with preset E, path tracing and Ultra Performance (640×360 → 1920×1080). The user also verified gameplay. This is a functional check, not a comparative FPS benchmark. + +On the tested GE-Proton11-3 setup, Wine's builtin ICU aborted on `icuuc.dll.u_setMemoryFunctions_65`. Use the game's native ICU DLLs for that setup: + +```sh +WINEDLLOVERRIDES='icuuc=n,b;icuin=n,b;icudt=n,b;d3d12=n,b;d3d12core=n,b' \ + D4R_RR_ENABLE=1 D4R_RR_PRESET=5 \ + D4R_SHIM_VRAM_INTEROP=1 D4R_SHIM_SPLIT_FRAME=1 D4R_SHIM_LINEAR_INPUTS=1 \ + D4R_ZLUDA_WMMA=1 D4R_ZLUDA_WMMA_FP8=1 D4R_ZLUDA_WMMA_FP8_NATIVE=1 \ + scripts/d4r_play.sh +``` + +Merge these overrides with any existing `WINEDLLOVERRIDES`; do not discard overrides needed by other mods. Keep the patched vkd3d DLLs current, including game-local copies as described above. Select DLSS, Ray Reconstruction and path tracing in the game. `D4R_SHIM_RR_VRAM_GUIDES` defaults to `1` with VRAM interop; `0` selects host guide staging for A/B diagnostics. + +Leave `D4R_SHIM_VRAM_VERIFY`, `D4R_SHIM_ALPHA_VERIFY` and output capture disabled for normal play or benchmarking. The in-game diagnostic run enabled verification and reported zero differing bytes for all six supplied RR guides: normals, diffuse/specular albedo, specular hit distance, pre-particle color and the subsurface-scattering guide. Verification itself downloads GPU data. + + ## 7. Package a release ```sh @@ -160,3 +179,29 @@ Run `bash scripts/build_release_glibc241.sh` inside the container. The script ch - `D4R_HARNESS_RECREATE_OUTPUT=1` releases and recreates the game's output texture between completed frames; compare its copy and in-place sequences to catch imported-memory handle reuse. - `D4R_HARNESS_RECREATE=4 D4R_HARNESS_VERIFY_RECREATION=1` checks repeated feature/quality recreation. `D4R_HARNESS_SKIP_RECORDED_INPUT=1` checks discarded command-list input readiness without injecting stale history. - Native kernels have their own validation path; see [native-kernels.md](native-kernels.md). + +### Ray Reconstruction VRAM comparison + +On the validated gfx1201/native-FP8 runtime, build the harness and shim, then compare host and GPU guide transport: + +```sh +scripts/build_d3d12_dlss_harness.sh +D4R_SHIM_SPLIT_FRAME=1 D4R_SHIM_LINEAR_INPUTS=1 \ + D4R_SHIM_VRAM_VERIFY=1 D4R_SHIM_ALPHA_VERIFY=1 \ + D4R_RR_ALPHA=1 D4R_RR_GUIDE_PROBE=1 \ + D4R_HARNESS_TEMPORAL=1 \ + D4R_HARNESS_SIZE='193 109 288 162' \ + scripts/run_rr_guide_compare.sh /tmp/rr-compare \ + rgba16f,rgba16f,rgba8unorm,rgba8unorm \ + rgba8snorm,rgba16unorm,bgra8unorm,rgb10a2unorm \ + rgba32f,r32f,rgb11b10float,rgba8unorm \ + rgba16f,r32uint,bgra8unorm,rgba16f +``` + +The script stages the matching vkd3d DLLs, real DXVK-NVAPI, shim, and feature libraries together. It defaults to RR preset E and enables the required native-FP8 flags. Override `D4R_RUNTIME_DIR`, `D4R_PROTON_DIR`, or `D4R_VKD3D_DIR` for an isolated runtime. Neither feature discovery nor failed evaluation is bypassed. + +Every saved RGB and alpha frame is compared. The reset guide probe requires an image response, not spatial confinement: transformer attention may legitimately change both halves. `D4R_HARNESS_RR_ALPHA_FORMAT` selects `r32f`, `r16f`, `r8unorm`, or `r16unorm`; alpha captures are always decoded float32, with format-appropriate outside-subrect sentinels. + +Verification intentionally downloads GPU data. Unset `D4R_SHIM_VRAM_VERIFY` and `D4R_SHIM_ALPHA_VERIFY` for performance measurements and games. For same-frame conversion checks, combine alpha verification with `D4R_SHIM_OUTPUT_DUMP_DIR` and the existing dump frame selectors; compare each saved alpha region with the corresponding presented alpha capture, allowing one destination quantization step for blit rounding. + +`python3 -m unittest discover -s tests -v` includes output-comparator regressions for channel swaps, nonfinite pixels, missing/truncated captures, alpha frame mismatches, and writes outside the output subrect. diff --git a/docs/performance.md b/docs/performance.md index 0e179ab..7e46979 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,6 +1,6 @@ # Performance -Unless stated otherwise, the frame rates and kernel timings on this page were measured on an RX 7700 XT (gfx1101). Native FP8 changes both arithmetic and cost, so the gfx11 results cannot be projected to RDNA4; the one RDNA4 measurement is in [RDNA4](#rdna4). No RDNA4 game frame rate has been measured. +Unless stated otherwise, the frame rates and kernel timings on this page were measured on an RX 7700 XT (gfx1101). Native FP8 changes both arithmetic and cost, so the gfx11 results cannot be projected to RDNA4; the RDNA4 Super Resolution measurement is in [RDNA4](#rdna4). The Ray Reconstruction measurements and Cyberpunk verification use an RX 9070 XT (gfx1201); the RR timings are synthetic harness measurements, not RDNA4 game-FPS benchmarks. ## Method @@ -142,3 +142,26 @@ DLSS 4.5 (L) on the same card and harness, with enc0 and dec0 as ZLUDA's compile | **all kernels** | **5.83** | **4.06** | **5.93** | **4.34** | **8.63** | **5.69** | **5.99** | **4.14** | L's post kernel (1.2–1.5 ms) is the largest kernel left; it has no matrix multiplies. The 1920×1080 runs were taken in one of the two clock states the driver's governor settles in at that load (about 13% apart), so compare within the pair, not across resolutions. + +## Ray Reconstruction VRAM transport + +Measured separately from the Super Resolution game benchmarks above: RX 9070 XT (gfx1201), GE-Proton11-3, CUDA-capable DLSS-Denoiser 310.7, RR preset E, native FP8 WMMA and native Swin encoders. The synthetic D3D12 harness rendered 640×360 into 1280×720 with separate R32F alpha. Split-frame presentation and pitch-linear inputs were enabled; verification/capture downloads were disabled. + +Four back-to-back 40-frame runs used host guides, GPU guides, GPU guides, then host guides. The first five frames of each run were excluded; values below are warm-frame medians in milliseconds: + +| Guide transport | Worker time, first/repeat | Frame interval, first/repeat | +|---|---:|---:| +| Host staging | 14.967 / 14.296 | 17.373 / 16.795 | +| Shared VRAM | 13.960 / 13.953 | 16.482 / 16.680 | + +All four final RGBA captures were byte-identical. The GPU guide runs recorded no host staging readbacks. This is a small synthetic transport comparison, not a game-FPS claim; network execution and clocks contribute to run-to-run variation. + +With the usual two RGBA16F and two RGBA8 guides, VRAM transport eliminates approximately 11.1 MB of guide payload crossing GPU↔CPU per frame at this render size (5.5 MB in each direction, excluding padding). Separate alpha also no longer forces RGB through host staging. Unsupported raw-integer guide/alpha formats retain their existing host conversion without disabling the other GPU routes. + +Correctness checks additionally covered five guide-format sets at an odd 193-pixel input width: 35 final/per-frame RGBA files were byte-identical between host and GPU guide transport. Six-frame source/presentation captures preserved RGB bytes and alpha subrect sentinels. R32F alpha was exact; half/UNORM blits can differ from the CPU quantizer by one destination step. + +An aligned 192×108→288×162 synthetic capture against native CUDA on RTX 4090 measured RGB PSNR 42.15 dB, mean absolute component error 0.00357, and maximum error 0.13623; neither image contained nonfinite RGB values. This spot check does not establish RTX image-quality parity or game-wide temporal correctness. Transport A/B comparisons on the same AMD backend were exact; cross-backend neural arithmetic remains a separate accuracy limitation. + +The optimized route also rendered a loaded Cyberpunk 2077 2.31 save at 640×360→1920×1080 Ultra Performance with path tracing and RR preset E on gfx1201. The overlay identified DLSSD 310.7.0; the shim created CUDA feature 13 with split-frame VRAM interop. In-game verification found zero differing bytes for all six supplied guide arrays, and the user verified gameplay. This diagnostic run enabled GPU readbacks and does not establish an FPS improvement or a game-wide artifact/latency audit. + +The extended diagnostic log also recorded prolonged split-frame waits and a GPU-marker timeout later in the run; their cause was not isolated. The successful rendering and guide comparisons are not evidence of stall-free operation. diff --git a/packaging/RELEASE_NOTES.md b/packaging/RELEASE_NOTES.md index adb698e..07917da 100644 --- a/packaging/RELEASE_NOTES.md +++ b/packaging/RELEASE_NOTES.md @@ -21,11 +21,18 @@ - Run the signed CUDA-capable 3.10.7 DLSS-Denoiser through the NGX core's feature-13 lifecycle. Presets D and E select distinct network weights independently of the Super Resolution preset. - Preserve per-frame guide resources, camera matrices, subrects, and stable CUDA texture handles. - Support normalized 8-bit albedo guides with the CUDA array-format ABI and stage full guide rows. + Support normalized 8-bit albedo guides with the CUDA array-format ABI and preserve full guide rows. - Support pre-Init feature discovery through the denoiser's requirements export without initializing NGX with a provisional application identity. Initialized capability checks remain authoritative. -- Keep separate alpha and RGB outputs paired, including padded output subrects; consume alpha - readback synchronously before CPU conversion. +- Keep separate alpha and RGB outputs paired, including padded output subrects and current-frame + split presentation. Separate alpha no longer forces the whole feature through host staging. + R32F alpha stays in VRAM; half/UNORM alpha uses GPU conversion with hardware-dependent rounding. + Raw integer alpha retains its existing host conversion without disabling RGB VRAM interop. + Converted alpha can differ from CPU quantization by one destination step. + Normals, roughness, albedo, and other compatible RR guides use shared VRAM buffers; BGRA and + packed RGB guides convert on the GPU. Raw integer guides retain their host conversion. + Add actual-CUDA-array verification, per-frame rendered RGB/alpha comparisons, and source-alpha + capture for checking conversion against the exact frame being presented. - Bound R8_UINT alpha stores to one byte and balance guide/alpha references when evaluation cannot place its frame marker. Publish cached denoiser capabilities as a coherent snapshot. - Extend the ZLUDA patch stack for real denoiser PTX, LLVM lowering, half texture/store operations, @@ -37,6 +44,12 @@ 1920×1080 and 3840×2160 Ultra Performance on gfx1201. The actual overlay reported DLSSD 310.7.0, and CUDA feature-13 evaluations completed. The observed ~53 FPS at 1080p and ~20 FPS at 4K are snapshots, not comparative benchmarks. Newer DLSS 4.5 RR preset F is not implemented or validated. +- Rechecked the optimized VRAM guide path in a loaded Cyberpunk 2077 2.31 save with path tracing, + RR preset E and 1080p Ultra Performance on gfx1201; the user verified gameplay. + All six supplied guide arrays matched the host reference byte-for-byte in diagnostic checks. + The tested GE-Proton11-3 setup required native ICU overrides + (`icuuc=n,b;icuin=n,b;icudt=n,b`) to avoid Wine's unimplemented `u_setMemoryFunctions_65`. + Keep guide/alpha verification off for normal play; it intentionally adds GPU readbacks. # d4r 0.1.3 diff --git a/scripts/run_rr_guide_compare.sh b/scripts/run_rr_guide_compare.sh new file mode 100755 index 0000000..c75a613 --- /dev/null +++ b/scripts/run_rr_guide_compare.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runs the D3D12 harness's Ray Reconstruction path twice with identical inputs - +# once with the shim's guides travelling on the GPU (D4R_SHIM_RR_VRAM_GUIDES=1, +# the default) and once with them staged through the host (=0) - and compares the +# rendered images, every saved frame and the alpha output. +# +# The two runs differ only in that one variable, so any difference in the image is +# the guide path's, not the denoiser's: same colour, depth, motion, exposure, jitter, +# sizes and frame count. The harness additionally checks the guides themselves when +# D4R_HARNESS_RR_GUIDE_PROBE=1 (a guide change must affect the rendered image; +# transformer attention may spread that response beyond the changed half). +# +# usage: run_rr_guide_compare.sh OUTPUT_DIR [GUIDE_FORMAT_SET ...] +# each set is a comma-separated D4R_HARNESS_GUIDE_FORMATS value, one token per +# plane (GBuffer.Normals, GBuffer.Roughness, GBuffer.DiffuseAlbedo, +# GBuffer.SpecularAlbedo). The default is one set, rgba16f,rgba16f,rgba8unorm,rgba8unorm. +# +# environment: +# D4R_PROTON_DIR Proton build to launch through (default: GE-Proton11-3) +# D4R_RUNTIME_DIR install with nvngx_dlssd.dll, _nvngx.dll, kernels (default ~/.local/share/d4r-dlss) +# D4R_VKD3D_DIR d4r-patched d3d12.dll/d3d12core.dll (default ~/.cache/d4r-vkd3d-d4r) +# D4R_ZLUDA_DIR ZLUDA build whose libcuda.so the Wine bridge loads (default ~/.cache/d4r-zluda-current) +# D4R_ROCM_DIR ROCm user-space runtime (default ~/.cache/d4r-rocm-runtime, else /opt/rocm) +# D4R_NVAPI_DIR directory holding the real DXVK-NVAPI nvapi64.dll (default: from D4R_PROTON_DIR) +# D4R_RR_PRESET denoiser preset, 4 = D, 5 = E (default: E) +# D4R_HARNESS_SIZE "IN_W IN_H OUT_W OUT_H" (default "640 360 1280 720") +# D4R_HARNESS_FRAMES frames per run (default 6) +# D4R_RR_ALPHA=1 also register DLSSD.Alpha / DLSSD.OutputAlpha +# D4R_HARNESS_RR_ALPHA_FORMAT r32f (default), r16f, r8unorm, or r16unorm +# D4R_RR_GUIDE_PROBE=1 also run the in-harness guide response probe +# D4R_HARNESS_SKIP_BUILD=0 build the harness and shim first (default: skip) +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="$(realpath -m "${1:?usage: run_rr_guide_compare.sh OUTPUT_DIR [GUIDE_FORMAT_SET ...]}")" +shift +FORMAT_SETS=("$@") +if [[ ${#FORMAT_SETS[@]} -eq 0 ]]; then + FORMAT_SETS=("rgba16f,rgba16f,rgba8unorm,rgba8unorm") +fi + +PROTON_DIR="${D4R_PROTON_DIR:-$HOME/.local/share/Steam/compatibilitytools.d/GE-Proton11-3}" +RUNTIME_DIR="${D4R_RUNTIME_DIR:-$HOME/.local/share/d4r-dlss}" +VKD3D_DIR="${D4R_VKD3D_DIR:-$HOME/.cache/d4r-vkd3d-d4r}" +ZLUDA_DIR="${D4R_ZLUDA_DIR:-$HOME/.cache/d4r-zluda-current}" +ROCM_DIR="${D4R_ROCM_DIR:-$HOME/.cache/d4r-rocm-runtime}" +[[ -d "$ROCM_DIR/lib" ]] || ROCM_DIR=/opt/rocm +NVAPI_DIR="${D4R_NVAPI_DIR:-$PROTON_DIR/files/lib/wine/nvapi/x86_64-windows}" +FRAMES="${D4R_HARNESS_FRAMES:-6}" +read -r IN_W IN_H OUT_W OUT_H <<< "${D4R_HARNESS_SIZE:-640 360 1280 720}" +APP_DIR="${D4R_RR_APP_DIR:-$OUTPUT_DIR/app}" +HARNESS_EXE="$ROOT/build/d3d12_dlss_harness.exe" + +# Every input the run needs, named so a missing one is a message, not a mystery. +require() { [[ -f "$1" ]] || { printf 'Missing %s (%s)\n' "$1" "$2" >&2; exit 2; }; } +require "$HARNESS_EXE" 'run scripts/build_d3d12_dlss_harness.sh (it also builds the shim)' +require "$ROOT/build/d4r_nvngx.dll" 'run scripts/build_d4r_nvngx_shim.sh' +require "$RUNTIME_DIR/bin/nvngx_dlssd.dll" 'NVIDIA DLSS-Denoiser 3.10.7 (CUDA-capable build)' +require "$RUNTIME_DIR/ngx/_nvngx.dll" 'NVIDIA NGX core' +require "$VKD3D_DIR/d3d12.dll" 'd4r-patched vkd3d-proton (scripts/build_vkd3d_proton_d4r.sh)' +require "$VKD3D_DIR/d3d12core.dll" 'd4r-patched vkd3d-proton' +require "$NVAPI_DIR/nvapi64.dll" 'DXVK-NVAPI from a Proton build' +require "$PROTON_DIR/proton" 'a Proton build' +[[ -e "$ZLUDA_DIR/libcuda.so" ]] || { printf 'Missing %s/libcuda.so (ZLUDA)\n' "$ZLUDA_DIR" >&2; exit 2; } +[[ -d "$RUNTIME_DIR/kernels" ]] || { printf 'Missing %s/kernels (native kernels)\n' "$RUNTIME_DIR" >&2; exit 2; } + +mkdir -p "$APP_DIR" "$OUTPUT_DIR" +# One directory for everything the NGX core and the shim resolve relative to each +# other: the core loads nvngx_dlssd.dll from beside the shim, and the harness loads +# the d4r-patched vkd3d from beside its own executable. +cp -f "$HARNESS_EXE" "$ROOT/build/d4r_nvngx.dll" \ + "$RUNTIME_DIR/bin/nvngx_dlssd.dll" "$RUNTIME_DIR/bin/nvngx_dlss.dll" \ + "$VKD3D_DIR/d3d12.dll" "$VKD3D_DIR/d3d12core.dll" "$NVAPI_DIR/nvapi64.dll" "$APP_DIR/" + +export D4R_HARNESS_SKIP_BUILD="${D4R_HARNESS_SKIP_BUILD:-1}" +export D4R_HARNESS_APP_DIR="$APP_DIR" +export D4R_HARNESS_NGX_DLL="$APP_DIR/d4r_nvngx.dll" +export D4R_RUNTIME_DIR="$RUNTIME_DIR" +export D4R_ZLUDA_DIR="$ZLUDA_DIR" +export D4R_ROCM_DIR="$ROCM_DIR" +export D4R_ZLUDA_NATIVE_DIR="$RUNTIME_DIR/kernels" +export D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 +export D4R_ZLUDA_WMMA="${D4R_ZLUDA_WMMA:-1}" +export D4R_ZLUDA_WMMA_FP8="${D4R_ZLUDA_WMMA_FP8:-1}" +export D4R_ZLUDA_WMMA_FP8_NATIVE=1 +export D4R_ZLUDA_NATIVE_SWIN_ENCODERS=1 +export D4R_SHIM_VRAM_INTEROP=1 +export D4R_RR_ENABLE=1 +export D4R_RR_PRESET="${D4R_RR_PRESET:-5}" +export D4R_HARNESS_FEATURE_ID=13 +export D4R_HARNESS_SAVE_FRAMES=1 +export WINEDLLOVERRIDES="d3d12=n,b;d3d12core=n,b" + +run_one() { # run_one LABEL OUTPUT_RAW GUIDE_PATH + local label="$1" output="$2" guide_path="$3" + printf '\n== %s: guides on the %s path (%s)\n' "$label" "$guide_path" "$4" >&2 + D4R_SHIM_RR_VRAM_GUIDES="$guide_path" \ + "$ROOT/scripts/run_d3d12_dlss_harness_proton.sh" "$PROTON_DIR" "$output" \ + "$FRAMES" "$IN_W" "$IN_H" "$OUT_W" "$OUT_H" 2>&1 | tee "$OUTPUT_DIR/$label.log" + grep -E 'Ray Reconstruction|GetFeatureRequirements|CreateFeature|output read back|correlation|alpha:|Guide probe' \ + "$OUTPUT_DIR/$label.log" || true +} + +status=0 +for formats in "${FORMAT_SETS[@]}"; do + tag="${formats//,/_}" + printf '\n### guide formats: %s\n' "$formats" + export D4R_HARNESS_GUIDE_FORMATS="$formats" + if [[ "${D4R_RR_ALPHA:-0}" == 1 ]]; then export D4R_HARNESS_RR_ALPHA=1; else unset D4R_HARNESS_RR_ALPHA; fi + if [[ "${D4R_RR_GUIDE_PROBE:-0}" == 1 ]]; then + export D4R_HARNESS_RR_GUIDE_PHASE=1 D4R_HARNESS_RR_GUIDE_PROBE=1 + fi + + host_raw="$OUTPUT_DIR/$tag.host.raw" + gpu_raw="$OUTPUT_DIR/$tag.gpu.raw" + run_one "$tag.host" "$host_raw" 0 "$formats" + run_one "$tag.gpu" "$gpu_raw" 1 "$formats" + + compare=(python3 "$ROOT/tools/compare_rr_output.py" --a "$host_raw" --b "$gpu_raw" + --width "$OUT_W" --height "$OUT_H" --frames "$FRAMES" --same-frames) + [[ "${D4R_RR_ALPHA:-0}" == 1 ]] && compare+=(--alpha --alpha-width $((OUT_W + 8)) --alpha-height $((OUT_H + 4)) + --alpha-base 4 2) + if [[ "${D4R_HARNESS_RR_ALPHA_FORMAT:-r32f}" == r8unorm || + "${D4R_HARNESS_RR_ALPHA_FORMAT:-r32f}" == r16unorm ]]; then + compare+=(--alpha-sentinel 0) + fi + printf '\n== comparing %s host guides against %s GPU guides\n' "$tag" "$tag" + "${compare[@]}" || status=1 + + if [[ "${D4R_RR_GUIDE_PROBE:-0}" == 1 ]]; then + # The probe is a per-run check, so each guide path has to pass it on its own. + for label in "$tag.host" "$tag.gpu"; do + if [[ ! -f "$OUTPUT_DIR/$label.raw.probe.txt" ]]; then + printf 'Guide response probe did not run or did not pass for %s\n' "$label" >&2 + status=1 + else + cat "$OUTPUT_DIR/$label.raw.probe.txt" + fi + done + fi +done + +printf '\n%s\n' "$([[ $status -eq 0 ]] && echo 'All guide path comparisons passed.' || echo 'Guide path comparison FAILED.')" +exit $status \ No newline at end of file diff --git a/tests/test_rr_output.py b/tests/test_rr_output.py new file mode 100644 index 0000000..34c128d --- /dev/null +++ b/tests/test_rr_output.py @@ -0,0 +1,89 @@ +"""Rendered RGB, alpha subrect preservation, and per-frame output comparison.""" +from pathlib import Path +import struct +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +COMPARE = ROOT / "tools" / "compare_rr_output.py" + + +class RayReconstructionOutputTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.a = Path(self.directory.name) / "host.raw" + self.b = Path(self.directory.name) / "gpu.raw" + + def compare(self, width=1, height=1, *extra): + return subprocess.run( + ["python3", str(COMPARE), "--a", str(self.a), "--b", str(self.b), + "--width", str(width), "--height", str(height), *extra], + capture_output=True, text=True) + + def test_channel_swap_with_identical_luma_is_rejected(self): + self.a.write_bytes(struct.pack(" limits["max_mean"]: + failures.append(f"{name}: PSNR {stats['psnr']:.2f} dB below {limits['min_psnr']}, " + f"mean {stats['mean']:.6f} above {limits['max_mean']}") + if stats["nonfinite"]: + failures.append(f"{name}: {stats['nonfinite']} nonfinite samples") + + +def alpha_checks(path, width, height, base_x, base_y, out_width, out_height, failures, sentinel): + """Sentinel outside the subrect, finite values inside, coverage rising with x.""" + values = struct.unpack(f"<{width * height}f", load(path, width, height, 4)) + outside = 0 + invalid = 0 + columns = [0.0] * out_width + counts = [0] * out_width + for y in range(height): + for x in range(width): + value = values[y * width + x] + inside = (base_x <= x < base_x + out_width and base_y <= y < base_y + out_height) + if not inside: + if value != sentinel: + outside += 1 + continue + if not math.isfinite(value) or value < -0.001 or value > 1.001: + invalid += 1 + continue + local = x - base_x + if 0 <= local < out_width: + columns[local] += value + counts[local] += 1 + means = [(columns[i] / counts[i]) if counts[i] else 0.0 for i in range(out_width)] + rising = sum(1 for i in range(1, out_width) if means[i] >= means[i - 1]) + trend = rising / max(1, out_width - 1) + print(f" alpha {path}: outside-subrect writes {outside}, invalid {invalid}, " + f"rising-column fraction {trend:.3f}, first {means[0]:.4f} last {means[-1]:.4f}") + if outside: + failures.append(f"{path}: {outside} pixels outside the output subrect were overwritten") + if invalid: + failures.append(f"{path}: {invalid} alpha values outside [0, 1] or nonfinite") + if out_width > 1 and means[-1] - means[0] < 0.1: + failures.append(f"{path}: alpha output lost the input's left-to-right coverage ramp") + return trend + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--a", required=True, help="first run's output raw") + parser.add_argument("--b", required=True, help="second run's output raw") + parser.add_argument("--width", type=int, required=True) + parser.add_argument("--height", type=int, required=True) + parser.add_argument("--rgba8", action="store_true", help="inputs were 8-bit, not 16-bit float") + parser.add_argument("--frames", type=int, default=0, + help="compare this many .frameN files of both runs") + parser.add_argument("--alpha", action="store_true", help="also compare the .alpha.raw buffers") + parser.add_argument("--alpha-width", type=int, default=0) + parser.add_argument("--alpha-height", type=int, default=0) + parser.add_argument("--alpha-sentinel", type=float, default=-2.0, + help="expected value outside the alpha subrect (0 for UNORM destinations)") + parser.add_argument("--alpha-base", type=int, nargs=2, default=(0, 0), + metavar=("X", "Y")) + parser.add_argument("--min-psnr", type=float, default=45.0) + parser.add_argument("--max-mean", type=float, default=0.002) + parser.add_argument("--same-frames", action="store_true", + help="every saved frame of one run must match the other's") + parser.add_argument("--temporal-change", type=float, default=None, + help="required mean |delta| between consecutive saved frames") + parser.add_argument("--temporal-still", type=float, default=None, + help="upper bound on |delta| between consecutive saved frames of a run " + "whose inputs did not change") + args = parser.parse_args() + + bpp = 4 if args.rgba8 else 8 + limits = {"min_psnr": args.min_psnr, "max_mean": args.max_mean} + failures = [] + + print(f"Comparing {args.a} with {args.b} at {args.width}x{args.height} " + f"({4 if args.rgba8 else 8} bytes per texel)") + first = rgb_plane(args.a, args.width, args.height, bpp) + second = rgb_plane(args.b, args.width, args.height, bpp) + pixels = args.width * args.height * 3 + report("final frame", compare_planes(first, second, pixels), failures, limits, True) + + if args.frames: + for frame in range(1, args.frames + 1): + path_a = f"{args.a}.frame{frame}" + path_b = f"{args.b}.frame{frame}" + if not (os.path.exists(path_a) and os.path.exists(path_b)): + failures.append(f"frame {frame}: missing {path_a if not os.path.exists(path_a) else path_b}") + continue + plane_a = rgb_plane(path_a, args.width, args.height, bpp) + plane_b = rgb_plane(path_b, args.width, args.height, bpp) + report(f"frame {frame}", compare_planes(plane_a, plane_b, pixels), failures, limits, + args.same_frames) + + for run, plane in (("a", first), ("b", second)): + if args.temporal_change is None and args.temporal_still is None: + break + previous = None + for frame in range(1, args.frames + 1): + path = f"{getattr(args, run)}.frame{frame}" + if not os.path.exists(path): + continue + current = rgb_plane(path, args.width, args.height, bpp) + if previous is not None: + stats = compare_planes(previous, current, pixels) + limit = args.temporal_change if args.temporal_change is not None else args.temporal_still + if args.temporal_change is not None and stats["mean"] < limit: + failures.append(f"run {run} frame {frame}: consecutive frames differ by only " + f"{stats['mean']:.6f}, the per-frame change did not reach the image") + if args.temporal_still is not None and stats["mean"] > limit: + failures.append(f"run {run} frame {frame}: consecutive frames of an unchanged " + f"input differ by {stats['mean']:.6f}") + print(f" run {run} frame {frame - 1} -> {frame}: mean |delta| {stats['mean']:.6f}") + previous = current + + if args.alpha: + width = args.alpha_width or args.width + 8 + height = args.alpha_height or args.height + 4 + alpha_a_path, alpha_b_path = args.a + ".alpha.raw", args.b + ".alpha.raw" + missing = [path for path in (alpha_a_path, alpha_b_path) if not os.path.exists(path)] + if missing: + # An alpha run that wrote nothing is a failed comparison, not a reason to stop here. + failures.append("alpha buffers missing: " + ", ".join(missing)) + else: + alpha_failures = [] + trend_a = alpha_checks(alpha_a_path, width, height, args.alpha_base[0], + args.alpha_base[1], args.width, args.height, alpha_failures, args.alpha_sentinel) + trend_b = alpha_checks(alpha_b_path, width, height, args.alpha_base[0], + args.alpha_base[1], args.width, args.height, alpha_failures, args.alpha_sentinel) + alpha_a = struct.unpack(f"<{width * height}f", load(alpha_a_path, width, height, 4)) + alpha_b = struct.unpack(f"<{width * height}f", load(alpha_b_path, width, height, 4)) + stats = compare_planes(alpha_a, alpha_b, len(alpha_a)) + report("alpha", stats, alpha_failures, limits, True) + if trend_a < 0.5 or trend_b < 0.5: + alpha_failures.append("alpha coverage does not follow the input's left-to-right ramp") + failures += alpha_failures + for frame in range(1, args.frames + 1): + paths = (f"{args.a}.alpha{frame}", f"{args.b}.alpha{frame}") + missing = [path for path in paths if not os.path.exists(path)] + if missing: + failures.append(f"alpha frame {frame}: missing " + ", ".join(missing)) + continue + planes = [struct.unpack(f"<{width * height}f", load(path, width, height, 4)) + for path in paths] + report(f"alpha frame {frame}", compare_planes(*planes, width * height), + failures, limits, args.same_frames) + + if failures: + print("\nFAIL") + for failure in failures: + print(f" {failure}") + return 1 + print("\nPASS") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tools/d3d12_dlss_harness.cpp b/tools/d3d12_dlss_harness.cpp index a4a8fe3..3c74c94 100644 --- a/tools/d3d12_dlss_harness.cpp +++ b/tools/d3d12_dlss_harness.cpp @@ -343,6 +343,37 @@ static std::vector read_back(ID3D12Resource* texture, D3D12_RESOURCE_ST return result; } +static UINT alpha_texel_bytes(DXGI_FORMAT format) +{ + return format == DXGI_FORMAT_R32_FLOAT ? 4 : (format == DXGI_FORMAT_R8_UNORM ? 1 : 2); +} + +// Saved alpha stays float32 regardless of the caller's texture format. +static std::vector read_alpha(ID3D12Resource* texture, DXGI_FORMAT format, UINT width, UINT height) +{ + const UINT texel = alpha_texel_bytes(format); + auto raw = read_back(texture, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, width * texel); + if (format == DXGI_FORMAT_R32_FLOAT) + return raw; + const size_t pixels = static_cast(width) * height; + raw.resize(pixels * sizeof(float)); + for (size_t pixel = pixels; pixel-- > 0;) + { + float value; + if (texel == 1) + value = raw[pixel] / 255.0f; + else + { + uint16_t packed; + std::memcpy(&packed, raw.data() + pixel * texel, sizeof(packed)); + value = format == DXGI_FORMAT_R16_FLOAT ? static_cast(std::bit_cast<_Float16>(packed)) + : packed / 65535.0f; + } + std::memcpy(raw.data() + pixel * sizeof(float), &value, sizeof(value)); + } + return raw; +} + static void trace_stage(const char* stage) { const char* path = std::getenv("D4R_HARNESS_TRACE"); @@ -427,6 +458,240 @@ static float pole_scene(float x, float y, UINT outWidth, UINT outHeight) return 0.46f + 0.03f * v; } +// --- Ray Reconstruction guide inputs ------------------------------------------ +// +// A denoiser guide only proves it reached the network when its content changes the +// result, and only proves its layout when a transposed, channel-swapped, +// half-a-row-off or previous frame's copy changes it differently again. The guides +// are therefore filled with deterministic nonuniform patterns that depend on the +// texel's position, its channel, the plane it belongs to and the frame's phase, and +// the harness can perturb exactly one of those per probe evaluation. +struct GuideFormat +{ + DXGI_FORMAT format; + unsigned channels; + unsigned bytes; // per texel + const char* token; +}; + +// The formats a game plausibly passes, one per path the shim treats differently: +// verbatim GPU copies (rgba16f, rgba8unorm, rgba16unorm), Vulkan blits of packed +// or byte-swapped images (bgra8unorm, rgb10a2unorm, rgb11b10float) and the integer +// guides Vulkan cannot blit into floats, which stay on the host (r32uint, r32sint). +static bool guide_format(const char* token, GuideFormat& format) +{ + static const GuideFormat table[] = { + {DXGI_FORMAT_R16G16B16A16_FLOAT, 4, 8, "rgba16f"}, + {DXGI_FORMAT_R32G32B32A32_FLOAT, 4, 16, "rgba32f"}, + {DXGI_FORMAT_R16G16B16A16_UNORM, 4, 8, "rgba16unorm"}, + {DXGI_FORMAT_R8G8B8A8_UNORM, 4, 4, "rgba8unorm"}, + {DXGI_FORMAT_R8G8B8A8_SNORM, 4, 4, "rgba8snorm"}, + {DXGI_FORMAT_B8G8R8A8_UNORM, 4, 4, "bgra8unorm"}, + {DXGI_FORMAT_R10G10B10A2_UNORM, 4, 4, "rgb10a2unorm"}, + {DXGI_FORMAT_R11G11B10_FLOAT, 4, 4, "rgb11b10float"}, + {DXGI_FORMAT_R16G16_FLOAT, 2, 4, "rg16f"}, + {DXGI_FORMAT_R32_FLOAT, 1, 4, "r32f"}, + {DXGI_FORMAT_R32_UINT, 1, 4, "r32uint"}, + {DXGI_FORMAT_R32_SINT, 1, 4, "r32sint"}, + }; + for (const GuideFormat& entry : table) + if (std::strcmp(entry.token, token) == 0) + { + format = entry; + return true; + } + return false; +} + +// One guide texel's value. Plane, channel and phase each shift it, and every plane +// ramps along a different axis with its own checker period, so a guide that arrives +// transposed, from the wrong channel, from another plane or from another frame is a +// visibly different image rather than an indistinguishable one. +static float guide_value(unsigned plane, unsigned channel, UINT x, UINT y, UINT width, UINT height, + unsigned phase) +{ + if (channel == 3) + return 1.0f; + const float u = width > 1 ? static_cast(x) / static_cast(width - 1) : 0.0f; + const float v = height > 1 ? static_cast(y) / static_cast(height - 1) : 0.0f; + const float offset = 0.37f * static_cast(plane) + 0.19f * static_cast(channel) + + 0.5f * static_cast(phase); + const float ramp = (plane % 2u) != 0 ? v : u; + const float wave = 0.22f * std::sin(6.2831853f * (2.0f * u + 3.0f * v + offset)); + const unsigned shift = plane % 4u; + const float checker = ((((x >> shift) ^ (y >> shift)) & 1u) != 0) ? 0.05f : -0.05f; + const float value = 0.45f + 0.25f * ramp + wave + checker; + if (plane == 0) // normals: keep a unit-length, position-dependent normal + { + const float nx = 2.0f * value - 1.0f, ny = 2.0f * (0.45f + 0.25f * v + wave + checker) - 1.0f; + const float nzSquared = std::max(0.0f, 1.0f - nx * nx - ny * ny); + if (channel == 2) + return std::sqrt(nzSquared); + return 0.5f * (nx + 1.0f); + } + return std::clamp(value, 0.0f, 1.0f); +} + +// Packs one value into a small float field: 6 mantissa bits for R11G11B10_FLOAT's +// two 11-bit channels, 5 for its 10-bit channel. Values are clamped into range +// first, so everything this writes the format can represent. +static uint16_t pack_small_float(float value, unsigned mantissaBits) +{ + if (!(value > 0.0f)) + return 0; + constexpr int bias = 15; + const unsigned mantissaScale = 1u << mantissaBits; + const float limit = std::ldexp(1.0f, (1 << 5) - 2 - bias); + value = std::min(value, limit); + int exponent = 0; + std::frexp(value, &exponent); + --exponent; // floor(log2(value)) + const int clamped = std::clamp(exponent + bias, 1, (1 << 5) - 2); + const float scaled = std::ldexp(value, static_cast(mantissaBits) - clamped + bias); + const int mantissa = std::clamp(static_cast(std::lround(scaled)) - static_cast(mantissaScale), 0, + static_cast(mantissaScale) - 1); + return static_cast((static_cast(clamped) << mantissaBits) | + static_cast(mantissa)); +} + +static void encode_guide_texel(const GuideFormat& format, const float* values, uint8_t* destination) +{ + switch (format.format) + { + case DXGI_FORMAT_R16G16B16A16_FLOAT: + case DXGI_FORMAT_R16G16_FLOAT: + for (unsigned channel = 0; channel < format.channels; ++channel) + { + const uint16_t packed = std::bit_cast(static_cast<_Float16>(values[channel])); + std::memcpy(destination + channel * 2, &packed, sizeof(packed)); + } + return; + case DXGI_FORMAT_R32G32B32A32_FLOAT: + case DXGI_FORMAT_R32_FLOAT: + for (unsigned channel = 0; channel < format.channels; ++channel) + std::memcpy(destination + channel * 4, &values[channel], sizeof(float)); + return; + case DXGI_FORMAT_R16G16B16A16_UNORM: + for (unsigned channel = 0; channel < format.channels; ++channel) + { + const uint16_t packed = + static_cast(std::lround(std::clamp(values[channel], 0.0f, 1.0f) * 65535.0f)); + std::memcpy(destination + channel * 2, &packed, sizeof(packed)); + } + return; + case DXGI_FORMAT_R8G8B8A8_SNORM: + for (unsigned channel = 0; channel < format.channels; ++channel) + destination[channel] = static_cast( + static_cast(std::clamp(std::lround(values[channel] * 127.0f), -127L, 127L))); + return; + case DXGI_FORMAT_B8G8R8A8_UNORM: + // Stored B, G, R, A: a verbatim copy would hand the denoiser swapped channels. + for (unsigned channel = 0; channel < format.channels; ++channel) + destination[channel < 3 ? 2 - channel : 3] = + static_cast(std::lround(std::clamp(values[channel], 0.0f, 1.0f) * 255.0f)); + return; + case DXGI_FORMAT_R8G8B8A8_UNORM: + for (unsigned channel = 0; channel < format.channels; ++channel) + destination[channel] = static_cast(std::lround(std::clamp(values[channel], 0.0f, 1.0f) * 255.0f)); + return; + case DXGI_FORMAT_R10G10B10A2_UNORM: { + uint32_t packed = 0; + for (unsigned channel = 0; channel < format.channels; ++channel) + { + const unsigned bits = channel == 3 ? 2u : 10u; + const unsigned maximum = (1u << bits) - 1u; + const unsigned value = + static_cast(std::lround(std::clamp(values[channel], 0.0f, 1.0f) * maximum)); + packed |= (value & maximum) << (channel * 10u); + } + std::memcpy(destination, &packed, sizeof(packed)); + return; + } + case DXGI_FORMAT_R11G11B10_FLOAT: { + const uint32_t packed = static_cast(pack_small_float(values[0], 6)) | + (static_cast(pack_small_float(values[1], 6)) << 11u) | + (static_cast(pack_small_float(values[2], 5)) << 22u); + std::memcpy(destination, &packed, sizeof(packed)); + return; + } + case DXGI_FORMAT_R32_UINT: { + const uint32_t packed = static_cast(std::lround(values[0] * 4096.0f)); + std::memcpy(destination, &packed, sizeof(packed)); + return; + } + case DXGI_FORMAT_R32_SINT: { + const int32_t packed = + static_cast(std::lround((values[0] - 0.5f) * 8192.0f)); // roughly [-4096, 4096] + std::memcpy(destination, &packed, sizeof(packed)); + return; + } + default: + break; + } + std::memset(destination, 0, format.bytes); +} + +// --- comparing rendered frames ------------------------------------------------ +static float luma_of(const uint8_t* pixel, bool rgba8) +{ + float sum = 0.0f; + for (int channel = 0; channel < 3; ++channel) + sum += rgba8 ? static_cast(pixel[channel]) / 255.0f + : static_cast(std::bit_cast<_Float16>( + *reinterpret_cast(pixel + channel * 2))); + return std::clamp(sum / 3.0f, 0.0f, 1.0f); +} + +// True once a guide plane has been uploaded, so later phases update rather than create. +static bool g_uploadedGuides = false; + +static std::string join_guide_formats(const std::array& names) +{ + std::string joined; + for (const std::string& name : names) + joined += joined.empty() ? name : ", " + name; + return joined; +} + +// Writes each guide plane's nonuniform pattern for one phase. With `push` set the content +// also goes to the plane's texture: the first write creates it, later ones are copy updates, +// exactly as a game re-renders its G-buffer every frame. `leftBias` adds a constant to every +// texel of the left half only, which is what the guide response probe needs: a change the +// denoiser can only reproduce if that texel's guide value reached it, in that place. +template +static void fill_guide_planes(std::array& planes, + const std::array& specs, + std::array, Planes>& data, UINT width, UINT height, + unsigned phase, bool push, D3D12_RESOURCE_STATES srv, float leftBias = 0.0f) +{ + for (size_t input = 0; input < Planes; ++input) + { + const GuideFormat& spec = specs[input]; + data[input].assign(static_cast(width) * height * spec.bytes, 0); + for (UINT y = 0; y < height; ++y) + for (UINT x = 0; x < width; ++x) + { + float values[4] = {1.0f, 1.0f, 1.0f, 1.0f}; + for (unsigned channel = 0; channel < spec.channels; ++channel) + values[channel] = std::clamp( + guide_value(static_cast(input), channel, x, y, width, height, phase) + + (leftBias != 0.0f && x < width / 2 ? leftBias : 0.0f), + channel == 3 ? 1.0f : 0.0f, 1.0f); + encode_guide_texel(spec, values, + data[input].data() + + (static_cast(y) * width + x) * spec.bytes); + } + if (!push || planes[input] == nullptr) + continue; + if (g_uploadedGuides) + update_texture(planes[input], data[input].data(), width * spec.bytes, srv); + else + upload(planes[input], data[input].data(), width * spec.bytes, srv); + } + g_uploadedGuides = true; +} + + // D4R_HARNESS_REPLAY_DIR= replays frames captured in a game by the shim // (D4R_SHIM_INPUT_DUMP_DIR + D4R_SHIM_CAPTURE_COUNT): frame-NNNNNN-color.rgba16f, // -depth.r32f, -motion.rg16f and -params.txt, starting at D4R_HARNESS_REPLAY_START. @@ -654,39 +919,73 @@ int main(int argc, char** argv) const char* featureSetting = std::getenv("D4R_HARNESS_FEATURE_ID"); const unsigned int featureId = featureSetting ? static_cast(std::strtoul(featureSetting, nullptr, 0)) : 1; const bool reconstruction = featureId == 13; + // The four G-buffer planes a game registers, one per parameter. D4R_HARNESS_GUIDE_FORMATS + // picks the format each is passed in, one token per plane in registration order + // (GBuffer.Normals, GBuffer.Roughness, GBuffer.DiffuseAlbedo, GBuffer.SpecularAlbedo), + // so a single run can cover the paths the shim treats differently. The default keeps + // the formats games actually pass for these planes. + const char* guideFormatSetting = std::getenv("D4R_HARNESS_GUIDE_FORMATS"); + static const char* const kDefaultGuideFormats[] = {"rgba16f", "rgba16f", "rgba8unorm", "rgba8unorm"}; + std::vector guideTokens; + if (guideFormatSetting != nullptr && guideFormatSetting[0] != '\0') + { + std::string rest = guideFormatSetting; + while (!rest.empty()) + { + const size_t comma = rest.find(','); + guideTokens.push_back(comma == std::string::npos ? rest : rest.substr(0, comma)); + rest = comma == std::string::npos ? std::string() : rest.substr(comma + 1); + } + } + std::array guideSpecs{}; + std::array guideNames{}; + for (size_t input = 0; input < guideSpecs.size(); ++input) + { + guideNames[input] = input < guideTokens.size() ? guideTokens[input] : kDefaultGuideFormats[input]; + if (!guide_format(guideNames[input].c_str(), guideSpecs[input])) + { + std::fprintf(stderr, "Unknown guide format '%s' for guide %zu of 4\n", + guideNames[input].c_str(), input); + return 2; + } + } + // D4R_HARNESS_RR_GUIDE_PHASE=1 rewrites the guides with the next pattern phase on every + // frame, so a stale or one-frame-late guide upload shows up as an output whose guides + // belong to a different frame. + const bool guidePhase = std::getenv("D4R_HARNESS_RR_GUIDE_PHASE") != nullptr; + std::array, 4> guideData{}; std::array guides{}; if (reconstruction) { - // Planar synthetic scene: world-space +Z normals, rough diffuse material. - // Distinct guides exercise denoising inputs rather than aliasing the noisy colour. - std::vector guide(static_cast(resourceWidth) * inHeight * 4); - const std::array, 4> values{{ - {0.0f, 0.0f, 1.0f, 0.0f}, {0.5f, 0.0f, 0.0f, 0.0f}, - {0.7f, 0.4f, 0.2f, 1.0f}, {0.04f, 0.04f, 0.04f, 1.0f}}}; for (size_t input = 0; input < guides.size(); ++input) { - // Cyberpunk supplies UNORM8 albedos, while normals remain floating point. - const bool unorm = input >= 2; - guides[input] = create_texture(resourceWidth, inHeight, - unorm ? DXGI_FORMAT_R8G8B8A8_UNORM : DXGI_FORMAT_R16G16B16A16_FLOAT, + guides[input] = create_texture(resourceWidth, inHeight, guideSpecs[input].format, D3D12_RESOURCE_FLAG_NONE); if (!guides[input]) return 1; - for (size_t pixel = 0; pixel < guide.size() / 4; ++pixel) - for (size_t component = 0; component < 4; ++component) - if (unorm) - reinterpret_cast(guide.data())[pixel * 4 + component] = - static_cast(std::lround(values[input][component] * 255.0f)); - else - guide[pixel * 4 + component] = - std::bit_cast(static_cast<_Float16>(values[input][component])); - upload(guides[input], guide.data(), resourceWidth * (unorm ? 4 : 8), srv); } + fill_guide_planes(guides, guideSpecs, guideData, resourceWidth, inHeight, 0, true, srv); + std::printf("Ray Reconstruction guides: %s%s\n", join_guide_formats(guideNames).c_str(), + guidePhase ? " (rewritten every frame)" : ""); } const bool alphaScenario = reconstruction && std::getenv("D4R_HARNESS_RR_ALPHA") != nullptr; const UINT alphaBaseX = 4, alphaBaseY = 2; const UINT alphaWidth = outWidth + 8, alphaHeight = outHeight + 4; ID3D12Resource* alphaInput = nullptr; ID3D12Resource* alphaOutput = nullptr; + DXGI_FORMAT alphaFormat = DXGI_FORMAT_R32_FLOAT; + if (const char* format = std::getenv("D4R_HARNESS_RR_ALPHA_FORMAT")) + { + if (std::strcmp(format, "r16f") == 0) alphaFormat = DXGI_FORMAT_R16_FLOAT; + else if (std::strcmp(format, "r8unorm") == 0) alphaFormat = DXGI_FORMAT_R8_UNORM; + else if (std::strcmp(format, "r16unorm") == 0) alphaFormat = DXGI_FORMAT_R16_UNORM; + else if (std::strcmp(format, "r32f") != 0) + { + std::fprintf(stderr, "Unknown RR alpha output format: %s\n", format); + return 1; + } + } + const bool alphaUnorm = alphaFormat == DXGI_FORMAT_R8_UNORM || alphaFormat == DXGI_FORMAT_R16_UNORM; + const float alphaSentinel = alphaUnorm ? 0.0f : -2.0f; if (alphaScenario) { // A varying coverage guide and a padded destination expose dropped alpha writes and @@ -697,12 +996,24 @@ int main(int argc, char** argv) coverage[static_cast(y) * resourceWidth + x] = 0.25f + 0.5f * static_cast(x) / std::max(1u, resourceWidth - 1); alphaInput = create_texture(resourceWidth, inHeight, DXGI_FORMAT_R32_FLOAT, D3D12_RESOURCE_FLAG_NONE); - alphaOutput = create_texture(alphaWidth, alphaHeight, DXGI_FORMAT_R32_FLOAT, + alphaOutput = create_texture(alphaWidth, alphaHeight, alphaFormat, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS); if (!alphaInput || !alphaOutput) return 1; upload(alphaInput, coverage.data(), resourceWidth * sizeof(float), srv); - std::vector sentinel(static_cast(alphaWidth) * alphaHeight, -2.0f); - upload(alphaOutput, sentinel.data(), alphaWidth * sizeof(float), D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + const UINT texel = alpha_texel_bytes(alphaFormat); + std::vector sentinel(static_cast(alphaWidth) * alphaHeight * texel, 0); + if (!alphaUnorm) + for (size_t pixel = 0; pixel < static_cast(alphaWidth) * alphaHeight; ++pixel) + { + if (texel == 4) + std::memcpy(sentinel.data() + pixel * texel, &alphaSentinel, sizeof(alphaSentinel)); + else + { + const uint16_t packed = std::bit_cast(static_cast<_Float16>(alphaSentinel)); + std::memcpy(sentinel.data() + pixel * texel, &packed, sizeof(packed)); + } + } + upload(alphaOutput, sentinel.data(), alphaWidth * texel, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); } ID3D12Resource* colorTexture = create_texture(resourceWidth, inHeight, rgba8 ? DXGI_FORMAT_R8G8B8A8_UNORM : DXGI_FORMAT_R16G16B16A16_FLOAT, D3D12_RESOURCE_FLAG_NONE); @@ -965,6 +1276,13 @@ int main(int argc, char** argv) outWidth * (rgba8 || rgb10 ? 4 : 8), D3D12_RESOURCE_STATE_UNORDERED_ACCESS); d4r_ngx_set_d3d12_resource(parameters, "Output", outputTexture); } + if (reconstruction && guidePhase && frame > 1) + { + // The guides this frame are a different pattern, so the output can only be right + // if this frame's own guides were the ones the denoiser read. + fill_guide_planes(guides, guideSpecs, guideData, resourceWidth, inHeight, + static_cast(frame), true, srv); + } if (qualityScene) { // Halton(2,3) jitter in render pixels, [-0.5, 0.5); a render pixel @@ -1228,6 +1546,18 @@ int main(int argc, char** argv) std::fclose(frameFile); } } + // The alpha the denoiser produced for this frame, so a host and a GPU guide run can be + // compared frame by frame and not only on the last one. + if (saveFrames && alphaScenario) + { + const std::vector alphaFrame = read_alpha(alphaOutput, alphaFormat, alphaWidth, alphaHeight); + const std::string alphaPath = std::string(argv[2]) + ".alpha" + std::to_string(frame); + if (FILE* alphaFile = std::fopen(alphaPath.c_str(), "wb")) + { + std::fwrite(alphaFrame.data(), 1, alphaFrame.size(), alphaFile); + std::fclose(alphaFile); + } + } } const std::vector output = read_back(outputTexture, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, @@ -1242,6 +1572,82 @@ int main(int argc, char** argv) for (uint8_t value : output) nonzero += value != 0; std::printf("output read back: %zu of %zu bytes nonzero, written to %s\n", nonzero, output.size(), argv[2]); + // Two reset evaluations with identical colour and depth, perturbing only the + // left-half guides, check that guide changes affect the rendered scene. + // Transformer attention can spread the response beyond the changed half; + // exact plane identity and layout are checked separately by VRAM verification. + if (reconstruction && std::getenv("D4R_HARNESS_RR_GUIDE_PROBE") != nullptr) + { + auto evaluate_reset_once = [&]() { + d4r_ngx_set_int(parameters, "Reset", 1); + const NgxResult evaluated = evaluate(g_list, feature, parameters, nullptr); + submit_and_wait(); + Sleep(frameWaitMs); + return evaluated; + }; + fill_guide_planes(guides, guideSpecs, guideData, resourceWidth, inHeight, 0, true, srv); + if (evaluate_reset_once() != NGX_SUCCESS) + { + std::fprintf(stderr, "Guide probe: the baseline evaluation was refused\n"); + return 1; + } + const std::vector baseline = + read_back(outputTexture, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, outWidth * (rgba8 ? 4 : 8)); + fill_guide_planes(guides, guideSpecs, guideData, resourceWidth, inHeight, 0, true, srv, 0.4f); + if (evaluate_reset_once() != NGX_SUCCESS) + { + std::fprintf(stderr, "Guide probe: the left-half evaluation was refused\n"); + return 1; + } + const std::vector shifted = + read_back(outputTexture, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, outWidth * (rgba8 ? 4 : 8)); + const size_t texel = rgba8 ? 4u : 8u; + double left = 0.0, right = 0.0; + size_t leftCount = 0, rightCount = 0, nonfinite = 0; + for (UINT y = 0; y < outHeight; ++y) + for (UINT x = 0; x < outWidth; ++x) + { + const size_t pixel = (static_cast(y) * outWidth + x) * texel; + const float first = luma_of(baseline.data() + pixel, rgba8); + const float second = luma_of(shifted.data() + pixel, rgba8); + nonfinite += !std::isfinite(first) || !std::isfinite(second); + const double delta = std::fabs(static_cast(first) - static_cast(second)); + if (x < outWidth / 2) + { + left += delta; + ++leftCount; + } + else + { + right += delta; + ++rightCount; + } + } + const double leftMean = leftCount ? left / leftCount : 0.0; + const double rightMean = rightCount ? right / rightCount : 0.0; + auto report_probe = [&](FILE* report) { + std::fprintf(report, "Guide probe: left-half mean |delta| %.6f, right-half %.6f, ratio %.4f, " + "nonfinite %zu\n", + leftMean, rightMean, leftMean > 0.0 ? rightMean / leftMean : 0.0, nonfinite); + }; + report_probe(stdout); + const std::string probePath = std::string(argv[2]) + ".probe.txt"; + if (FILE* report = std::fopen(probePath.c_str(), "w")) + { + report_probe(report); + std::fclose(report); + } + // Require a visible response, not locality: transformer attention can + // legitimately change both halves even when only one guide half changes. + if (!(leftMean + rightMean > 0.002) || nonfinite != 0) + { + std::fprintf(stderr, + "Ray Reconstruction guide changes did not affect the rendered image: " + "left-half mean |delta| %.6f, right-half %.6f\n", + leftMean, rightMean); + return 1; + } + } // The shim queues each evaluation on its own worker, so a failure inside the denoiser is // invisible in this thread's EvaluateFeature return value: the frame is accepted and the // output texture is simply never written. The image is the only observable. On the static @@ -1314,8 +1720,7 @@ int main(int argc, char** argv) } if (alphaScenario) { - const std::vector raw = - read_back(alphaOutput, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, alphaWidth * sizeof(float)); + const std::vector raw = read_alpha(alphaOutput, alphaFormat, alphaWidth, alphaHeight); const float* values = reinterpret_cast(raw.data()); size_t invalid = 0, overwritten = 0, leftCount = 0, rightCount = 0; double left = 0, right = 0; @@ -1327,7 +1732,7 @@ int main(int argc, char** argv) y >= alphaBaseY && y < alphaBaseY + outHeight; if (!inside) { - overwritten += value != -2.0f; + overwritten += value != alphaSentinel; continue; } invalid += !std::isfinite(value) || value < -0.001f || value > 1.001f; diff --git a/tools/d4r_nvngx_shim.cpp b/tools/d4r_nvngx_shim.cpp index cf6e92c..9fe8b6b 100644 --- a/tools/d4r_nvngx_shim.cpp +++ b/tools/d4r_nvngx_shim.cpp @@ -7,14 +7,11 @@ // // A D3D12 EvaluateFeature call only records into the game's still-open command // list, while DLSS runs on a separate CUDA context. Each evaluation therefore: -// 1. records copies of the inputs into readback buffers plus a GPU-written -// frame marker (WriteBufferImmediate) on the game's command list, -// 2. records a copy of the most recent finished DLSS result from an upload -// buffer into the output texture, -// 3. queues a CUDA job that waits for the marker, uploads the inputs into -// CUDA arrays, runs the official DLSS evaluation, and stores the result -// into an upload buffer for a later frame. -// The output is therefore one or more frames behind the game's other passes. +// 1. records input copies into shared VRAM buffers (host readback fallback) +// and a GPU-written frame marker on the game's command list, +// 2. queues CUDA work after that marker, evaluates NGX, and writes a result slot, +// 3. copies that slot into the output texture. Split command lists present +// the current frame; without splitting the latest finished frame is used. // // Environment: // D4R_NGX_CORE Windows path of the official _nvngx.dll (required) @@ -52,6 +49,7 @@ #define WIDL_EXPLICIT_AGGREGATE_RETURNS #include #include +#include #include #include "d4r_motion_dilation.h" @@ -1700,6 +1698,17 @@ static NgxResult ensure_workers_started() return NGX_SUCCESS; } +static void set_cuda_adapter_luid(const LUID& luid) +{ + char low[24], high[24]; + std::snprintf(low, sizeof(low), "0x%08lx", static_cast(luid.LowPart)); + std::snprintf(high, sizeof(high), "0x%08lx", static_cast(luid.HighPart)); + SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", low); + SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", high); + SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", "1"); + logf("CUDA device LUID set to the D3D12 adapter LUID %s:%s", high, low); +} + static NgxResult initialize(unsigned long long applicationId, const wchar_t* dataPath, ID3D12Device* device, const NgxFeatureCommonInfo* featureInfo, unsigned int sdkVersion, const ProjectIdentity* project = nullptr) @@ -1731,13 +1740,7 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat // the nvcuda bridge reports this LUID from cuDeviceGetLuid. LUID luid; device->GetAdapterLuid(&luid); - char low[24], high[24]; - std::snprintf(low, sizeof(low), "0x%08lx", static_cast(luid.LowPart)); - std::snprintf(high, sizeof(high), "0x%08lx", static_cast(luid.HighPart)); - SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", low); - SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", high); - SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", "1"); - logf("CUDA device LUID set to the D3D12 adapter LUID %s:%s", high, low); + set_cuda_adapter_luid(luid); } g.paths.clear(); @@ -1982,7 +1985,7 @@ static bool aux_array_layout(DXGI_FORMAT format, uint32_t& arrayFormat, uint32_t {DXGI_FORMAT_R32G32B32A32_UINT, 4}, {DXGI_FORMAT_B8G8R8A8_UNORM, 4}, {DXGI_FORMAT_R10G10B10A2_UNORM, 4}, - {DXGI_FORMAT_R11G11B10_FLOAT, 3}, + {DXGI_FORMAT_R11G11B10_FLOAT, 4}, }; for (const Converted& entry : converted) if (entry.format == format) @@ -2096,9 +2099,10 @@ static void convert_aux_row(DXGI_FORMAT format, const uint8_t* source, float* de { uint32_t packed; std::memcpy(&packed, source + x * 4, sizeof(packed)); - destination[x * 3] = small_float_to_float(packed & 0x7FF, 6); - destination[x * 3 + 1] = small_float_to_float((packed >> 11) & 0x7FF, 6); - destination[x * 3 + 2] = small_float_to_float((packed >> 22) & 0x3FF, 5); + destination[x * 4] = small_float_to_float(packed & 0x7FF, 6); + destination[x * 4 + 1] = small_float_to_float((packed >> 11) & 0x7FF, 6); + destination[x * 4 + 2] = small_float_to_float((packed >> 22) & 0x3FF, 5); + destination[x * 4 + 3] = 1.0f; // missing alpha follows shader texture reads } return; default: @@ -2321,7 +2325,10 @@ struct AuxSurface CudaImage image; // worker only CudaObject handle = 0; // what the parameters point at; its address must stay put Staging staging[kSlots]; - HostPlane host[kSlots]; // unpacked floats, only for a converted surface (worker only) + HostPlane host[kSlots]; // stable upload bytes for host fallback (worker only) + VramBuffer vram[kSlots]; // per-frame guide bytes; imported lazily on the CUDA worker + VramImage conversion; // packed/BGRA guides converted by Vulkan, not the CPU + bool gpu[kSlots] = {}; }; // What one frame's slot bound: which surface answered which denoiser name, for that frame @@ -2357,6 +2364,12 @@ struct InputSlot DXGI_FORMAT alphaFormat = DXGI_FORMAT_UNKNOWN; uint32_t alphaWriteback = 0; bool alphaRequested = false; + // GPU-resident alpha (VRAM interop): the caller's image and whether this frame's result can be + // blitted into it, so the float32 result never has to come back through host memory. Set with + // the rest of the alpha capture, on the game thread, before the frame is queued. + VkImage alphaImageHandle = VK_NULL_HANDLE; + bool alphaConvert = false; + bool alphaGpu = false; std::atomic busy{false}; // readback staging owned by the pipeline HostPlane host[4]; std::atomic hostBusy{false}; // host planes not yet uploaded by the worker @@ -2370,6 +2383,9 @@ struct InputSlot struct AlphaSlot { Staging staging; // upload buffer holding one result + VramBuffer vram; // VRAM interop: the evaluated region as float32 + VramImage conversion; // the R32F image that result is blitted from + bool gpu = false; // filled in vram, presented from it; else staging std::atomic lastReadFrame{0}; // reserved/last-read frame, as OutputSlot's std::atomic producedFrame{0}; // frame that filled it DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN; @@ -3095,7 +3111,7 @@ static bool take_pooled_vram_buffer(VramBuffer& target, size_t bytes, uint8_t ki } // Game thread. The CUDA import runs on the worker, which owns the context. -static bool create_vram_buffer(VramBuffer& target, size_t bytes) +static bool create_vram_buffer(VramBuffer& target, size_t bytes, bool importNow = true) { if (take_pooled_vram_buffer(target, bytes)) return true; @@ -3124,8 +3140,8 @@ static bool create_vram_buffer(VramBuffer& target, size_t bytes) result = g_vk.allocate(g_vk.device, &allocation, nullptr, &buffer.memory); if (result == VK_SUCCESS) result = g_vk.bindBuffer(g_vk.device, buffer.buffer, buffer.memory, 0); - int imported = -1; - if (result == VK_SUCCESS) + int imported = 0; + if (result == VK_SUCCESS && importNow) imported = g.worker.call([&] { return g_vk.import(g_vk.device, reinterpret_cast(buffer.memory), requirements.size, &buffer.device, &buffer.external); @@ -3685,6 +3701,11 @@ static void release_vram(Feature& feature) recycle_vram_buffer(buffer); for (OutputSlot& slot : feature.outputs) recycle_vram_buffer(slot.vram); + for (AlphaSlot& slot : feature.alpha) + { + recycle_vram_buffer(slot.vram); + destroy_vram_image(slot.conversion); + } for (VramBuffer& buffer : feature.retiredBuffers) recycle_vram_buffer(buffer); feature.retiredBuffers.clear(); @@ -3699,20 +3720,22 @@ static void release_vram(Feature& feature) } // Prep stage: converts one readback staging buffer into the canonical layout. -static void stage_plane(Plane plane, const Staging& staging, HostPlane& host, FrameTiming* timing, int timingIndex) +static void stage_plane(Plane plane, const Staging& staging, HostPlane& host, FrameTiming* timing, int timingIndex, + size_t rowPitch) { const size_t texelBytes = (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane); - reserve_host(host, texelBytes * staging.width * staging.height); + const size_t packedRowBytes = texelBytes * staging.width; + host.rowBytes = std::max(rowPitch, packedRowBytes); + reserve_host(host, host.rowBytes * staging.height); host.width = staging.width; host.height = staging.height; - host.rowBytes = texelBytes * staging.width; uint8_t* rows = host.bytes; const size_t rowBytes = host.rowBytes; const uint8_t* source = staging.mapped + staging.layout.Offset; const size_t sourcePitch = staging.layout.Footprint.RowPitch; const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{}; if (canonical_input(plane, staging.format)) - copy_rows(g_inputRows, rows, rowBytes, source, sourcePitch, rowBytes, staging.height); + copy_rows(g_inputRows, rows, rowBytes, source, sourcePitch, packedRowBytes, staging.height); else g_inputRows.run(staging.height, rowBytes * staging.height, [&](UINT begin, UINT end) { for (UINT y = begin; y < end; ++y) @@ -4274,6 +4297,169 @@ static void write_alpha_texel(uint32_t writeback, uint8_t* destination, float va } } +// --- GPU-resident alpha ----------------------------------------------------------- +// +// Separate alpha no longer forces RGB through host memory. R32F is copied +// verbatim through an imported result buffer; half/UNORM destinations use a +// Vulkan blit. Integer destinations retain the existing host conversion. +// Blit rounding is hardware-dependent: gfx1201 rounds half toward zero, and +// half/UNORM output can differ from the CPU quantizer by one destination step. + +// The destination image's Vulkan format, and whether reaching it from the float32 result needs a +// conversion at all (an R32F destination is copied verbatim). +static bool alpha_vram_format(DXGI_FORMAT format, VkFormat& vulkan, bool& convert) +{ + switch (format) + { + case DXGI_FORMAT_R32_FLOAT: + vulkan = VK_FORMAT_R32_SFLOAT; + convert = false; + return true; + case DXGI_FORMAT_R16_FLOAT: + vulkan = VK_FORMAT_R16_SFLOAT; + convert = true; + return true; + case DXGI_FORMAT_R16_UNORM: + vulkan = VK_FORMAT_R16_UNORM; + convert = true; + return true; + case DXGI_FORMAT_R8_UNORM: + vulkan = VK_FORMAT_R8_UNORM; + convert = true; + return true; + // R8_UINT and R16_UINT deliberately do not qualify. Vulkan will not blit a float image into an + // integer one, and the host conversion of those is not a value conversion at all: it truncates + // the float toward zero (write_alpha_texel, cases 2 and 5), so an alpha of 3.9 must land on 3, + // not on the 0 a normalized blit would give. Only the host route can express that. + default: + return false; + } +} + +// The R32F image a converted destination is blitted from is an ordinary conversion image, shared +// with the rest of the file only in kind: one per alpha result slot, since a recorded copy reads +// it a frame or more later. + +// Game thread. Decides whether this frame's alpha result stays on the GPU: the colour output's +// VRAM interop has to be on, the caller's texture has to support the conversion, +// and every result slot needs a buffer (and, for a converted format, an R32F image) +// big enough for the evaluated region. Anything short of that uses the host route, +// which needs none of it, so a caller with an exotic alpha format still gets its alpha. +static void prepare_alpha_vram(Feature& feature, InputSlot& slot, bool vram) +{ + slot.alphaImageHandle = VK_NULL_HANDLE; + slot.alphaConvert = false; + slot.alphaGpu = false; + if (!slot.alphaRequested || !vram || g_vk.interop == nullptr) + return; + VkFormat format = VK_FORMAT_UNDEFINED; + bool convert = false; + if (!alpha_vram_format(slot.alphaFormat, format, convert) || + !vram_blit_supported(VK_FORMAT_R32_SFLOAT, format)) + return; + UINT64 handle = 0, offset = 0; + VkFormat actual = VK_FORMAT_UNDEFINED; + if (FAILED(g_vk.interop->GetVulkanResourceInfo1(slot.alphaResource, &handle, &offset, &actual)) || + handle == 0 || actual != format) + return; + // Only the evaluated region travels, not the whole extent the denoiser stores into: the result + // is read back as the caller's rectangle and blitted into the caller's rectangle. + const size_t bytes = sizeof(float) * static_cast(slot.alphaExtentW) * slot.alphaExtentH; + for (AlphaSlot& alpha : feature.alpha) + { + if (!ensure_vram_buffer(feature, alpha.vram, bytes)) + { + logf("DLSSD.OutputAlpha: no VRAM result buffer of %zu bytes; this frame's alpha is read back " + "through host memory", + bytes); + return; + } + if (convert && + !ensure_conversion_image(feature, alpha.conversion, slot.alphaExtentW, slot.alphaExtentH, + VK_FORMAT_R32_SFLOAT)) + { + logf("DLSSD.OutputAlpha: no R32F conversion image of %ux%u; this frame's alpha is read back " + "through host memory", + slot.alphaExtentW, slot.alphaExtentH); + return; + } + } + slot.alphaImageHandle = reinterpret_cast(handle); + slot.alphaConvert = convert; + slot.alphaGpu = true; +} + +// One frame's evaluated alpha rectangle: where the caller's subrect starts and how much of it +// the denoiser filled. Taken from a published result when a later frame presents it, and from the +// current frame's own capture when a split frame presents the result the worker is about to make. +struct AlphaRegion +{ + UINT baseX = 0, baseY = 0; + UINT width = 0, height = 0; +}; + +// Records the copy of a GPU-resident alpha result into the caller's texture, which is in COPY_DEST +// state already. The result buffer holds the evaluated region as float32, so an R32F destination is +// copied verbatim at the caller's subrect origin and a converted one is blitted there. +static bool record_alpha_vram(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, VkImage image, + const VramBuffer& buffer, const VramImage& conversion, const AlphaRegion& region, + bool convert) +{ + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd))) + return false; + VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL; + g_vk.interop->GetVulkanImageLayout(resource, D3D12_RESOURCE_STATE_COPY_DEST, &layout); + VkBufferImageCopy regionCopy = {}; + regionCopy.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + regionCopy.imageSubresource.layerCount = 1; + regionCopy.imageOffset = {static_cast(region.baseX), static_cast(region.baseY), 0}; + regionCopy.imageExtent = {region.width, region.height, 1}; + g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0, + nullptr, 0, nullptr); + if (!convert) + g_vk.copyBufferToImage(cmd, buffer.buffer, image, layout, 1, ®ionCopy); + else + { + VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toDestination.image = conversion.image; + toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, + &toDestination); + regionCopy.imageOffset = {0, 0, 0}; + g_vk.copyBufferToImage(cmd, buffer.buffer, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, + ®ionCopy); + VkImageMemoryBarrier toSource = toDestination; + toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, + &toSource); + // Source offsets are the region's own origin, destination offsets the caller's subrect, so + // every texel of the caller's texture outside the rectangle is left as it was. + VkImageBlit blit = {}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.dstSubresource = blit.srcSubresource; + blit.srcOffsets[0] = {0, 0, 0}; + blit.srcOffsets[1] = {static_cast(region.width), static_cast(region.height), 1}; + blit.dstOffsets[0] = {static_cast(region.baseX), static_cast(region.baseY), 0}; + blit.dstOffsets[1] = {static_cast(region.baseX + region.width), + static_cast(region.baseY + region.height), 1}; + g_vk.blit(cmd, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, image, layout, 1, &blit, + VK_FILTER_NEAREST); + } + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0, + nullptr, 0, nullptr); + return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list)); +} + // Game thread. Records which DLSSD.OutputAlpha texture this frame wants written, and holds a // reference on it until the frame retires. Refuses loudly a texture this file cannot fill: // accepting the frame and leaving the caller's surface untouched is exactly the silent wrong @@ -4399,62 +4585,175 @@ static int claim_alpha_slot(Feature& feature) } } -// Worker. Reads back the surface the denoiser just wrote and converts the evaluated region into -// the claimed slot's upload buffer. A failure fails the frame: an accepted frame with a missing -// alpha would leave the caller believing it was written. -static bool download_alpha_output(Feature& feature, uint32_t frame) +// Defined with the worker's other synchronization helpers, below; the alpha read-back needs it. +static int synchronize_default_stream(Feature& feature, bool* querySleep = nullptr, double* waitEmaUs = nullptr); + +// D4R_SHIM_EVAL_SYNC: whether the worker waits for NGX's kernels before it reads a result back. +// With VRAM interop the wait is deferred to publish_vram, which releases the colour result only +// after it - so anything that reads the arrays earlier, such as the alpha copy below, has to +// bring the wait forward for itself. +static bool eval_sync_enabled() +{ + static const bool sync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0; + return sync; +} + +// Worker. Reads the surface the denoiser just wrote and puts the evaluated region in the claimed +// slot: straight into that slot's imported buffer when the game thread qualified the GPU route, +// otherwise into an upload buffer as converted texels. A failure fails the frame: an accepted +// frame with a missing alpha would leave the caller believing it was written. +static bool download_alpha_output(Feature& feature, uint32_t frame, const FrameParams& params) { const InputSlot& slot = feature.inputs[frame % kSlots]; if (!slot.alphaRequested) return true; if (feature.alphaImage.array == nullptr) return false; - const int index = claim_alpha_slot(feature); + // A split frame presents its own result in the second half of its own command list, so it takes + // the slot that frame's colour takes rather than claiming one: a claim would be free to pick a + // slot whose reader - frame N's own second half - the GPU marker has passed but whose copy has + // not run yet, and overwrite it underneath that copy. Frames in flight are capped below + // kOutputSlots, so frame N's slot is not reachable again until its second half has run. + const int index = params.split ? static_cast(frame % kOutputSlots) : claim_alpha_slot(feature); if (index < 0) { logf("frame %u: no free DLSSD.OutputAlpha result slot; the evaluation is dropped", frame); return false; } AlphaSlot& target = feature.alpha[index]; - if (!ensure_staging(target.staging, slot.alphaResource, D3D12_HEAP_TYPE_UPLOAD)) + const size_t regionBytes = sizeof(float) * static_cast(slot.alphaExtentW) * slot.alphaExtentH; + // Only the evaluated region moves, from either route: the denoiser stores into the whole + // extent its surface was created over, and the rest of the caller's texture is not its to + // touch. + // `params.vram` is the worker's copy of this frame's decision, because the colour output's + // interop can be cut back to host staging between the capture and the evaluation. + // With VRAM interop run_evaluation skipped its own wait (publish_vram takes it before releasing + // the colour result), but the alpha is read out of that array here, on either route: the GPU + // one copies it, the host one converts the bytes the moment they land. Bring the wait forward + // or the denoiser's alpha would be read while it is still being written. + if (params.vram && !eval_sync_enabled()) { - logf("frame %u: no upload buffer for the DLSSD.OutputAlpha result; the evaluation is dropped", frame); - return false; + const int synced = synchronize_default_stream(feature, nullptr); + if (synced != 0) + { + logf("frame %u: DLSSD.OutputAlpha copy synchronize failed: %d; the evaluation is dropped", frame, + synced); + return false; + } } - reserve_host(feature.alphaHost, sizeof(float) * slot.alphaWidth * slot.alphaHeight); - if (feature.alphaHost.bytes == nullptr || - feature.alphaHost.capacity < sizeof(float) * slot.alphaWidth * slot.alphaHeight) + if (slot.alphaGpu && params.vram && target.vram.device != 0 && target.vram.bytes >= regionBytes) { - logf("frame %u: no room to read the DLSSD.OutputAlpha surface back; the evaluation is dropped", frame); - return false; + CudaMemcpy2D copy = {}; + // The source origin is a byte offset: the surface is float32 single channel, so the + // caller's subrect origin in pixels is that many bytes into each row. The result buffer + // holds the region alone, which is why the destination offsets stay at zero. + copy.srcXInBytes = slot.alphaBaseX * sizeof(float); + copy.srcY = slot.alphaBaseY; + copy.srcMemoryType = CUDA_MEMORY_ARRAY; + copy.srcArray = feature.alphaImage.array; + copy.srcPitch = sizeof(float) * slot.alphaWidth; + copy.dstMemoryType = CUDA_MEMORY_DEVICE; + copy.dstDevice = target.vram.device; + copy.dstPitch = sizeof(float) * slot.alphaExtentW; + copy.WidthInBytes = copy.dstPitch; + copy.Height = slot.alphaExtentH; + // Queued on the null stream, where the colour result's copy is queued too, and picked up by + // the Vulkan transfer that blits this buffer into the caller's texture. + if (copy_2d(copy) != 0) + { + logf("frame %u: copying the DLSSD.OutputAlpha result into VRAM failed; the evaluation is dropped", + frame); + return false; + } + if (env_uint("D4R_SHIM_ALPHA_VERIFY", 0) != 0) + { + // The region as the denoiser left it in its array, against the buffer the Vulkan blit + // reads it from: the two must be the same float32 the host route would have converted. + std::vector fromArray(regionBytes), fromBuffer(regionBytes); + CudaMemcpy2D readArray = copy; + readArray.dstMemoryType = CUDA_MEMORY_HOST; + readArray.dstHost = fromArray.data(); + CudaMemcpy2D readBuffer = {}; + readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE; + readBuffer.srcDevice = target.vram.device; + readBuffer.srcPitch = copy.dstPitch; + readBuffer.dstMemoryType = CUDA_MEMORY_HOST; + readBuffer.dstHost = fromBuffer.data(); + readBuffer.dstPitch = copy.dstPitch; + readBuffer.WidthInBytes = copy.WidthInBytes; + readBuffer.Height = copy.Height; + if (g.cu.memcpy2D(&readArray) == 0 && g.cu.memcpy2D(&readBuffer) == 0) + { + size_t differing = 0; + for (size_t byte = 0; byte < fromArray.size(); ++byte) + differing += fromArray[byte] != fromBuffer[byte]; + if (differing != 0 || frame <= 3 || frame % 120 == 0) + logf("frame %u VRAM alpha verify: %zu of %zu bytes differ between the result array and " + "the buffer", + frame, differing, fromArray.size()); + if (differing != 0) + return false; + char dumpPath[MAX_PATH], alphaPath[MAX_PATH]; + if (output_dump_path(frame, dumpPath)) + { + const int written = std::snprintf(alphaPath, sizeof(alphaPath), "%s.alpha.f32", dumpPath); + if (written > 0 && written < static_cast(sizeof(alphaPath))) + write_capture_file(frame, "raw DLSS alpha region", alphaPath, fromArray.data(), + fromArray.size()); + } + } + else + { + logf("frame %u: reading DLSSD.OutputAlpha verification data failed", frame); + return false; + } + } + target.gpu = true; } - CudaMemcpy2D copy = {}; - copy.srcMemoryType = CUDA_MEMORY_ARRAY; - copy.srcArray = feature.alphaImage.array; - copy.dstMemoryType = CUDA_MEMORY_HOST; - copy.dstHost = feature.alphaHost.bytes; - copy.dstPitch = sizeof(float) * slot.alphaWidth; - copy.WidthInBytes = sizeof(float) * slot.alphaWidth; - copy.Height = slot.alphaHeight; - // Conversion reads the host bytes immediately, so this transfer must complete on return. - if (g.cu.memcpy2D(©) != 0) + else { - logf("frame %u: reading the DLSSD.OutputAlpha surface back failed; the evaluation is dropped", frame); - return false; + target.gpu = false; + if (!ensure_staging(target.staging, slot.alphaResource, D3D12_HEAP_TYPE_UPLOAD)) + { + logf("frame %u: no upload buffer for the DLSSD.OutputAlpha result; the evaluation is dropped", frame); + return false; + } + reserve_host(feature.alphaHost, sizeof(float) * slot.alphaWidth * slot.alphaHeight); + if (feature.alphaHost.bytes == nullptr || + feature.alphaHost.capacity < sizeof(float) * slot.alphaWidth * slot.alphaHeight) + { + logf("frame %u: no room to read the DLSSD.OutputAlpha surface back; the evaluation is dropped", frame); + return false; + } + CudaMemcpy2D copy = {}; + copy.srcMemoryType = CUDA_MEMORY_ARRAY; + copy.srcArray = feature.alphaImage.array; + copy.dstMemoryType = CUDA_MEMORY_HOST; + copy.dstHost = feature.alphaHost.bytes; + copy.dstPitch = sizeof(float) * slot.alphaWidth; + copy.WidthInBytes = sizeof(float) * slot.alphaWidth; + copy.Height = slot.alphaHeight; + // Conversion reads the host bytes immediately, so this transfer must complete on return. + if (g.cu.memcpy2D(©) != 0) + { + logf("frame %u: reading the DLSSD.OutputAlpha surface back failed; the evaluation is dropped", frame); + return false; + } + // Only the evaluated rectangle is written, at its position in the buffer. The footprint + // offset from GetCopyableFootprints is left alone - it must stay 512-byte aligned - and the + // region is instead selected by the source box of the copy below. + const float* const source = reinterpret_cast(feature.alphaHost.bytes); + uint8_t* const destination = target.staging.mapped + target.staging.layout.Offset; + const size_t rowPitch = target.staging.layout.Footprint.RowPitch; + const size_t texel = alpha_texel_bytes(slot.alphaWriteback); + for (UINT row = 0; row < slot.alphaExtentH; ++row) + for (UINT x = 0; x < slot.alphaExtentW; ++x) + write_alpha_texel( + slot.alphaWriteback, + destination + static_cast(slot.alphaBaseY + row) * rowPitch + + static_cast(slot.alphaBaseX) * texel + x * texel, + source[static_cast(slot.alphaBaseY + row) * slot.alphaWidth + slot.alphaBaseX + x]); } - // Only the evaluated rectangle is written, at its position in the buffer. The footprint - // offset from GetCopyableFootprints is left alone - it must stay 512-byte aligned - and the - // region is instead selected by the source box of the copy below. - const float* const source = reinterpret_cast(feature.alphaHost.bytes); - uint8_t* const destination = target.staging.mapped + target.staging.layout.Offset; - const size_t rowPitch = target.staging.layout.Footprint.RowPitch; - const size_t texel = alpha_texel_bytes(slot.alphaWriteback); - for (UINT row = 0; row < slot.alphaExtentH; ++row) - for (UINT x = 0; x < slot.alphaExtentW; ++x) - write_alpha_texel(slot.alphaWriteback, - destination + static_cast(slot.alphaBaseY + row) * rowPitch + - static_cast(slot.alphaBaseX) * texel + x * texel, - source[static_cast(slot.alphaBaseY + row) * slot.alphaWidth + slot.alphaBaseX + x]); { // Published under the same lock the claim uses, exactly as the colour output publishes. std::lock_guard lock(feature.outputMutex); @@ -4471,67 +4770,124 @@ static bool download_alpha_output(Feature& feature, uint32_t frame) return true; } -// Game thread. Presents the newest finished alpha result into the caller's texture. The slot's -// lastReadFrame is stamped before the copy is recorded and producedFrame is NOT cleared: the -// colour output's claim relies on both, and clearing here would let the worker overwrite a -// result whose command list has not run yet. +// Game thread. Presents the alpha result that goes with the colour result about to be presented. +// The slot's lastReadFrame is stamped before the copy is recorded and producedFrame is NOT +// cleared: the colour output's claim relies on both, and clearing here would let the worker +// overwrite a result whose command list has not run yet. static bool record_alpha_copyback(Feature& feature, ID3D12GraphicsCommandList* list, InputSlot& slot, - D3D12_RESOURCE_STATES outputState, uint32_t colorFrame) + D3D12_RESOURCE_STATES outputState, uint32_t colorFrame, bool split) { if (!slot.alphaRequested || slot.alphaResource == nullptr) return true; - if (colorFrame == 0) - return true; // no colour result to pair with this frame + AlphaRegion region = {slot.alphaBaseX, slot.alphaBaseY, slot.alphaExtentW, slot.alphaExtentH}; int index = -1; + if (split) { - std::lock_guard lock(feature.outputMutex); - // The frame whose colour is about to be presented is `colorFrame`; only an alpha result - // produced by that very same frame may go with it. Picking "the newest alpha" instead - // would pair alpha N with colour N-1, since the alpha read-back completes before the - // colour's, and the caller would composite two different frames. No match, no copy: the - // colour is presented on its own and the alpha waits for its own frame to come round. - for (int candidate = 0; candidate < kOutputSlots; ++candidate) - if (feature.alpha[candidate].producedFrame.load() == colorFrame) - { - index = candidate; - break; - } - if (index < 0) - return true; // the matching frame has not been read back yet; colour goes alone + // A split frame presents its own colour result in the second half of its own command list, + // so it presents its own alpha result: the very slot the worker is about to fill for this + // frame, which does not exist yet. The copy is recorded now and runs after the split + // semaphore, by which time the worker has written it - the same ordering the colour copy + // above relies on, and the reason the region comes from this frame's capture rather than + // from a published result. + index = static_cast(feature.frame % kOutputSlots); AlphaSlot& chosen = feature.alpha[index]; - if (chosen.staging.buffer == nullptr || chosen.producedFrame.load() == 0) - return true; - if (chosen.width != slot.alphaExtentW || chosen.height != slot.alphaExtentH || - chosen.format != slot.alphaFormat || chosen.baseX != slot.alphaBaseX || - chosen.baseY != slot.alphaBaseY) + if (slot.alphaGpu) { - // The caller's rectangle changed under a result already in flight; presenting it - // would write where the caller no longer expects. Leave it for the next result. - chosen.producedFrame = 0; - return true; + if (chosen.vram.buffer == VK_NULL_HANDLE) + return false; + } + else if (!ensure_staging(chosen.staging, slot.alphaResource, D3D12_HEAP_TYPE_UPLOAD)) + { + logf("frame %u: no upload buffer for the DLSSD.OutputAlpha result", feature.frame); + return false; + } + { + std::lock_guard lock(feature.outputMutex); + chosen.lastReadFrame = feature.frame; + } + } + else + { + if (colorFrame == 0) + return true; // no colour result to pair with this frame + { + std::lock_guard lock(feature.outputMutex); + // The frame whose colour is about to be presented is `colorFrame`; only an alpha result + // produced by that very same frame may go with it. Picking "the newest alpha" instead + // would pair alpha N with colour N-1, since the alpha read-back completes before the + // colour's, and the caller would composite two different frames. No match, no copy: the + // colour is presented on its own and the alpha waits for its own frame to come round. + for (int candidate = 0; candidate < kOutputSlots; ++candidate) + if (feature.alpha[candidate].producedFrame.load() == colorFrame) + { + index = candidate; + break; + } + if (index < 0) + return true; // the matching frame has not been read back yet; colour goes alone + AlphaSlot& chosen = feature.alpha[index]; + if ((chosen.gpu && chosen.vram.buffer == VK_NULL_HANDLE) || + (!chosen.gpu && chosen.staging.buffer == nullptr) || chosen.producedFrame.load() == 0) + return true; + // A result the worker left on the GPU can only be presented by a frame that qualified + // the GPU route itself: that frame's capture is what recorded the caller's image and + // whether its format needs the conversion blit. Anything else discards the result, + // exactly as a result whose rectangle no longer matches is discarded below. + if (chosen.gpu && !slot.alphaGpu) + { + chosen.producedFrame = 0; + return true; + } + if (chosen.width != slot.alphaExtentW || chosen.height != slot.alphaExtentH || + chosen.format != slot.alphaFormat || chosen.baseX != slot.alphaBaseX || + chosen.baseY != slot.alphaBaseY) + { + // The caller's rectangle changed under a result already in flight; presenting it + // would write where the caller no longer expects. Leave it for the next result. + chosen.producedFrame = 0; + return true; + } + chosen.lastReadFrame = feature.frame; + region = {chosen.baseX, chosen.baseY, chosen.width, chosen.height}; } - chosen.lastReadFrame = feature.frame; } AlphaSlot& chosen = feature.alpha[index]; + const bool gpu = split ? slot.alphaGpu : chosen.gpu; transition(list, slot.alphaResource, outputState, D3D12_RESOURCE_STATE_COPY_DEST); - D3D12_TEXTURE_COPY_LOCATION destination = {}; - destination.pResource = slot.alphaResource; - destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; - destination.SubresourceIndex = 0; - D3D12_TEXTURE_COPY_LOCATION source = {}; - source.pResource = chosen.staging.buffer; - source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; - source.PlacedFootprint = chosen.staging.layout; - // The source box picks out the evaluated rectangle inside the full-extent staging buffer, - // and the destination origin places it at the caller's subrect. Together they leave every - // pixel of the caller's texture outside the rectangle untouched. - D3D12_BOX box = {}; - box.left = chosen.baseX; - box.top = chosen.baseY; - box.right = chosen.baseX + chosen.width; - box.bottom = chosen.baseY + chosen.height; - box.back = 1; - list->CopyTextureRegion(&destination, chosen.baseX, chosen.baseY, 0, &source, &box); + if (gpu) + { + // The result is still the float32 the denoiser wrote, so it becomes the caller's texels + // here rather than on the CPU. The region lands at the caller's subrect, and the + // transitions around it are the same ones the host copy below records. + if (!record_alpha_vram(list, slot.alphaResource, slot.alphaImageHandle, chosen.vram, chosen.conversion, + region, slot.alphaConvert)) + { + transition(list, slot.alphaResource, D3D12_RESOURCE_STATE_COPY_DEST, outputState); + logf("frame %u: recording DLSSD.OutputAlpha GPU copy failed", feature.frame); + return false; + } + } + else + { + D3D12_TEXTURE_COPY_LOCATION destination = {}; + destination.pResource = slot.alphaResource; + destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + destination.SubresourceIndex = 0; + D3D12_TEXTURE_COPY_LOCATION source = {}; + source.pResource = chosen.staging.buffer; + source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + source.PlacedFootprint = chosen.staging.layout; + // The source box picks out the evaluated rectangle inside the full-extent staging buffer, + // and the destination origin places it at the caller's subrect. Together they leave every + // pixel of the caller's texture outside the rectangle untouched. + D3D12_BOX box = {}; + box.left = region.baseX; + box.top = region.baseY; + box.right = region.baseX + region.width; + box.bottom = region.baseY + region.height; + box.back = 1; + list->CopyTextureRegion(&destination, region.baseX, region.baseY, 0, &source, &box); + } transition(list, slot.alphaResource, D3D12_RESOURCE_STATE_COPY_DEST, outputState); return true; } @@ -4570,12 +4926,114 @@ static void destroy_aux_surface(AuxSurface& surface) g.cu.arrayDestroy(surface.image.array); for (Staging& staging : surface.staging) staging.release(); + for (HostPlane& host : surface.host) + release_host(host); + for (VramBuffer& buffer : surface.vram) + { + if (buffer.buffer == VK_NULL_HANDLE) + continue; + // Retirement can run on the finish thread. Release imports on the CUDA worker + // without a nested worker.call when retirement itself runs there. + g.worker.post([buffer] { + if (buffer.external != nullptr) + { + g.cu.memFree(buffer.device); + g_vk.release(buffer.external); + } + g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr); + g_vk.free(g_vk.device, buffer.memory, nullptr); + }); + buffer = {}; + } + destroy_vram_image(surface.conversion); if (surface.resource != nullptr) surface.resource->Release(); logf("Ray Reconstruction: released denoiser input %s (%ux%u fmt=0x%x)", join_aux_names(surface.publish).c_str(), surface.width, surface.height, surface.format); } +// Vulkan blits cannot convert integer images to float. Raw UINT/SINT guides +// retain host staging; other formats preserve the host path's sampled values. +static bool describe_aux_vram(AuxSurface& surface, VramCopy& copy) +{ + if (!vram_interop_available() || env_uint("D4R_SHIM_RR_VRAM_GUIDES", 1) == 0) + return false; + UINT64 handle = 0, offset = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + if (FAILED(g_vk.interop->GetVulkanResourceInfo1(surface.resource, &handle, &offset, &format)) || handle == 0) + return false; + copy.resource = surface.resource; + copy.image = reinterpret_cast(handle); + copy.width = surface.width; + copy.height = surface.height; + if (surface.how == AuxLayout::Verbatim) + return true; + if (surface.format != DXGI_FORMAT_B8G8R8A8_UNORM && + surface.format != DXGI_FORMAT_R10G10B10A2_UNORM && + surface.format != DXGI_FORMAT_R11G11B10_FLOAT) + return false; + copy.convert = true; + return vram_blit_supported(format, VK_FORMAT_R32G32B32A32_SFLOAT); +} + +static bool record_aux_vram(ID3D12GraphicsCommandList* list, AuxSurface& surface, int slotIndex, + const VramCopy& copy, D3D12_RESOURCE_STATES state) +{ + const size_t bytes = aux_texel_bytes(surface.arrayFormat, surface.channels) * surface.width * surface.height; + VramBuffer& buffer = surface.vram[slotIndex]; + // Resource geometry is immutable. Never worker.call under auxMutex: the + // worker could be waiting for this lock to upload an earlier frame. + if (buffer.buffer == VK_NULL_HANDLE && !create_vram_buffer(buffer, bytes, false)) + return false; + if (copy.convert && surface.conversion.image == VK_NULL_HANDLE && + !create_vram_image(surface.conversion, surface.width, surface.height, VK_FORMAT_R32G32B32A32_SFLOAT)) + return false; + transition(list, copy.resource, state, D3D12_RESOURCE_STATE_COPY_SOURCE); + VkCommandBuffer cmd = VK_NULL_HANDLE; + if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd))) + { + transition(list, copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, state); + return false; + } + g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, + &kBeforeTransfer, 0, nullptr, 0, nullptr); + VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL; + g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, &layout); + VkImage image = copy.image; + if (copy.convert) + { + image = surface.conversion.image; + VkImageMemoryBarrier barrier = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &barrier); + VkImageBlit blit = {}; + blit.srcSubresource = blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcOffsets[1] = blit.dstOffsets[1] = + {static_cast(copy.width), static_cast(copy.height), 1}; + g_vk.blit(cmd, copy.image, layout, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_NEAREST); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, + 0, nullptr, 1, &barrier); + layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + } + const VkBufferImageCopy region = full_region(copy); + g_vk.copyImageToBuffer(cmd, image, layout, buffer.buffer, 1, ®ion); + g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, + &kAfterTransfer, 0, nullptr, 0, nullptr); + const bool recorded = SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list)); + transition(list, copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, state); + return recorded; +} + // Game thread. Finds the auxiliary surfaces the caller registered this frame and records a // readback copy of each into this frame's slot, ahead of the frame marker so the worker can // read them as soon as that marker is in. The result is this slot's binding snapshot: the @@ -4616,6 +5074,7 @@ static bool gather_aux_inputs(Feature& feature, ID3D12GraphicsCommandList* list, binding = &candidate; break; } + const bool firstBinding = binding == nullptr; if (binding == nullptr) { bindings.emplace_back(); @@ -4633,6 +5092,8 @@ static bool gather_aux_inputs(Feature& feature, ID3D12GraphicsCommandList* list, bound = bound || strcmp(name, input.publish) == 0; if (!bound) binding->names.push_back(input.publish); + if (!firstBinding) + continue; // aliases share one captured plane and one upload D3D12_RESOURCE_DESC desc; resource->GetDesc(&desc); @@ -4652,9 +5113,19 @@ static bool gather_aux_inputs(Feature& feature, ID3D12GraphicsCommandList* list, surface->how = how; surface->width = static_cast(desc.Width); surface->height = static_cast(desc.Height); - if (!ensure_staging(surface->staging[slotIndex], resource, D3D12_HEAP_TYPE_READBACK)) - return false; - copy_to_staging(list, resource, surface->staging[slotIndex], inputState); + VramCopy gpuCopy; + surface->gpu[slotIndex] = describe_aux_vram(*surface, gpuCopy); + if (surface->gpu[slotIndex]) + { + if (!record_aux_vram(list, *surface, slotIndex, gpuCopy, inputState)) + return false; + } + if (!surface->gpu[slotIndex] || env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0) + { + if (!ensure_staging(surface->staging[slotIndex], resource, D3D12_HEAP_TYPE_READBACK)) + return false; + copy_to_staging(list, resource, surface->staging[slotIndex], inputState); + } } // Drop surfaces the caller stopped registering, but only once no queued frame names one: // erasing a surface whose handle is still in a parameter object would leave the denoiser @@ -4695,9 +5166,28 @@ static bool upload_aux_inputs(Feature& feature, int slotIndex, uint32_t frame) return false; } CudaMemcpy2D copy = {}; - copy.srcMemoryType = CUDA_MEMORY_HOST; - if (surface.how == AuxLayout::Verbatim) + if (surface.gpu[slotIndex]) { + VramBuffer& buffer = surface.vram[slotIndex]; + if (buffer.external == nullptr) + { + VkMemoryRequirements requirements = {}; + g_vk.bufferRequirements(g_vk.device, buffer.buffer, &requirements); + const int result = g_vk.import(g_vk.device, reinterpret_cast(buffer.memory), + requirements.size, &buffer.device, &buffer.external); + if (result != 0) + { + logf("frame %u: Ray Reconstruction guide import failed: %d", frame, result); + return false; + } + } + copy.srcMemoryType = CUDA_MEMORY_DEVICE; + copy.srcDevice = buffer.device; + copy.srcPitch = aux_texel_bytes(surface.arrayFormat, surface.channels) * surface.width; + } + else if (surface.how == AuxLayout::Verbatim) + { + copy.srcMemoryType = CUDA_MEMORY_HOST; // Use HostPlane staging: direct ROCm image copies from Vulkan-mapped memory // truncate guide rows on gfx1201, even when the source footprint is tightly packed. HostPlane& host = surface.host[slotIndex]; @@ -4717,6 +5207,7 @@ static bool upload_aux_inputs(Feature& feature, int slotIndex, uint32_t frame) } else { + copy.srcMemoryType = CUDA_MEMORY_HOST; // Packed, integer and BGRA inputs are unpacked into floats first: a CUDA array // has no three-channel layout, a uint array is sampled normalized, and BGRA bytes // are in the wrong order for a read as RGBA. @@ -4750,6 +5241,51 @@ static bool upload_aux_inputs(Feature& feature, int slotIndex, uint32_t frame) join_aux_names(binding.names).c_str(), surface.width, surface.height); return false; } + if (surface.gpu[slotIndex] && env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0) + { + // Compare the array NGX actually samples, not just the imported buffer. + const size_t rowBytes = copy.WidthInBytes; + std::vector expected(rowBytes * surface.height), actual(expected.size()); + for (UINT row = 0; row < surface.height; ++row) + { + const uint8_t* source = staging.mapped + staging.layout.Offset + + static_cast(row) * staging.layout.Footprint.RowPitch; + uint8_t* destination = expected.data() + static_cast(row) * rowBytes; + if (surface.how == AuxLayout::Verbatim) + memcpy(destination, source, rowBytes); + else + convert_aux_row(surface.format, source, reinterpret_cast(destination), + surface.width, surface.channels); + } + CudaMemcpy2D read = {}; + read.srcMemoryType = CUDA_MEMORY_ARRAY; + read.srcArray = surface.image.array; + read.dstMemoryType = CUDA_MEMORY_HOST; + read.dstHost = actual.data(); + read.dstPitch = read.WidthInBytes = rowBytes; + read.Height = surface.height; + if (g.cu.ctxSynchronize() != 0 || g.cu.memcpy2D(&read) != 0) + return false; + size_t differing = 0; + if (surface.how == AuxLayout::Verbatim) + { + for (size_t byte = 0; byte < expected.size(); ++byte) + differing += actual[byte] != expected[byte]; + } + else + { + const float* a = reinterpret_cast(actual.data()); + const float* e = reinterpret_cast(expected.data()); + for (size_t i = 0; i < expected.size() / sizeof(float); ++i) + differing += !(a[i] == e[i] || (std::isnan(a[i]) && std::isnan(e[i])) || + std::abs(a[i] - e[i]) <= 1e-6f * std::max(1.0f, std::abs(e[i]))); + } + logf("frame %u RR VRAM verify %s: %zu differing %s", frame, + join_aux_names(binding.names).c_str(), differing, + surface.how == AuxLayout::Verbatim ? "bytes" : "components"); + if (differing != 0) + return false; + } surface.handle = surface.image.object; } return true; @@ -5280,7 +5816,7 @@ static bool upload_inputs_vram(Feature& feature, InputSlot& slot, bool hasExposu // Only called on the CUDA worker. A completion event covers queued work on // the default stream; do not rely on HIP's blocking flag to release the CPU. -static int synchronize_default_stream(Feature& feature, bool* querySleep = nullptr, double* waitEmaUs = nullptr) +static int synchronize_default_stream(Feature& feature, bool* querySleep, double* waitEmaUs) { if (feature.outputReadyEvent != nullptr) { @@ -5731,12 +6267,12 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{}; // D4R_SHIM_EVAL_SYNC=0 (VRAM interop): no wait between NGX's kernels and the output copy queued // behind them on the same stream; publish_vram synchronises before the result is released. - static const bool evalSync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0; - const int syncResult = params.vram && !evalSync ? 0 : synchronize_default_stream(*feature, nullptr, &feature->evalWaitEmaUs); + const int syncResult = + params.vram && !eval_sync_enabled() ? 0 : synchronize_default_stream(*feature, nullptr, &feature->evalWaitEmaUs); const auto evaluated = ProfileClock::now(); const bool gpuEventsRecorded = eventStartResult == 0 && eventEndResult == 0; float gpuEvalMs = -1.0f; - if (gpuEventsRecorded && syncResult == 0 && (!params.vram || evalSync) && + if (gpuEventsRecorded && syncResult == 0 && (!params.vram || eval_sync_enabled()) && g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) != 0) gpuEvalMs = -1.0f; // Keep the input slot owned until output completion, including linear reads. @@ -5750,7 +6286,7 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram // The alpha surface is complete once the synchronize above has passed. A failure here drops // the whole frame, exactly as a failed synchronize does: an accepted frame with an unwritten // alpha would leave the caller believing it was written. - if (feature->rayReconstruction && !download_alpha_output(*feature, frame)) + if (feature->rayReconstruction && !download_alpha_output(*feature, frame, params)) { slot.busy = false; frame_retired(feature, frame); @@ -5761,7 +6297,7 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram timing.ngxHost = profile_ms(evalStart, evalReturned); timing.ctxSync = profile_ms(syncStart, evaluated); timing.gpuEval = gpuEvalMs; - timing.gpuEventsRecorded = gpuEventsRecorded && params.vram && !evalSync; + timing.gpuEventsRecorded = gpuEventsRecorded && params.vram && !eval_sync_enabled(); } if (params.vram) { @@ -5879,12 +6415,15 @@ static void prepare_inputs(Feature* feature, int slotIndex, uint32_t frame, Fram return; } FrameTiming* stages = timing.enabled ? &timing : nullptr; - stage_plane(Plane::Color, slot.color, slot.host[0], stages, 0); - stage_plane(Plane::Depth, slot.depth, slot.host[1], stages, 1); - stage_plane(Plane::Motion, slot.motion, slot.host[2], stages, 2); + // Verification must not replace a pitch-linear texture's padded GPU stride + // with the tightly packed host stride. + stage_plane(Plane::Color, slot.color, slot.host[0], stages, 0, params.vram ? slot.host[0].rowBytes : 0); + stage_plane(Plane::Depth, slot.depth, slot.host[1], stages, 1, params.vram ? slot.host[1].rowBytes : 0); + stage_plane(Plane::Motion, slot.motion, slot.host[2], stages, 2, params.vram ? slot.host[2].rowBytes : 0); log_motion_stats(slot.host[2], slot.host[0], frame); if (params.hasExposure) - stage_plane(Plane::Exposure, slot.exposure, slot.host[3], stages, 3); + stage_plane(Plane::Exposure, slot.exposure, slot.host[3], stages, 3, + params.vram ? slot.host[3].rowBytes : 0); char dumpDirectory[MAX_PATH]; if (input_dump_directory(frame, dumpDirectory)) dump_input_planes(slot.host, frame, params, dumpDirectory); @@ -6286,6 +6825,13 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetFeatureRequirements(void* adapter, const } { std::lock_guard lock(g.mutex); + // Discovery can precede Init; CUDA still needs the caller's adapter identity. + if (adapter != nullptr && !g.ngxInitialized) + { + DXGI_ADAPTER_DESC desc = {}; + if (SUCCEEDED(static_cast(adapter)->GetDesc(&desc))) + set_cuda_adapter_luid(desc.AdapterLuid); + } // Requirements must not depend on a prior Init. Load the provider without // initializing NGX here: the later Init retains the game's identity and device. if (ensure_workers_started() != NGX_SUCCESS || (g.core == nullptr && !load_libraries())) @@ -7964,11 +8510,13 @@ static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* hand const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure}; const int inputCount = exposure != nullptr ? 4 : 3; VramCopy vramInputs[4], vramOutput; - // Alpha readback uses the host result ring; color must use the same pipeline so both - // outputs can present one produced frame. The existing cutover below drains VRAM work. - const bool alphaOutputRequested = - feature->rayReconstruction && get_resource(parameters, "DLSSD.OutputAlpha") != nullptr; - if (!alphaOutputRequested && (!feature->vramDecided || feature->vram) && vram_interop_available()) + // A separate alpha output no longer holds the feature off VRAM interop: the float32 result + // stays on the GPU and is blitted into the caller's texture when that texture's format + // supports the conversion (see prepare_alpha_vram). A format that does not - or a texture Vulkan + // cannot hand over - keeps the alpha on the host route for that frame alone, without the + // colour output having to fall back with it. The existing cutover below still drains VRAM + // work if the colour output itself stops qualifying. + if ((!feature->vramDecided || feature->vram) && vram_interop_available()) { p.vram = describe_vram_copy(output, Plane::Color, vramOutput, true); for (int index = 0; index < inputCount && p.vram; ++index) @@ -8158,6 +8706,10 @@ static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* hand list2->Release(); return NGX_FAIL_PLATFORM_ERROR; } + // The alpha's own route is decided here, on the game thread, next to the capture that + // named the texture: allocating on the worker is not an option (the CUDA import runs on + // the worker) and this is the same place that already knows the region and the format. + prepare_alpha_vram(*feature, slot, p.vram); } if (timing.enabled) timing.inputRecord = profile_ms(inputRecordStart, ProfileClock::now()); @@ -8240,8 +8792,9 @@ static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* hand list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr); transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState); } - if (feature->rayReconstruction) - record_alpha_copyback(*feature, list, slot, outputState, colorFrame); + if (feature->rayReconstruction && + !record_alpha_copyback(*feature, list, slot, outputState, colorFrame, p.split)) + return NGX_FAIL_PLATFORM_ERROR; if (timing.enabled) { timing.outputRecord = profile_ms(outputRecordStart, ProfileClock::now()); diff --git a/tools/wine_nvcuda_bridge.c b/tools/wine_nvcuda_bridge.c index 18f1bae..9a796ea 100644 --- a/tools/wine_nvcuda_bridge.c +++ b/tools/wine_nvcuda_bridge.c @@ -1466,7 +1466,6 @@ static const char* find_function_name(CUfunction function) pthread_mutex_unlock(&instrumentation_lock); return name; } - /* Opt-in per-kernel GPU elapsed time. Event pairs are recorded on the same stream as each launch, then read only after an existing context sync. This adds profiling overhead and must never be used as the FPS baseline. */ @@ -2261,11 +2260,18 @@ CUresult WINAPI cuDeviceGetLuid(char* luid, unsigned int* device_node_mask, CUde if (function == NULL) return missing("cuDeviceGetLuid"); CUresult result = function(luid, device_node_mask, device); - if (result == CUDA_SUCCESS && luid != NULL && device_node_mask != NULL) + if ((result == CUDA_SUCCESS || result == CUDA_ERROR_NOT_SUPPORTED) && + luid != NULL && device_node_mask != NULL) { unsigned int low = get_process_u32("D4R_CUDA_LUID_LOW", 0xffffffffu); unsigned int high = get_process_u32("D4R_CUDA_LUID_HIGH", 0xffffffffu); unsigned int node_mask = get_process_u32("D4R_CUDA_NODE_MASK", 0xffffffffu); + // Linux NVIDIA cannot report a Windows LUID. The shim supplies the actual + // D3D12 adapter mapping; only a complete mapping can replace NOT_SUPPORTED. + if (result == CUDA_ERROR_NOT_SUPPORTED && + (low == 0xffffffffu || high == 0xffffffffu || node_mask == 0xffffffffu)) + return result; + result = CUDA_SUCCESS; if (low != 0xffffffffu) memcpy(luid, &low, sizeof(low)); if (high != 0xffffffffu)