Change d4r license to GPL-3.0-only

- Set Disable.Watermark on the game path (the Linux shim already does);
  [DLSS] ShowWatermark = true keeps NVIDIA's stamp.

- Always write d4r\d4r_nvngx.log in player mode. It was skipped when the
  launcher gave the game a valid stderr pipe, e.g. Steam.

- Accept NVIDIA's 310.7.0 + _nvngx 32.0.15.9636 as "less tested" pins.
  Native kernels are still gated by PTX identity, so a different DLL
  falls back to translated kernels.

- Add d4r-check.ps1 (ok/MISSING/note lines with a fix for each; -Report
  writes a bug-report ZIP) with 15 fixture tests.

- package-windows-release.ps1: -Architectures subset, ships the check
  script, GPU registry and one GPU-query tool.

- Add a shorter WINDOWS_README.txt; trim README's Windows section and
  merge in the RX 9060 XT / distributor-DLL notes.

- Diagnostic probe accepts preset 12 (L).

Tested on RX 7700 XT without a game: loader and shim start-up, K and M
synthetic NGX frames with both DLL pairs (finite output, native kernels
used). Not tested in a game.

.gitattributes

@@ -0,0 +1,3 @@
+# Patches are applied with `git apply`, which fails on CRLF. Keep them LF even
+# where core.autocrlf=true (Windows checkouts).
+*.patch text eol=lf

.gitignore

@@ -15,6 +15,9 @@
 /traces/
 /build/
 /dist/
+/.tools/
+/external/
+/test-results/
 *.trace
 *.dump
 *.log

CMakeLists.txt

@@ -0,0 +1,324 @@
+cmake_minimum_required(VERSION 3.24)
+project(d4r_windows_diagnostics LANGUAGES C CXX)
+if(NOT WIN32 OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
+  message(FATAL_ERROR "These diagnostics require native Windows x64")
+endif()
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(D4R_HIP_ROOT "$ENV{HIP_PATH}" CACHE PATH "One stable HIP SDK or TheRock installation")
+if(NOT D4R_HIP_ROOT AND EXISTS "C:/Program Files/AMD/ROCm/7.2")
+  set(D4R_HIP_ROOT "C:/Program Files/AMD/ROCm/7.2")
+endif()
+if(NOT EXISTS "${D4R_HIP_ROOT}/include/hip/hip_runtime_api.h")
+  message(FATAL_ERROR "Set D4R_HIP_ROOT to a Windows HIP SDK (headers and bin/clang++.exe required)")
+endif()
+find_program(D4R_HIP_CLANG NAMES clang++.exe PATHS "${D4R_HIP_ROOT}/bin" "${D4R_HIP_ROOT}/lib/llvm/bin" NO_DEFAULT_PATH REQUIRED)
+# Architecture parameterization adapted from realdody/d4r ae530b0.
+# Keep explicit subtargets: no gfx11 substitution or untested future gfx12 ISA.
+file(READ "${CMAKE_CURRENT_SOURCE_DIR}/tools/windows/gpu-targets.json" d4r_target_json)
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS tools/windows/gpu-targets.json)
+string(JSON d4r_target_count LENGTH "${d4r_target_json}" targets)
+math(EXPR d4r_target_last "${d4r_target_count} - 1")
+set(d4r_supported_targets)
+set(d4r_target_cpp_entries "")
+foreach(index RANGE ${d4r_target_last})
+  foreach(field architecture elfMachine isaMajor family)
+    string(JSON target_${field} GET "${d4r_target_json}" targets ${index} ${field})
+  endforeach()
+  list(APPEND d4r_supported_targets "${target_architecture}")
+  string(APPEND d4r_target_cpp_entries "  GpuTarget{\"${target_architecture}\", ${target_elfMachine}, ${target_isaMajor}, \"${target_family}\"},\n")
+endforeach()
+configure_file(tools/windows/gpu_targets.generated.h.in gpu_targets.generated.h @ONLY)
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+set(D4R_GPU_ARCH "gfx1201" CACHE STRING "Native Windows package GPU target")
+set_property(CACHE D4R_GPU_ARCH PROPERTY STRINGS ${d4r_supported_targets})
+if(NOT D4R_GPU_ARCH IN_LIST d4r_supported_targets)
+  message(FATAL_ERROR "D4R_GPU_ARCH must be one of: ${d4r_supported_targets}")
+endif()
+add_compile_definitions(D4R_TARGET_ARCH="${D4R_GPU_ARCH}")
+configure_file(tools/windows/gpu-target.json.in gpu-target.json @ONLY)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/gpu-target.json" DESTINATION .)
+install(FILES tools/windows/gpu-targets.json DESTINATION .)
+set(d4r_device_flags)
+if(D4R_GPU_ARCH MATCHES "^gfx11")
+  # Windows-only workaround informed by realdody/d4r 1897b4a. Keep the real
+  # gfx11 layout; do not route RDNA3 through gfx12 fragments.
+  list(APPEND d4r_device_flags -DD4R_WINDOWS_GFX11_LANE_EXCHANGE)
+endif()
+set(probe_object "${CMAKE_CURRENT_BINARY_DIR}/probe_${D4R_GPU_ARCH}.hsaco")
+add_custom_command(OUTPUT "${probe_object}"
+  COMMAND "${D4R_HIP_CLANG}" -x hip --offload-arch=${D4R_GPU_ARCH}
+    --offload-device-only --no-gpu-bundle-output -mno-wavefrontsize64
+    -nogpuinc -nogpulib -O2 "${CMAKE_CURRENT_SOURCE_DIR}/tools/windows/probe_kernel.hip"
+    -o "${probe_object}"
+  DEPENDS tools/windows/probe_kernel.hip VERBATIM)
+add_custom_target(d4r_probe_code_object ALL DEPENDS "${probe_object}")
+set(pixel_object "${CMAKE_CURRENT_BINARY_DIR}/pixel_convert_${D4R_GPU_ARCH}.hsaco")
+add_custom_command(OUTPUT "${pixel_object}"
+  COMMAND "${D4R_HIP_CLANG}" -x hip -std=c++17 --offload-arch=${D4R_GPU_ARCH}
+    --offload-device-only --no-gpu-bundle-output -mno-wavefrontsize64
+    -nogpuinc -nogpulib -O2 "${CMAKE_CURRENT_SOURCE_DIR}/tools/windows/pixel_convert_gfx1201.hip"
+    -o "${pixel_object}"
+  DEPENDS tools/windows/pixel_convert_gfx1201.hip tools/windows/pixel_format.h kernels/common/hip_device_minimal.h VERBATIM)
+add_custom_target(d4r_pixel_code_object ALL DEPENDS "${pixel_object}")
+set(wmma_object "${CMAKE_CURRENT_BINARY_DIR}/wmma_${D4R_GPU_ARCH}.hsaco")
+add_custom_command(OUTPUT "${wmma_object}"
+  COMMAND "${D4R_HIP_CLANG}" -x hip --offload-arch=${D4R_GPU_ARCH}
+    --offload-device-only --no-gpu-bundle-output -mno-wavefrontsize64
+    -nogpuinc -nogpulib -O2 ${d4r_device_flags} "${CMAKE_CURRENT_SOURCE_DIR}/tools/windows/wmma_gfx1201.hip"
+    -o "${wmma_object}"
+  DEPENDS tools/windows/wmma_gfx1201.hip kernels/common/wmma_backend.h
+    kernels/common/wmma_layout.h kernels/common/wave32_exchange.h VERBATIM)
+add_custom_target(d4r_wmma_code_object ALL DEPENDS "${wmma_object}")
+# K/M objects stay in experimental/, outside the ZLUDA native override
+# directory, until each layer and then the complete network pass replay.
+# M uses the existing FP16 widening baseline; native FP8 is not enabled here.
+set(native_objects)
+option(D4R_K_FP16_BASELINE "Round every K16 accumulation step like NVIDIA f16 WMMA" ON)
+option(D4R_M_FP16_BASELINE "Preserve M e4m3 quantisation and f16 accumulation per K32" ON)
+foreach(family k m)
+  file(GLOB native_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${family}/*.hip")
+  file(GLOB native_headers CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/kernels/${family}/*.h")
+  foreach(source IN LISTS native_sources)
+    get_filename_component(kernel_name "${source}" NAME_WE)
+    set(native_object "${CMAKE_CURRENT_BINARY_DIR}/${kernel_name}_${D4R_GPU_ARCH}.hsaco")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${source}")
+    file(READ "${source}" kernel_source)
+    string(REGEX MATCH "// d4r-build-flags: ([^\r\n]+)" flags_line "${kernel_source}")
+    set(native_flags)
+    if(family STREQUAL "k" AND D4R_K_FP16_BASELINE)
+      list(APPEND native_flags -DD4R_K_FP16_BASELINE)
+    endif()
+    if(family STREQUAL "m" AND D4R_M_FP16_BASELINE)
+      list(APPEND native_flags -DSWIN_EXACT -DSWIN_EXACT_PV)
+    endif()
+    if(flags_line)
+      separate_arguments(source_flags NATIVE_COMMAND "${CMAKE_MATCH_1}")
+      list(APPEND native_flags ${source_flags})
+    endif()
+    add_custom_command(OUTPUT "${native_object}"
+      COMMAND "${D4R_HIP_CLANG}" -x hip -std=c++17 --offload-arch=${D4R_GPU_ARCH}
+        --offload-device-only --no-gpu-bundle-output -mno-wavefrontsize64
+        -nogpuinc -nogpulib -O3 -DD4R_DEVICE_ONLY_MINIMAL ${d4r_device_flags} ${native_flags}
+        "${source}" -o "${native_object}"
+      DEPENDS "${source}" ${native_headers} kernels/common/wmma_layout.h kernels/common/wave32_exchange.h
+        kernels/common/hip_device_minimal.h VERBATIM)
+    list(APPEND native_objects "${native_object}")
+    install(FILES "${native_object}" DESTINATION "experimental/${family}")
+  endforeach()
+endforeach()
+add_custom_target(d4r_native_compile ALL DEPENDS ${native_objects})
+# Retain the old manual build target for existing scripts.
+add_custom_target(d4r_native_gfx1201_compile DEPENDS d4r_native_compile)
+foreach(probe gpu_inventory hip_gfx1201_probe gfx12_wmma_probe k_module_probe native_replay_probe cuda_driver_probe cuda_mma_probe cuda_image_probe d3d12_command_probe d3d12_pixel_probe d3d12_hip_interop_probe hip_stream_lifecycle_probe ngx_cuda_init_probe d3d12_dlss_probe)
+  add_executable(d4r_${probe} tools/windows/${probe}.cpp)
+  target_compile_definitions(d4r_${probe} PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
+  target_include_directories(d4r_${probe} PRIVATE "${D4R_HIP_ROOT}/include")
+  target_link_libraries(d4r_${probe} PRIVATE version dbghelp)
+  if(MINGW)
+    target_link_options(d4r_${probe} PRIVATE -static)
+  endif()
+  if(MSVC)
+    target_compile_options(d4r_${probe} PRIVATE /W4 /EHsc)
+    set_property(TARGET d4r_${probe} PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+  else()
+    target_compile_options(d4r_${probe} PRIVATE -Wall -Wextra)
+  endif()
+  install(TARGETS d4r_${probe} RUNTIME DESTINATION bin)
+endforeach()
+foreach(probe binary16_probe gpu_target_probe)
+  add_executable(d4r_${probe} tools/windows/${probe}.cpp)
+  if(MINGW)
+    target_link_options(d4r_${probe} PRIVATE -static)
+  elseif(MSVC)
+    set_property(TARGET d4r_${probe} PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+  endif()
+  install(TARGETS d4r_${probe} RUNTIME DESTINATION bin)
+endforeach()
+add_dependencies(d4r_gpu_target_probe d4r_probe_code_object d4r_pixel_code_object)
+set(D4R_MINHOOK_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/external/MinHook" CACHE PATH "MinHook 1.3.4 source checkout")
+if(NOT EXISTS "${D4R_MINHOOK_ROOT}/include/MinHook.h")
+  message(FATAL_ERROR "Clone MinHook v1.3.4 into external/MinHook or set D4R_MINHOOK_ROOT")
+endif()
+add_subdirectory("${D4R_MINHOOK_ROOT}" "${CMAKE_CURRENT_BINARY_DIR}/minhook" EXCLUDE_FROM_ALL)
+add_library(d4r_nvngx_windows SHARED tools/windows/d4r_nvngx_windows.cpp tools/windows/d3d12_command_hooks.cpp)
+add_dependencies(d4r_nvngx_windows d4r_pixel_code_object)
+set_target_properties(d4r_nvngx_windows PROPERTIES PREFIX "" OUTPUT_NAME "d4r_nvngx")
+target_compile_definitions(d4r_nvngx_windows PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
+target_include_directories(d4r_nvngx_windows PRIVATE "${D4R_HIP_ROOT}/include")
+target_link_libraries(d4r_nvngx_windows PRIVATE version dbghelp d3d12 dxguid minhook bcrypt)
+install(FILES "${D4R_MINHOOK_ROOT}/LICENSE.txt" DESTINATION licenses RENAME MinHook.txt)
+if(MINGW)
+  target_link_options(d4r_nvngx_windows PRIVATE -static)
+endif()
+install(TARGETS d4r_nvngx_windows RUNTIME DESTINATION bin)
+# Target-independent _nvngx.dll of drag-in releases: loads _nvngx_<arch>.dll
+# for the game's actual HIP architecture. Its own D4R_TARGET_ARCH is unused.
+add_library(d4r_nvngx_loader SHARED tools/windows/d4r_nvngx_loader.cpp)
+set_target_properties(d4r_nvngx_loader PROPERTIES PREFIX "" OUTPUT_NAME "d4r_nvngx_loader")
+target_compile_definitions(d4r_nvngx_loader PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__)
+target_include_directories(d4r_nvngx_loader PRIVATE "${D4R_HIP_ROOT}/include")
+target_link_libraries(d4r_nvngx_loader PRIVATE version dbghelp)
+if(MINGW)
+  target_link_options(d4r_nvngx_loader PRIVATE -static)
+endif()
+install(TARGETS d4r_nvngx_loader RUNTIME DESTINATION bin)
+if(MSVC)
+  target_sources(d4r_ngx_cuda_init_probe PRIVATE tools/d4r_ngx_param_msvc.cpp tools/d4r_ngx_param_host.cpp)
+  target_sources(d4r_d3d12_dlss_probe PRIVATE tools/d4r_ngx_param_msvc.cpp tools/d4r_ngx_param_host.cpp)
+  target_sources(d4r_nvngx_windows PRIVATE tools/d4r_ngx_param_msvc.cpp tools/d4r_ngx_param_host.cpp)
+else()
+  find_program(D4R_CLANG_CL NAMES clang-cl.exe PATHS "${D4R_HIP_ROOT}/bin" "${D4R_HIP_ROOT}/lib/llvm/bin" NO_DEFAULT_PATH REQUIRED)
+  set(ngx_parameter_object "${CMAKE_CURRENT_BINARY_DIR}/ngx_parameter_msvc.obj")
+  add_custom_command(OUTPUT "${ngx_parameter_object}"
+    COMMAND "${D4R_CLANG_CL}" /c /O2 /Zl /GS- /GR- /EHs-c-
+      "${CMAKE_CURRENT_SOURCE_DIR}/tools/d4r_ngx_param_msvc.cpp" "/Fo${ngx_parameter_object}"
+    DEPENDS tools/d4r_ngx_param_msvc.cpp VERBATIM)
+  set_source_files_properties("${ngx_parameter_object}" PROPERTIES GENERATED TRUE EXTERNAL_OBJECT TRUE)
+  target_sources(d4r_ngx_cuda_init_probe PRIVATE "${ngx_parameter_object}" tools/d4r_ngx_param_host.cpp)
+  target_sources(d4r_d3d12_dlss_probe PRIVATE "${ngx_parameter_object}" tools/d4r_ngx_param_host.cpp)
+  target_sources(d4r_nvngx_windows PRIVATE "${ngx_parameter_object}" tools/d4r_ngx_param_host.cpp)
+endif()
+add_library(d4r_nvapi_trace SHARED tools/windows/nvapi_trace.cpp)
+set_target_properties(d4r_nvapi_trace PROPERTIES PREFIX "" OUTPUT_NAME "nvapi64"
+  ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/nvapi-trace"
+  RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/nvapi-trace")
+target_compile_definitions(d4r_nvapi_trace PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX)
+target_link_libraries(d4r_nvapi_trace PRIVATE version dbghelp)
+if(MINGW)
+  target_link_options(d4r_nvapi_trace PRIVATE -static)
+endif()
+install(TARGETS d4r_nvapi_trace RUNTIME DESTINATION nvapi-trace)
+add_library(d4r_nvapi_compat SHARED tools/windows/nvapi_trace.cpp)
+set_target_properties(d4r_nvapi_compat PROPERTIES PREFIX "" OUTPUT_NAME "nvapi64"
+  ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/nvapi-compat"
+  RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/nvapi-compat")
+target_compile_definitions(d4r_nvapi_compat PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__ D4R_NVAPI_COMPAT)
+target_include_directories(d4r_nvapi_compat PRIVATE "${D4R_HIP_ROOT}/include")
+target_link_libraries(d4r_nvapi_compat PRIVATE version dbghelp)
+if(MINGW)
+  target_link_options(d4r_nvapi_compat PRIVATE -static)
+endif()
+install(TARGETS d4r_nvapi_compat RUNTIME DESTINATION nvapi-compat)
+target_link_libraries(d4r_d3d12_hip_interop_probe PRIVATE d3d12 dxgi dxguid)
+target_link_libraries(d4r_d3d12_dlss_probe PRIVATE d3d12 dxgi dxguid d3dcompiler)
+add_executable(d4r_debug_launcher tools/windows/debug_launcher.cpp)
+target_include_directories(d4r_debug_launcher PRIVATE tools/windows)
+target_link_libraries(d4r_debug_launcher PRIVATE version dbghelp)
+install(TARGETS d4r_debug_launcher RUNTIME DESTINATION bin)
+target_link_libraries(d4r_d3d12_command_probe PRIVATE d3d12 dxgi dxguid)
+target_link_libraries(d4r_d3d12_pixel_probe PRIVATE d3d12 dxgi dxguid)
+add_dependencies(d4r_d3d12_pixel_probe d4r_pixel_code_object)
+add_dependencies(d4r_d3d12_command_probe d4r_nvngx_windows)
+set(D4R_AGILITY_ROOT "" CACHE PATH "Optional unpacked Microsoft.Direct3D.D3D12 1.619.5 debug SDK")
+if(EXISTS "${D4R_AGILITY_ROOT}/build/native/bin/x64/d3d12SDKLayers.dll")
+  add_executable(d4r_d3d12_command_debug_probe EXCLUDE_FROM_ALL tools/windows/d3d12_command_probe.cpp)
+  target_compile_definitions(d4r_d3d12_command_debug_probe PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX __HIP_PLATFORM_AMD__ D4R_DIAGNOSTIC_AGILITY)
+  target_include_directories(d4r_d3d12_command_debug_probe PRIVATE "${D4R_HIP_ROOT}/include")
+  target_link_libraries(d4r_d3d12_command_debug_probe PRIVATE version dbghelp d3d12 dxgi dxguid)
+  if(MINGW)
+    target_link_options(d4r_d3d12_command_debug_probe PRIVATE -static)
+  endif()
+  foreach(dll D3D12Core.dll d3d12SDKLayers.dll)
+    configure_file("${D4R_AGILITY_ROOT}/build/native/bin/x64/${dll}" "${CMAKE_CURRENT_BINARY_DIR}/debug-d3d12/${dll}" COPYONLY)
+  endforeach()
+  configure_file("${D4R_AGILITY_ROOT}/LICENSE.txt" "${CMAKE_CURRENT_BINARY_DIR}/debug-d3d12/LICENSE.txt" COPYONLY)
+endif()
+add_dependencies(d4r_hip_gfx1201_probe d4r_probe_code_object)
+add_dependencies(d4r_gfx12_wmma_probe d4r_wmma_code_object)
+add_dependencies(d4r_k_module_probe d4r_native_compile)
+install(FILES "${probe_object}" DESTINATION bin)
+install(FILES "${pixel_object}" DESTINATION bin)
+install(FILES "${wmma_object}" DESTINATION bin)
+install(FILES scripts/windows/test-windows-rdna4.ps1 DESTINATION .)
+install(FILES scripts/windows/gpu-target.ps1 DESTINATION .)
+install(FILES scripts/windows/gpu-preflight.ps1 DESTINATION .)
+install(FILES scripts/windows/test-gpu-target-contract.ps1 DESTINATION .)
+install(FILES scripts/windows/test-k-replay.ps1 DESTINATION .)
+install(FILES scripts/windows/test-k-captures.ps1 DESTINATION .)
+install(FILES scripts/windows/test-native-overrides.ps1 DESTINATION .)
+install(FILES tools/windows/k_layer_validate.py DESTINATION .)
+install(FILES tools/windows/k_replay_validate.py DESTINATION .)
+install(FILES tools/windows/k_ptx_validate.py DESTINATION .)
+install(FILES tools/windows/frame_compare.py DESTINATION .)
+install(FILES tools/windows/m_replay_validate.py DESTINATION .)
+install(FILES tools/windows/m_ptx_validate.py DESTINATION .)
+install(FILES tools/windows/m_temporal_validate.py tools/windows/m_capture_compare.py DESTINATION .)
+install(FILES scripts/windows/test-m-replay.ps1 DESTINATION .)
+install(FILES scripts/windows/test-m-captures.ps1 DESTINATION .)
+install(FILES kernels/tools/pwin_model.py kernels/tools/pwin_check.py kernels/tools/ptxsim.py
+  kernels/tools/extract_dlss_ptx.py kernels/tools/kernel_manifest.py DESTINATION reference)
+install(FILES kernels/tools/model_enc3.py kernels/tools/swin_model.py kernels/tools/swin_check.py DESTINATION reference)
+install(FILES docs/windows-rdna4-port.md DESTINATION docs)
+install(FILES docs/windows-gpu-support.md DESTINATION docs)
+include(CTest)
+add_test(NAME binary16_rounding COMMAND d4r_binary16_probe)
+set_tests_properties(binary16_rounding PROPERTIES TIMEOUT 30 LABELS "software;numerics")
+add_test(NAME gpu_target_contract COMMAND d4r_gpu_target_probe "${probe_object}" "${pixel_object}")
+set_tests_properties(gpu_target_contract PROPERTIES TIMEOUT 30 LABELS "software;architecture")
+find_program(D4R_POWERSHELL NAMES pwsh.exe powershell.exe REQUIRED)
+add_test(NAME gpu_target_script_contract COMMAND "${D4R_POWERSHELL}" -NoProfile -File
+  "${CMAKE_CURRENT_SOURCE_DIR}/scripts/windows/test-gpu-target-contract.ps1"
+  -PackageRoot "${CMAKE_CURRENT_BINARY_DIR}" -CodeObject "${probe_object}")
+set_tests_properties(gpu_target_script_contract PROPERTIES TIMEOUT 30 LABELS "software;architecture")
+add_test(NAME d3d12_gpu_pixel_formats COMMAND d4r_d3d12_pixel_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "${pixel_object}" --iterations 2)
+set_tests_properties(d3d12_gpu_pixel_formats PROPERTIES TIMEOUT 120 LABELS "hardware;${D4R_GPU_ARCH};formats")
+add_test(NAME d3d12_gpu_enhanced_formats COMMAND d4r_d3d12_pixel_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "${pixel_object}" --interop-mode enhanced --iterations 2)
+set_tests_properties(d3d12_gpu_enhanced_formats PROPERTIES TIMEOUT 120 LABELS "hardware;${D4R_GPU_ARCH};formats;enhanced_barriers")
+add_test(NAME d3d12_command_backend COMMAND d4r_d3d12_command_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "$<TARGET_FILE:d4r_nvngx_windows>" --interop-mode indirect --iterations 32)
+set_tests_properties(d3d12_command_backend PROPERTIES TIMEOUT 120 LABELS "hardware;d3d12;command_lists;lifetime")
+add_test(NAME d3d12_command_initial_pso COMMAND d4r_d3d12_command_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "$<TARGET_FILE:d4r_nvngx_windows>" --interop-mode initial-pso --iterations 32)
+set_tests_properties(d3d12_command_initial_pso PROPERTIES TIMEOUT 120 LABELS "hardware;d3d12;command_lists;lifetime")
+add_test(NAME d3d12_command_root_reset_warp COMMAND d4r_d3d12_command_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "$<TARGET_FILE:d4r_nvngx_windows>" --interop-mode indirect-root-reset --iterations 32)
+set_tests_properties(d3d12_command_root_reset_warp PROPERTIES TIMEOUT 120 LABELS "software;d3d12;command_lists;lifetime"
+  ENVIRONMENT "D4R_COMMAND_PROBE_WARP=1;D4R_COMMAND_PROBE_NO_HIP=1")
+add_test(NAME hip_gfx1201 COMMAND d4r_hip_gfx1201_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "${probe_object}")
+set_tests_properties(hip_gfx1201 PROPERTIES TIMEOUT 90 LABELS "hardware;${D4R_GPU_ARCH}")
+add_test(NAME gfx12_wmma COMMAND d4r_gfx12_wmma_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "${wmma_object}" --iterations 16)
+set_tests_properties(gfx12_wmma PROPERTIES TIMEOUT 90 LABELS "hardware;${D4R_GPU_ARCH};wmma")
+foreach(layer enc1 enc2)
+  add_test(NAME k_${layer}_execution COMMAND d4r_k_module_probe
+    --hip-root "${D4R_HIP_ROOT}"
+    --module "${CMAKE_CURRENT_BINARY_DIR}/dltss_pwin_${layer}_layer_${D4R_GPU_ARCH}.hsaco"
+    --kernel-name ${layer})
+  set_tests_properties(k_${layer}_execution PROPERTIES TIMEOUT 90 LABELS "hardware;${D4R_GPU_ARCH};k_execution")
+endforeach()
+add_test(NAME d3d12_hip_interop COMMAND d4r_d3d12_hip_interop_probe
+  --hip-root "${D4R_HIP_ROOT}" --module "${probe_object}" --iterations 32)
+set_tests_properties(d3d12_hip_interop PROPERTIES TIMEOUT 120 LABELS "hardware;interop")
+add_test(NAME d3d12_hip_map_lifetime COMMAND d4r_d3d12_hip_interop_probe
+  --hip-root "${D4R_HIP_ROOT}" --interop-mode map --iterations 64)
+set_tests_properties(d3d12_hip_map_lifetime PROPERTIES TIMEOUT 120 LABELS "hardware;interop;lifetime")
+set(D4R_ZLUDA_ROOT "" CACHE PATH "Local Windows ZLUDA directory (nvcuda.dll)")
+if(EXISTS "${D4R_ZLUDA_ROOT}/nvcuda.dll")
+  foreach(context primary created)
+    add_test(NAME cuda_${context} COMMAND d4r_cuda_driver_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll"
+      --context ${context} --iterations 32)
+    set_tests_properties(cuda_${context} PROPERTIES TIMEOUT 180 LABELS "hardware;zluda")
+  endforeach()
+  if(EXISTS "${D4R_ZLUDA_ROOT}/build-info.json")
+    add_test(NAME cuda_f16_mma_reference COMMAND d4r_cuda_mma_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll")
+    set_tests_properties(cuda_f16_mma_reference PROPERTIES TIMEOUT 120
+      LABELS "hardware;zluda;mma;reference"
+      ENVIRONMENT "D4R_ZLUDA_WMMA=1;D4R_ZLUDA_WMMA_F16_REFERENCE=1")
+    add_test(NAME cuda_images COMMAND d4r_cuda_image_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32)
+    set_tests_properties(cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors")
+    add_test(NAME cuda_images_async COMMAND d4r_cuda_image_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --iterations 32)
+    set_tests_properties(cuda_images_async PROPERTIES TIMEOUT 90 LABELS "hardware;zluda;descriptors;async"
+      ENVIRONMENT "D4R_DIAG_ASYNC_ARRAY_COPY=1")
+    add_test(NAME d3d12_cuda_images COMMAND d4r_d3d12_dlss_probe
+      --hip-root "${D4R_HIP_ROOT}" --cuda-dll "${D4R_ZLUDA_ROOT}/nvcuda.dll" --interop-mode images --iterations 1)
+    set_tests_properties(d3d12_cuda_images PROPERTIES TIMEOUT 90 LABELS "hardware;interop;zluda;descriptors")
+  endif()
+endif()

LICENSE

@@ -1,73 +1,674 @@
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
 
-1. Definitions.
+                            Preamble
 
-"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document.
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
 
-"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License.
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
 
-"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
 
-"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License.
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
 
-"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files.
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
 
-"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types.
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
 
-"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below).
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
 
-"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
 
-"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution."
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
 
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work.
+  The precise terms and conditions for copying, distribution and
+modification follow.
 
-2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form.
+                       TERMS AND CONDITIONS
 
-3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed.
+  0. Definitions.
 
-4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions:
+  "This License" refers to version 3 of the GNU General Public License.
 
-     (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
 
-     (b) You must cause any modified files to carry prominent notices stating that You changed the files; and
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
 
-     (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
 
-     (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License.
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
 
-     You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License.
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
 
-5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions.
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
 
-6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file.
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
 
-7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License.
+  1. Source Code.
 
-8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages.
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
 
-9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability.
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
 
-END OF TERMS AND CONDITIONS
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
 
-APPENDIX: How to apply the Apache License to your work.
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
 
-To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
 
-Copyright [yyyy] [name of copyright owner]
+  The Corresponding Source for a work in source code form is that
+same work.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+  2. Basic Permissions.
 
-http://www.apache.org/licenses/LICENSE-2.0
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

NOTICE

@@ -1,7 +1,8 @@
 d4r
 Copyright 2026 countervolts
 
-This product is licensed under the Apache License, Version 2.0 (see LICENSE), except for the files
+This product is licensed under the GNU General Public License, version 3 only
+(GPL-3.0-only; see LICENSE), except for the files
 listed below.
 
 patches/zluda/*.patch

README.md

@@ -1,10 +1,18 @@
 # d4r (dlss 4 radeon)
 
-d4r runs NVIDIA's official DLSS Super Resolution library (`nvngx_dlss.dll`) in Windows games on AMD Radeon GPUs under Linux and Proton. The game asks for DLSS as usual; the DLSS network runs on the AMD GPU through [ZLUDA](https://github.com/vosen/ZLUDA) (CUDA on ROCm/HIP), with the heaviest DLSS kernels replaced by hand-written RDNA3 code.
+d4r runs NVIDIA's official DLSS Super Resolution library (`nvngx_dlss.dll`) in Windows games on AMD Radeon GPUs under Linux and Proton. The game asks for DLSS as usual; the DLSS network runs on the AMD GPU through [ZLUDA](https://github.com/vosen/ZLUDA) (CUDA on ROCm/HIP), with the heaviest DLSS kernels replaced by hand-written RDNA3 and RDNA4 code.
 
-**Supported DLSS models:** DLSS 3 CNN (E), DLSS 4 transformer (K, default), and DLSS 4.5 transformer (M). DLSS 5 is not supported.
+**Supported DLSS models:** DLSS 3 CNN (E), DLSS 4 transformer (K, default), and DLSS 4.5 transformer (M, plus experimental L). DLSS 5 is PURPOSELY not supported.
 
-It is an experiment. It has been tested on one GPU in a handful of games, it depends on unreleased patches to ZLUDA and vkd3d-proton.
+Preset L is NVIDIA's DLSS 4.5 model for Ultra Performance, especially at 4K. Set `[DLSS] Model = L` in `d4r.ini` and select Ultra Performance in game or OptiScaler; selecting L alone does not change the input resolution. L shares M's native Swin layers and has its own texture-kernel build. Its unfolded input/output arithmetic remains translated, so performance and image quality need game testing; the M benchmarks below do not apply to L.
+
+**Proof of concept:** d4r shows that DLSS can run on an AMD GPU, but it is not really that practical for everyday use yet. It has been tested on one GPU in a handful of games and depends on unreleased patches to ZLUDA and vkd3d-proton.
+
+**Windows (experimental):** d4r also runs natively on Windows 11, in D3D12 games, with no Wine or Proton. It supports the DLSS 4 (K) and DLSS 4.5 (M) models. Installing works like the Linux release: extract the ZIP into the game folder, copy in two NVIDIA files, and edit `d4r\d4r.ini`. See [the Windows guide](packaging/windows/WINDOWS_README.txt); `d4r\d4r-check.ps1` checks an install and writes a bug-report ZIP.
+
+Only the Radeon RX 9070 XT (gfx1201) has been tested on real hardware here. Its backend checks pass for K and M in Silent Hill 2 and Cyberpunk 2077 at 4K; those runs are correctness checks, not FPS benchmarks, and Windows performance is still under development. Community logs from an RX 9060 XT (gfx1200) show native K/M execution and finite output in Cyberpunk and Dawnwalker, but texture/LOD and motion artifacts are still reported, so that is backend coverage and not completed image-quality validation. The other targets (RDNA3 `gfx1100..gfx1103`, RDNA3.5 `gfx1150..gfx1154`) are compile-tested only. Windows preset E and L are not available. NVIDIA DLLs are supplied by you and are never part of the repository; the quick launcher can use a distributor-supplied DLL pair, but that packaging support does not establish redistribution permission.
+
+More: [Windows GPU coverage](docs/windows-gpu-support.md), [measurements](docs/windows-performance.md), [build and validation notes](docs/windows-rdna4-port.md), and the [diagnostic launcher](docs/windows-game.md) and [quick test](docs/windows-quick-test.txt) used for bug reports.
 
 See [supported games](SUPPORTED_GAMES.md) for the tested games and DLSS models.
 
@@ -16,12 +24,14 @@ See [supported games](SUPPORTED_GAMES.md) for the tested games and DLSS models.
 
 Radeon RX 7700 XT (RDNA3, gfx1101), SILENT HILL Townfall at 2560×1440, every DLSS result is presented in the frame it belongs to (no added latency).
 
-| Mode | DLSS 3 CNN (preset E) | DLSS 4 (preset K) | DLSS 4.5 (preset M) | FSR 4 |
+| Mode | DLSS 3 CNN (preset E) | DLSS 4 (preset K) | DLSS 4.5 (preset M) | FSR 4\* |
 |---|---|---|---|---|
-| Quality (1705×960) | **72.4** fps | 69.4 | 51.5 | 76.0 |
-| Balanced (1488×837) | **80.5** | 76.6 | 59.6 | 84.5 |
-| Performance (1280×720) | **87.8** | 84.0 | 69.9 | 94.0 |
-| Ultra Performance (853×480) | 88.9 | **94.5** | 90.3 | 107.3 |
+| Quality (1705×960) | **71.7** fps | 67.9 | 49.3 | 76.0 |
+| Balanced (1488×837) | **80.8** | 75.9 | 58.0 | 84.5 |
+| Performance (1280×720) | **89.7** | 84.1 | 69.0 | 94.0 |
+| Ultra Performance (853×480) | 89.1 | **95.5** | 92.2 | 107.3 |
+
+\* FSR 4 was measured in an earlier session (2026-09-27); the DLSS columns on 2026-09-29, when the same machine ran about 2–4% slower overall. Run back to back on the same day, this release is faster than 0.1.1 at Quality with every model (E 71.2 → 71.9, K 67.9 → 68.1, M 48.4 → 49.3 fps), and more in games whose DLSS settings differ from Townfall's, whose kernel variants now run natively too.
 
 For reference, native 2560×1440 without upscaling (the game's TSR at 100%) runs at 49.1 fps on the same walk.
 
@@ -29,20 +39,25 @@ On the same GPU DLSS starts at a disadvantage: its networks were designed for NV
 
 ## Known issues
 
-- In some games native upscaling can show visible artifacting.
+- In some games using some models native upscaling can show visible artifacting.
+
+For users who prioritize fidelity over speed, set `[Kernels] PreferAccuracy = true` in `d4r/d4r.ini` and restart the game. It defaults to `false`. This selects accuracy variants of every native kernel and restores conservative translation settings. It aims to match NVIDIA's arithmetic; 1:1 image quality against RTX DLSS is not yet proven. See [native kernel numerics](docs/native-kernels.md#numerics).
 
 ## GPU support
 
-d4r needs RDNA3 (Radeon RX 7000 series), but full speed depends on the chip: the release ships its native kernels only for **gfx1101 (Navi 32)**, and on other GPUs DLSS 4 and 4.5 fall back to the much slower translated NVIDIA code.
+d4r builds for RDNA3 and RDNA4. A newly built release compiles native DLSS 4 and 4.5 network kernels for the targets below and selects the matching set at runtime. Only the RX 7700 XT has been tested by this project on a real GPU; an external video reports DLSS 4.5 running through d4r 0.1.2 on an RX 7900 XTX. RDNA4 runtime and performance remain unverified on hardware.
 
-| GPU | Chip | DLSS 3 CNN (E) | DLSS 4 (K) | DLSS 4.5 (M) |
-|---|---|---|---|---|
-| RX 7700 XT (tested), RX 7800 XT, RX 7700, Radeon PRO W7700 | gfx1101 | full speed | full speed | full speed |
-| RX 7900 GRE / XT / XTX, Radeon PRO W7800 / W7900 | gfx1100 | full speed | runs without its transformer (worse image, slower) | about half speed |
-| RX 7600 / 7600 XT, RX 7650 GRE | gfx1102 | full speed | runs without its transformer (worse image, slower) | about half speed |
-| RDNA4, RDNA2 and older | | not supported | not supported | not supported |
+| GPU | Chip | FP8 math | Runtime testing |
+|---|---|---|---|
+| RX 7700 XT, RX 7800 XT, RX 7700, Radeon PRO W7700 | gfx1101 | widened to f16 | RX 7700 XT only |
+| RX 7900 GRE / XT / XTX, Radeon PRO W7800 / W7900 | gfx1100 | widened to f16 | RX 7900 XTX: [external video](https://www.youtube.com/watch?v=_GLjJ2Dn5pU) (DLSS 4.5); other cards untested |
+| RX 7600 / 7600 XT, RX 7650 GRE | gfx1102 | widened to f16 | untested |
+| RDNA3 integrated GPUs | gfx1103 | widened to f16 | untested |
+| RX 9070 XT / 9070 / 9070 GRE, Radeon AI PRO R9700 | gfx1201 | native FP8 (`NativeFp8`, default on) | RX 9070 XT: [external video](https://www.youtube.com/watch?v=lZ0BLYqAtCs), other cards untested |
+| RX 9060 XT / 9060 | gfx1200 | native FP8 (`NativeFp8`, default on) | compile only |
+| RDNA2 and older | | | unsupported |
 
-Only the RX 7700 XT has been tested; the other gfx1101 cards use the same chip. Without the native kernels, NVIDIA's translated DLSS 4 layers produce invalid values and DLSS discards the network's output, so K loses the transformer's image quality. Preset E is unaffected on any RDNA3 GPU. The kernels can be built for gfx1100 and gfx1102 (`D4R_GPU_ARCHS` in `scripts/package_release.sh`), but those builds are untested.
+Native network kernels are built for all listed gfx11/gfx12 targets. Texture kernels are compiled for each target by `d4r_emit` when supplied to the package script; RDNA4 also has a `-fp8` variant. The bridge selects the KFD GPU with the most SIMDs, avoiding an integrated GPU when a discrete GPU is present; `D4R_GPU_ARCH` overrides that choice. Missing native kernels fall back to ZLUDA and can be much slower. Earlier translated K layers produced invalid values; preserving FP16 denormal handling fixes that failure in recorded captures, but RTX image-quality parity remains unverified. Preset E does not use the native network kernels.
 
 ## How it works
 
@@ -57,13 +72,13 @@ game (D3D12) ──► OptiScaler (DLSS inputs) ──► d4r_nvngx.dll  (NGX D3
                                                    ▼
                           ZLUDA  (patches/zluda: PTX → AMDGPU, WMMA, native kernel overrides)
                                                    ▼
-                          native RDNA3 kernels  (kernels/: DLSS 4 and 4.5 network layers)
+                          native RDNA3 and RDNA4 kernels  (kernels/: DLSS 4 and 4.5 network layers)
 ```
 
 - **The shim** (`d4r_nvngx.dll`) implements the D3D12 NGX entry points OptiScaler calls, copies the game's colour, depth and motion vectors into buffers shared with HIP, evaluates DLSS through the CUDA version of NGX, and writes the result back into the game's output texture.
 - **The bridge** is a Wine builtin `nvcuda.dll` that forwards the CUDA driver API to ZLUDA on the Linux side, plus a few helpers the shim needs (Vulkan memory import, asynchronous array copies, GPU-side waits).
-- **ZLUDA** compiles NVIDIA's PTX for the AMD GPU. The patches add what DLSS needs (textures, surfaces, FP8 and tensor-core MMA on RDNA3 WMMA) and a hook that serves hand-written kernels in place of selected PTX kernels.
-- **Native kernels** reimplement the DLSS 4 and 4.5 network layers for RDNA3 and replace parts of a few texture-heavy kernels. They are 2–3× faster than the translated versions and are checked against numpy reference models of the layers.
+- **ZLUDA** compiles NVIDIA's PTX for the AMD GPU. The patches add what DLSS needs (textures, surfaces, FP8 and tensor-core MMA on RDNA3/RDNA4 WMMA) and a hook that serves hand-written kernels in place of selected PTX kernels.
+- **Native kernels** reimplement the DLSS 4 and 4.5 network layers for RDNA3 and RDNA4 and replace parts of a few texture-heavy kernels. The 2–3× speedup was measured on the RX 7700 XT; RDNA4 speed is unverified.
 
 Details: [docs/architecture.md](docs/architecture.md) and [docs/native-kernels.md](docs/native-kernels.md).
 
@@ -77,11 +92,13 @@ The release zip works like an OptiScaler release: its contents go into the folde
    - an `d4r` folder with the shim, the CUDA bridge, ZLUDA, the ROCm 7.2.4 runtime, the native kernels, this game's `d4r.ini`, and NVIDIA's `nvngx_dlss.dll` (310.7) and `_nvngx.dll`.
 2. In Steam, select GE-Proton 11 for the game and set these launch options: `PROTON_FORCE_NVAPI=1 DXVK_NVAPI_GPU_ARCH=AD100 %command%`.
 
+When updating, replace `d3d12.dll` and `d3d12core.dll` together with `d4r/nvngx.dll`. The shim requires their matching resource-lifetime extension.
+
 ROCm does not need to be installed: the zip includes its runtime (from AMD's Ubuntu 22.04 packages, which run under Steam's container runtime on any distribution). The zip's NVIDIA files and the kernels built from NVIDIA's code are not covered by this repository's license (see [NOTICE](NOTICE)). The Proton prefix and the system are not changed. [packaging/D4R_README.txt](packaging/D4R_README.txt) is the full guide that ships in the zip; `scripts/package_release.sh` builds the zip (see [docs/building.md](docs/building.md#7-package-a-release)).
 
 ## Requirements (building from source)
 
-- Linux with an AMD RDNA3 GPU. The native kernels use gfx11 WMMA; they are built for gfx1101 by default (`D4R_GPU_ARCH` for other RDNA3 chips, untested). Without them DLSS runs through ZLUDA alone: much more slowly, and DLSS 4 (K) without its transformer (see [GPU support](#gpu-support)).
+- Linux with an AMD RDNA3 or RDNA4 GPU. The native kernels use gfx11 or gfx12 WMMA. `scripts/package_release.sh` builds network kernels for gfx1100–gfx1103 and gfx1200–gfx1201 by default; `D4R_GPU_ARCH` selects one target for a standalone `kernels/build.sh` invocation. Runtime support outside the RX 7700 XT remains unverified on hardware (see [GPU support](#gpu-support)).
 - ROCm with HIP and its clang (tested with ROCm 7.2).
 - GE-Proton with OptiScaler integration (tested with GE-Proton11-3).
 - Build tools: a Rust toolchain and git-lfs (ZLUDA), meson and ninja (vkd3d-proton), `winegcc`/`winebuild` (bridge), `x86_64-w64-mingw32-g++` and `clang-cl` (shim), Python 3.
@@ -91,27 +108,58 @@ ROCm does not need to be installed: the zip includes its runtime (from AMD's Ubu
 
 The full sequence is in [docs/building.md](docs/building.md). In short:
 
-1. Build ZLUDA `ee2f25a` with `patches/zluda/0002`–`0005` applied.
+1. Build ZLUDA `ee2f25a` with `patches/zluda/0002`–`0007` applied, including its `d4r_emit` example for offline texture builds.
 2. Build the patched vkd3d-proton: `scripts/build_vkd3d_proton_d4r.sh OUT_DIR`.
 3. Build the shim and bridge: `scripts/build_d4r_nvngx_shim.sh`, `scripts/build_wine_nvcuda_bridge.sh`.
 4. Stage the runtime with your NVIDIA files: `scripts/install_d4r_runtime.sh _nvngx.dll nvngx_dlss.dll`.
-5. Build the native kernels: `D4R_ROCM_DIR=… D4R_DLSS_DLL=… D4R_ZLUDA_BUILD=… kernels/build.sh`.
+5. Build the native kernels: `D4R_ROCM_DIR=… D4R_DLSS_DLL=… D4R_ZLUDA_EMIT=… kernels/build.sh`.
 6. Fill in `~/.config/d4r/d4r.ini` (created from `config/d4r.ini.default` on first launch) and start the game with `scripts/d4r_play.sh`.
 
 The launcher backs up and restores everything it touches in the Proton prefix: OptiScaler.ini, the prefix's `d3d12.dll`/`d3d12core.dll`, and the game's Engine.ini when cvars are set.
 
+## Preset videos
+
+Gameplay recordings of each DLSS model at the Quality, Performance and Ultra Performance presets.
+
+**Ready or Not**
+
+| Model | Quality | Performance |
+|---|---|---|
+| DLSS 4.5 | [video](https://cdn.ayois.gay/sk-ant-api03-OPG1vqqsmDkNduG5fXJBelDI-1APyL9_ytJrnOSekZxoh-2NL8NqNuqL3pCI24kJQsudaB896GYpu5TFv93tabAgK1H2sAA) | [video](https://cdn.ayois.gay/sk-ant-api03-d-4XS1it5mtMkm-QRz0Qnkfu9FAjz_dFJURvsh_3gPZTnC1sljpWc28F6fyjUWIC6xlZm-FckWGjaP-KuIipYmroa3dQ8AA) |
+| DLSS 4 | [video](https://cdn.ayois.gay/sk-ant-api03-uXTWm4hN5jZksiSm4fvgIOr0dw43hGJ0TpZLe2ZghIbGb7aR9CGGlTV-1paBYO7G6lzIf83tOrCdaKdJerJ4re6epRWDEAA) | [video](https://cdn.ayois.gay/sk-ant-api03-fcZCkctdFOv513z1u6rPBrRseY47Qnw43Gnv5mZf_GekHw-yUoWYcBixjp2qqqbBIf7sVmht0Y676U96n2cwmFIsRgWkHAA) |
+| DLSS 3 | [video](https://cdn.ayois.gay/sk-ant-api03-_vuywLcsMzZiJxcNgucBeeXFzWi44mes5A1777bg_4XQ2X0MH22J9M7JFpTS0_q6yaGWMXf3FvL8MDiv_dc7UANxrCmlGAA) | [video](https://cdn.ayois.gay/sk-ant-api03-WzaIy34WIesQeOOTVQNXZwpPPE8khmbUynvMw9B-Luj8GFcZ_sJvGXfg6aW00TD0Rr3nWu43ELJGx2K6qs_HL6FkLdmb1AA) |
+
+**Townfall**
+
+| Model | Quality | Ultra Performance |
+|---|---|---|
+| DLSS 4.5 | [video](https://cdn.ayois.gay/sk-ant-api03-p8uL0hOUklvA-BSeEafS8LQU-gayzbqAE3NacWAmckLoXOSf2sXGf7OeG1LmvOqeGFRh_ceCWi4pLBl9yxJlvYwQWH3Y1AA) | [video](https://cdn.ayois.gay/sk-ant-api03-Mz0BzYTrLjaYSuQXtOEHTw4ydYVbPvHmMjflm8UhFp976ftXqMExD8BCMYlo93JLEGwVL3rMWoZR0CH_deJJU7jX8_s3GAA) |
+| DLSS 4 | [video](https://cdn.ayois.gay/sk-ant-api03-WgNoUPTwH0vt55Ag7RaWprXXINU5OOOEtuDXOvIjHgfAScSiIPIJGVaERoF-EA3JO7LS8xkohUlb6AFsW8AVZ2vMB14JwAA) | [video](https://cdn.ayois.gay/sk-ant-api03-skXZVD6G4DUdAfQw0RLz_UJW8GOfBlpZx1N4ezaF_UIXm7OIY1ciQysfRu4zRsxSXA8plZJi4NSzkXkV7Oxmg9ytg9zaaAA) |
+
 ## Repository layout
 
 | Path | Contents |
 |---|---|
 | `tools/` | the NGX shim, the Wine CUDA bridge, a D3D12 DLSS harness, probes and kernel replay tools |
-| `kernels/` | native RDNA3 kernels (`k/` DLSS 4, `m/` DLSS 4.5, `tex/` texture-kernel parts), their build script, validation tools and numpy reference models |
+| `kernels/` | native RDNA3/RDNA4 kernels (`k/` DLSS 4, `m/` DLSS 4.5, `tex/` texture-kernel parts, `common/` WMMA layouts), their build script, validation tools and numpy reference models |
 | `patches/` | ZLUDA and vkd3d-proton patches |
 | `scripts/` | build, install, launch and probe scripts |
 | `config/` | the default `d4r.ini` for the developer launcher |
 | `packaging/` | the release's `d4r.ini`, OptiScaler settings, user guide and install check |
 | `docs/` | architecture, build, native kernel and performance notes |
 
+## Star History
+
+<a href="https://www.star-history.com/?type=date&repos=countervolts%2Fd4r">
+ <picture>
+   <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=countervolts/d4r&type=date&theme=dark&legend=top-left" />
+   <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=countervolts/d4r&type=date&legend=top-left" />
+   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=countervolts/d4r&type=date&legend=top-left" />
+ </picture>
+</a>
+
 ## License
 
-Apache License 2.0 (see [LICENSE](LICENSE)). The patches in `patches/` are offered under the licenses of the projects they modify: ZLUDA (Apache-2.0 or MIT) and vkd3d-proton (LGPL-2.1). See [NOTICE](NOTICE).
+GNU General Public License, version 3 only (GPL-3.0-only; see [LICENSE](LICENSE)). The patches in `patches/` are offered under the licenses of the projects they modify: ZLUDA (Apache-2.0 or MIT) and vkd3d-proton (LGPL-2.1). See [NOTICE](NOTICE).
+
+To contact me my discord is `._ayo`.

SUPPORTED_GAMES.md

@@ -19,6 +19,10 @@ The following DirectX 12 games run DLSS Super Resolution through d4r on the test
 | Ratchet & Clank: Rift Apart | Works; occasional artifacts in the pause menu | Works; minor artifacts with HDR enabled and occasionally in the pause menu | Works; occasional artifacts in the pause menu |
 | Dying Light: The Beast | Works | Works | Works; small visual artifacts throughout |
 
+Unsupported games:
+
+- **DOOM: The Dark Ages:** the game uses Vulkan and has no D3D12 mode. d4r currently supports D3D12 DLSS only, so its DLSS calls cannot reach d4r. The game's hardware ray-tracing requirement is separate from this limitation.
+
 General notes:
 
 - Select DLSS in the game's graphics settings where it is offered. OptiScaler can also feed DLSS from a game's FSR or XeSS inputs, but those are a fallback for games that hide DLSS and can ghost or show a black screen.

config/d4r.ini.default

@@ -17,14 +17,20 @@ CompatDataDir =
 GameExe =
 
 [DLSS]
-; Which DLSS network ("preset") to run. fps measured on an RX 7700 XT, Townfall 2560x1440, same street
-; walk, for Quality / Balanced / Performance / Ultra Performance (FSR 4: 76 / 85 / 94 / 107):
-;   E (or CNN)     = DLSS 3 CNN, the fastest at Quality..Performance  72 / 81 / 88 / 89
-;   K (or DLSS4)   = DLSS 4 transformer, sharper; best at Ultra Perf  69 / 77 / 84 / 95
-;   M (or DLSS4.5) = DLSS 4.5 transformer, best image, slowest        52 / 60 / 70 / 90
-;   F = the other CNN (same speed as E); A, B, C, D, G, J run the K network in this DLSS version;
-;   L (DLSS 4.5 Ultra Performance model) is not accelerated here and runs very slowly.
-;   auto = whatever the game/OptiScaler asks for
+; DLSS model recommendations:
+;
+; Model | DLSS  | Recommended modes               | Notes
+; ------+-------+---------------------------------+----------------------------------
+; K     | 4     | DLAA / Quality / Balanced       | Default; faster alternative to M/L
+; M     | 4.5   | Performance                     | More demanding on RDNA3
+; L     | 4.5   | Ultra Performance               | Especially 4K; experimental
+; E     | 3 CNN | Quality / Balanced / Performance | Legacy faster fallback
+; F     | 3 CNN | DLAA / Ultra Performance        | Deprecated; prefer K or L
+;
+; Select the upscaling mode separately in game/OptiScaler. These are recommendations;
+; other modes are allowed. L requires the L texture build.
+; Aliases: CNN = E, DLSS4 = K, DLSS4.5 = M. A/B/C/D/G/J use K in this DLL version.
+; auto = use the game/OptiScaler choice. Benchmarks: docs/performance.md.
 Model = K
 [Latency]
 ; How many frames old the upscaled image the game presents may be.
@@ -36,14 +42,28 @@ Model = K
 FrameAge = 0
 
 [Kernels]
-; Native RDNA3 kernels for the DLSS 4 and 4.5 network layers (replace ZLUDA's translated code).
-;   fast  = native kernels with relaxed numerics (visually identical to exact, ~15% faster)
-;   exact = native kernels that reproduce NVIDIA's FP8/f16 rounding bit for bit
+; Prefer NVIDIA's original arithmetic over speed for every native kernel and the translated fallback.
+; Restores intermediate FP8 quantization, per-MMA f16 rounding and original denormal handling; keeps
+; NGX's synchronizations. Selects accuracy variants in NativeKernelDir*/accuracy (or a marked accuracy
+; directory directly). Missing accuracy kernels use translated PTX, never the fast set. Slower; this
+; aims for fidelity, but RTX image parity is not proven. (D4R_PREFER_ACCURACY)
+PreferAccuracy = false
+; false: use original translated enc1/enc2 layers for L/M, retaining other native kernels.
+; Reduces the native encoders' motion trails seen in Townfall; best paired with PreferAccuracy.
+; Costs some GPU time. Manifest required for selective fallback; flat sets fall back entirely.
+; (D4R_NATIVE_SWIN_ENCODERS)
+NativeSwinEncoders = true
+; Native RDNA3 / RDNA4 kernels for the DLSS 4 and 4.5 network layers (replace ZLUDA's translated code).
+;   fast  = native kernels with relaxed numerics (unless PreferAccuracy is true)
+;   exact = a separately built directory with conservative FP8/f16 rounding; not an RTX parity guarantee
 ;   off   = ZLUDA-translated kernels only
 ; fast/exact need the code objects built by the native-kernel tools ([Paths] NativeKernelDir*).
 NativeKernels = off
 ; FP8 tensor-core emulation on RDNA3 WMMA (much faster than the scalar path).
 Fp8Wmma = true
+; RDNA4 only: FP8 MMAs on the native FP8 WMMA (and the -fp8 native kernel variants); off widens FP8 to f16 as
+; on RDNA3. Ignored on other GPUs. (D4R_ZLUDA_WMMA_FP8_NATIVE)
+NativeFp8 = true
 ; Drop per-instruction flush-to-zero mode switches in ZLUDA-compiled kernels
 ; (only values below ~1e-38 differ; a little faster).
 IgnoreDenormals = true
@@ -51,6 +71,12 @@ IgnoreDenormals = true
 ImplicitMaxBlock = 256
 
 [Interop]
+; Experimental foreground motion coverage for thin geometry. 0 = original vectors;
+; 1/2 = opaque-surface dilation radius in render pixels; both automatically fill sky gaps
+; from the nearest foreground within 8 render pixels. Needs render- or display-resolution, unjittered motion vectors
+; and same-frame linear VRAM inputs; unsupported features retain original vectors.
+; Default off. May affect disocclusion edges. (D4R_MOTION_DILATION)
+MotionVectorDilation = 0
 ; Keep DLSS inputs and output in VRAM (vkd3d-proton Vulkan interop) instead of copying through
 ; system memory. Falls back automatically when the game's resources do not qualify.
 VramInterop = true
@@ -58,19 +84,18 @@ VramInterop = true
 InputSync = true
 ; Faster hand-offs between the game and DLSS (same frame, same result). Need FrameAge = 0 and VRAM
 ; interop, and a runtime built from this checkout (scripts/install_d4r_runtime.sh).
-;   GpuWait: DLSS is queued ahead and starts on the GPU the moment the game's inputs are ready
-;   LinearInputs: DLSS reads the game's inputs in place (no input copies; needs GpuWait)
+; Input readiness is checked with CPU sleeps; recorded-but-discarded lists never queue GPU waits.
+;   LinearInputs: DLSS reads the game's inputs in place once the input marker is ready
 ;   DirectOutput: the result is written straight into the game-side buffer (preset K, and only while
 ;     NGX runs the native output kernel; drops NVIDIA's "DLSS SDK" watermark)
 ;   ElideNgxSync: removes NGX's internal CPU waits between its GPU stages
 ;   EvalSync = false: no CPU wait between DLSS and the output copy
-;   MarkerPollUs: how often (microseconds) the CPU checks for the game's inputs without GpuWait
-GpuWait = true
+;   MarkerPollUs: how often (microseconds) the CPU checks for the game's inputs
 LinearInputs = true
 DirectOutput = true
 ElideNgxSync = true
 EvalSync = false
-MarkerPollUs = 20
+MarkerPollUs = 200
 
 [Paths]
 ; auto = defaults of scripts/d4r_proton_env.sh
@@ -124,3 +149,4 @@ LogDir = auto
 [Env]
 ; Anything else, passed through as environment variables (one per line):
 ;   D4R_SHIM_SPLIT_TIMEOUT_MS = 200
+;   D4R_SHIM_BLOCKING_SYNC = 0  ; A/B only: restore context waits instead of default event queries/sleeps

docs/architecture.md

@@ -12,7 +12,7 @@ d4r keeps NVIDIA's DLSS library unmodified and gives it what it expects: an NGX
 | CUDA bridge `nvcuda.dll` | `tools/wine_nvcuda_bridge.c` | A Wine builtin whose Windows exports forward to ZLUDA's Linux `libcuda.so`, plus d4r helpers (Vulkan memory import, asynchronous 2D copies, GPU-side waits, output redirect, linear textures) |
 | ZLUDA | `patches/zluda` | CUDA driver API on HIP; compiles NVIDIA's PTX to AMDGPU code; serves native kernels in place of PTX kernels |
 | Native kernels | `kernels/` | RDNA3 implementations of the DLSS network layers and parts of texture kernels |
-| vkd3d-proton patch | `patches/vkd3d-proton` | Lets the shim split a D3D12 command list around the DLSS call |
+| vkd3d-proton patch | `patches/vkd3d-proton` | Splits a D3D12 command list around DLSS and retains external resources through command allocator completion |
 | NVAPI identity | `tools/d4r_nvapi_identity.c` | Reports an Ada (sm_89) GPU so NGX selects the network weights matching the PTX ZLUDA compiles |
 
 ## One frame
@@ -24,23 +24,22 @@ With VRAM interop and split frames (the defaults when the patched vkd3d-proton i
    - A frame marker.
    - A split. vkd3d-proton submits everything the game records after the DLSS call separately, gated on a timeline semaphore reaching this frame's number.
    - At the start of that second part, a copy of the DLSS result into the game's output texture. Linear R11G11B10, RGB10A2, and RGBA8/BGRA8 outputs are converted from RGBA16F on the GPU. sRGB formats continue through host staging to preserve their existing byte interpretation.
-2. **The CUDA worker queues DLSS.** A worker thread owns the CUDA context. For each frame it queues on the GPU, without waiting on the CPU:
-   - a wait for the marker (`GpuWait`);
+2. **The CUDA worker queues DLSS.** The prep thread checks the GPU-written input marker with short CPU sleeps. It drops recorded-but-unsubmitted frames when they time out or their feature retires, before any DLSS work is queued. The CUDA worker then queues:
    - the input copies into CUDA arrays, or none when NGX samples the buffers in place (`LinearInputs`);
    - NGX's evaluation (NGX's own CPU synchronisations are elided with `ElideNgxSync`);
    - the result, written straight into the output buffer by the native output kernel (`DirectOutput`, preset K) or copied there.
 3. **Release.** When the GPU finishes, the worker signals the timeline semaphore and the rest of the game's frame runs, now with this frame's result.
 
-A watchdog releases any frame that never completes (for example a command list that is recorded but never executed), so the game cannot stall on DLSS.
+The watchdog reports delayed output but never signals an unfinished producer as ready. Retiring features remain registered until their CPU/CUDA jobs drain. The lifetime-aware vkd3d extension retains their staging buffers, imported buffers, conversion images and split semaphore through command-allocator reset or destruction after GPU completion. Discarded recordings are covered too; cleanup no longer relies on a 50 ms delay.
 
-Without the vkd3d-proton patch the shim shows the newest finished result instead (one or more frames old). Without VRAM interop, inputs and output are staged through host memory.
+The shim requires the matching lifetime-aware d3d12.dll and d3d12core.dll, even for host staging. It rejects older runtimes rather than recording references whose lifetime it cannot protect. FrameAge > 0 shows the newest finished result; without VRAM interop, inputs and output are staged through host memory.
 
 ## Why each piece exists
 
 - **CUDA NGX instead of D3D12 NGX.** NVIDIA's D3D12 path needs NVIDIA's driver. The CUDA path only needs a CUDA driver API, which ZLUDA provides.
 - **The Ada identity.** NGX picks network weights by GPU architecture. The weights have to match the PTX that ZLUDA compiles, which is sm_89 for DLSS 310.
-- **Native kernels.** ZLUDA translates NVIDIA's warp-level matrix code faithfully but slowly (register pressure, lane shuffles, per-instruction mode switches). In the DLSS 4 transformer, ZLUDA's translation also produced non-finite outputs, which the final kernel masks, so the network had no effect. The native layers fix both.
-- **The GPU-side hand-offs.** With DLSS itself fast, most of the remaining gap to FSR 4 was the GPU idling while CPU threads waited on each other. Moving the waits onto the GPU raised GPU utilisation from about 95% to 99%.
+- **Native kernels.** Translating NVIDIA's warp-level matrix code is slow (register pressure, lane shuffles, per-instruction mode switches). Earlier translated K layers also produced non-finite outputs that the final kernel masked. Native layers avoid that path; the compiler now preserves FP16 denormal requirements too, which fixes the translated failure in recorded captures. Numerical and image-quality parity with RTX hardware remains unverified.
+- **Sleeping readiness checks.** Waiting for input submission before queueing DLSS avoids an uncancellable GPU wait. Explicit output event queries with short sleeps avoid HIP event/context waits that can consume a CPU core. Submillisecond waits use NtDelayExecution because MinGW std::sleep_for can truncate them to zero. Linear inputs and direct output remain available.
 
 ## Configuration
 
@@ -51,8 +50,8 @@ Without the vkd3d-proton patch the shim shows the newest finished result instead
 | `[DLSS] Model` | `D4R_DLSS_PRESET` | DLSS network: `E` (CNN), `K` (DLSS 4), `M` (DLSS 4.5) |
 | `[Latency] FrameAge` | `D4R_SHIM_SPLIT_FRAME`, `D4R_SHIM_MAX_IN_FLIGHT` | 0 = same-frame results; higher values pipeline frames |
 | `[Kernels] NativeKernels` | `D4R_ZLUDA_NATIVE_DIR` | native kernel directory (`fast` / `exact` / `off`) |
+| `[Kernels] PreferAccuracy` | `D4R_PREFER_ACCURACY` | default off; accuracy variants for every native family and conservative translation/synchronization settings |
 | `[Interop] VramInterop` | `D4R_SHIM_VRAM_INTEROP` | keep inputs and output in VRAM |
-| `[Interop] GpuWait` | `D4R_SHIM_GPU_WAIT` | GPU-side wait for the game's inputs |
 | `[Interop] LinearInputs` | `D4R_SHIM_LINEAR_INPUTS` | NGX samples the input buffers in place |
 | `[Interop] DirectOutput` | `D4R_SHIM_OUTPUT_DIRECT` | the output kernel writes the game-side buffer directly (preset K) |
 | `[Interop] ElideNgxSync` | `D4R_ELIDE_NGX_SYNC` | skip NGX's internal CPU waits |
@@ -88,3 +87,12 @@ The release zip (`scripts/package_release.sh`) is unpacked into the folder that
 ## Logs
 
 The launcher writes each run's logs to `~/.cache/d4r-dlss-captures/game-<exe>-<time>/`. `d4r_nvngx.log` holds the shim's log. With `[Debug] Profile = true` it also records one `D4R_PROFILE` line per frame, with the stage timings.
+
+Output completion uses event queries with 200 us NtDelayExecution sleeps by default. `D4R_SHIM_BLOCKING_SYNC = 0`
+under `[Env]` restores context synchronization for an A/B check. The profile records
+`output_sync_wall`, `output_sync_cpu`, and `output_sync_method=event_query_sleep` for the new path.
+An unavailable or failed event falls back to context synchronization. The old `GpuWait` setting
+is no longer used: input readiness is checked on the CPU before CUDA work is queued.
+
+The `d2h` field on this path includes the output synchronization wait, even when direct output
+skips the array-to-buffer copy.

docs/building.md

@@ -4,8 +4,8 @@ These steps produce the pieces the launcher needs: a patched ZLUDA, a patched vk
 
 | Component | Tested version |
 |---|---|
-| ZLUDA | `ee2f25a` (upstream), plus `patches/zluda/0002`, `0003`, `0004`, `0005` in that order |
-| vkd3d-proton | `3dfc6f07` (the base GE-Proton11-3 ships), plus `patches/vkd3d-proton/0001` |
+| ZLUDA | `ee2f25a` (upstream), plus `patches/zluda/0002` through `0007` in order |
+| vkd3d-proton | `3dfc6f07` (the base GE-Proton11-3 ships), plus `patches/vkd3d-proton/0001` and `0002` |
 | ROCm | 7.2 (HIP runtime, clang, device libraries) |
 | Proton | GE-Proton11-3 (with its OptiScaler integration) |
 | DLSS | `nvngx_dlss.dll` 310.7.0 |
@@ -17,15 +17,23 @@ git clone https://github.com/vosen/ZLUDA zluda && cd zluda
 git checkout ee2f25a
 git submodule update --init --recursive
 git lfs pull
-for p in 0002 0003 0004 0005; do git apply /path/to/d4r/patches/zluda/$p-*.patch; done
+for p in 0002 0003 0004 0005 0006 0007; do git apply /path/to/d4r/patches/zluda/$p-*.patch; done
 # rebuild the device helpers the patches changed (ptx/lib/zluda_ptx_impl*.bc)
 ZLUDA_SOURCE_ROOT=$PWD ROCM_ROOT=/opt/rocm /path/to/d4r/scripts/build_zluda_ptx_helpers.sh
 LIBRARY_PATH=/opt/rocm/lib cargo build --release -p zluda
+LIBRARY_PATH=/opt/rocm/lib cargo build --release -p ptx --example d4r_emit
 mkdir -p ~/.cache/d4r-zluda-current
 cp target/release/libnvcuda.so ~/.cache/d4r-zluda-current/
 ln -sf libnvcuda.so ~/.cache/d4r-zluda-current/libcuda.so
 ```
 
+Build the release ZLUDA library on a system or in a container with GLIBC 2.41 or older. A binary
+linked on a newer system can require newer GLIBC symbols even when the ZLUDA source does not need new
+GLIBC features. Before packaging, check it with
+`scripts/check_glibc_compat.sh 2.41 /path/to/libnvcuda.so`. The release packager runs the same
+check on ZLUDA, the Wine CUDA bridge, and every bundled ROCm library. Test the rebuilt library
+with the D3D12 harness and in a game before publishing a release.
+
 The launcher looks for ZLUDA in `D4R_ZLUDA_DIR` (default `~/.cache/d4r-zluda-current`, or `ZludaDir` in d4r.ini).
 
 What the patches add:
@@ -34,6 +42,10 @@ What the patches add:
 - **0003**: an implicit 256-thread launch bound for kernels without PTX bounds (removes massive register spilling, `D4R_ZLUDA_IMPLICIT_MAX_BLOCK`) and f16 tensor-core MMA on RDNA3 WMMA (`D4R_ZLUDA_WMMA`).
 - **0004**: the native kernel override hook (`D4R_ZLUDA_NATIVE_DIR`, see [native-kernels.md](native-kernels.md)), FP8 MMA on WMMA (`D4R_ZLUDA_WMMA_FP8`), optional elision of per-instruction denormal mode switches (`D4R_ZLUDA_IGNORE_DENORMAL`), inlined image helpers, and linking extra bitcode into a PTX module (`D4R_ZLUDA_EXTRA_BC`, used by the texture-kernel build).
 - **0005**: a null texture object (CUDA handle 0) reads as zeros, as on NVIDIA GPUs, instead of faulting the GPU. DLSS samples absent optional inputs that way in some configurations (low-resolution motion vectors without HDR, as in Ghost of Tsushima).
+- **0006**: `m16n8k8` f16 MMAs (DLSS 3 CNN, presets E/F) on RDNA3 WMMA like the k16 ones (`D4R_ZLUDA_WMMA_K8=0` disables it); weight-image slots for native prep kernels (`d4r_prep_key_at`, `d4r_prep_key_slots`); and wave64 compilation for offline texture-kernel builds (`D4R_ZLUDA_WAVE64=1`, with wave64 builds of the helper bitcode). `scripts/build_zluda_ptx_helpers.sh` now writes the `_w64` helper variants too.
+- **0007**: gfx12 WMMA layout lowering, optional native e4m3 FP8 WMMA (`D4R_ZLUDA_WMMA_FP8_NATIVE=1`), and an architecture argument for `d4r_emit`. `D4R_ZLUDA_WMMA_LAYOUT=12` on gfx11 is a validation shim, not a release setting.
+
+When linking on a system with ROCm libraries outside the default search path, include their library directory in `LIBRARY_PATH`. The build also needs the appropriate ROCm link libraries. `CARGO_BUILD_JOBS=8` caps parallel Rust compilation if memory is limited.
 
 ## 2. vkd3d-proton
 
@@ -65,11 +77,13 @@ This stages the shim, the bridge, the NGX core and the DLSS feature library in `
 ```sh
 D4R_ROCM_DIR=/opt/rocm \
 D4R_DLSS_DLL=/path/to/nvngx_dlss.dll \
-D4R_ZLUDA_BUILD=~/.cache/d4r-zluda-current \
+D4R_ZLUDA_EMIT=/path/to/zluda/target/release/examples/d4r_emit \
 kernels/build.sh all kernels/out/native
 ```
 
-`kernels/out/native` then holds one code object per replaced DLSS kernel; set `NativeKernelDirFast` in d4r.ini (or `D4R_ZLUDA_NATIVE_DIR`) to it. `kernels/build.sh k` or `m` builds only the network layers and needs neither the DLL nor ZLUDA. The texture kernels (`tex`) extract PTX from your DLL into `kernels/extracted/`; that directory and the build output are git-ignored and must not be redistributed.
+`kernels/out/native` then holds one code object per replaced DLSS kernel; set `NativeKernelDirFast` in d4r.ini (or `D4R_ZLUDA_NATIVE_DIR`) to it. `kernels/build.sh k` or `m` builds only the network layers and needs neither the DLL nor ZLUDA. `kernels/build.sh l` builds L's shared network layers and unfolded texture variants; it needs the DLL and emitter. The texture kernels (`tex`, including L) extract PTX from your DLL into `kernels/extracted/` and compile it offline for `D4R_GPU_ARCH` through `D4R_ZLUDA_EMIT`; that directory and the build output are git-ignored and must not be redistributed. On gfx12, build a second variant with `D4R_NATIVE_FP8=1` and place it in an `<arch>-fp8` folder.
+
+To add the accuracy option to a developer set, repeat the build with `D4R_PREFER_ACCURACY=1` and output `kernels/out/native/accuracy` (multi-target builds: `kernels/accuracy/<arch>` and `<arch>-fp8`). Use a separate empty directory; successfully built accuracy sets receive `d4r-accuracy.txt`. Enable `[Kernels] PreferAccuracy = true` and restart. The option defaults to false and never substitutes the fast set when accuracy binaries are absent.
 
 ## 6. Configure and play
 
@@ -89,18 +103,44 @@ D4R_OPTISCALER=/path/to/OptiScaler_0.9.4.7z \
 D4R_DLSS_DLLS=/path/to/310.7/nvngx_dlss.dll:/path/to/310.9/nvngx_dlss.dll \
 D4R_BUNDLE_DLSS=/path/to/nvngx_dlss.dll D4R_BUNDLE_NGX=/path/to/_nvngx.dll D4R_BUNDLE_TEX=kernels/out/native \
 D4R_ZLUDA_DIR=~/.cache/d4r-zluda-current D4R_VKD3D_DIR=~/.cache/d4r-vkd3d-d4r D4R_ROCM_DIR=/opt/rocm \
+D4R_ZLUDA_EMIT=/path/to/zluda/target/release/examples/d4r_emit \
 scripts/package_release.sh            # -> dist/d4r-<version>.zip
 ```
 
 The script:
-- builds the shim, the bridge and the network-layer kernels;
+- builds the shim, the bridge and both fast and accuracy network-layer kernels for gfx1100–gfx1103 and gfx1200–gfx1201 by default (`D4R_GPU_ARCHS` can select fewer targets); each gfx12 target also gets an `<arch>-fp8` variant;
 - writes the kernel manifest from the DLLs you list (it records hashes of their PTX, nothing else);
 - stages OptiScaler as `dxgi.dll` with the settings in `packaging/optiscaler.settings`;
-- adds NVIDIA's two DLLs and the texture kernels (built from NVIDIA's PTX by `kernels/build.sh tex`);
+- adds NVIDIA's two DLLs and any supplied texture kernels (built from NVIDIA's PTX by `kernels/build.sh tex`); builds accuracy texture sets with `D4R_ZLUDA_EMIT` unless marked prebuilt sets are supplied in `D4R_BUNDLE_TEX/accuracy/<target>`;
 - zips the result together with the ZLUDA and vkd3d-proton builds, the ROCm runtime (as `d4r/rocm`), `packaging/d4r.ini`, the licenses and the patches.
 
 The NVIDIA files are not covered by d4r's license; redistributing them is up to whoever publishes the zip. `D4R_BUNDLE_NVIDIA=0` builds `d4r-<version>-nonvidia.zip` without them and without the texture kernels. [architecture.md](architecture.md#portable-installs-the-release-zip) describes how the installed files work together.
 
+### GLIBC 2.41 release builds
+
+Build the pinned environment with:
+
+```sh
+docker build -f packaging/build/Dockerfile.glibc241 -t d4r-build:glibc241 packaging/build
+```
+
+Inside that image, `scripts/build_release_glibc241.sh` rebuilds the GPU device helpers, ZLUDA and its LLVM, `d4r_emit`, the NGX shim, Wine CUDA bridge, patched vkd3d-proton, and performance/accuracy texture sets for every release target. Packaging then builds both network sets, checks all bundled Linux libraries against 2.41, and records the toolchain in `d4r/source/BUILD_INFO.txt`.
+
+Mount the repository at `/work`, ROCm's compiler/headers/device libraries at `/opt/rocm`, and an official Rust toolchain (1.98.0 used for 0.1.3) at `/opt/rust`, with `/opt/rust/bin` on `PATH`. Keep toolchain and third-party runtime inputs read-only. Supply the normal package inputs listed above and these build settings:
+
+| Variable | Input |
+|---|---|
+| `D4R_RELEASE_BUILD_ROOT` | writable directory under `/work/build/` |
+| `D4R_ZLUDA_SRC` | isolated checkout inside that directory, with both pinned LLVM and HiGHS submodules, the real OCKL LFS payload, and patches `0002`–`0007` applied |
+| `D4R_VKD3D_SRC` | isolated vkd3d-proton checkout inside that directory, with its submodules and patches `0001` and `0002` applied |
+| `CARGO_HOME` | writable build-local Cargo cache; prefetch the locked dependencies for an offline build |
+| `D4R_ROCM_DIR` | `/opt/rocm` |
+| `D4R_ROCM_LINK_STUBS` | optional directory for additional ROCm link libraries |
+
+Run `bash scripts/build_release_glibc241.sh` inside the container. The script checks that the build system actually reports `glibc 2.41`, rejects external source-cache checkouts, builds with the locked Cargo dependencies offline, and runs the configuration tests before packaging. The ZIP and checksum go in `$D4R_RELEASE_BUILD_ROOT/dist/`; logs and `toolchain.txt` remain in the build directory. AMD's bundled ROCm runtime, OptiScaler and NVIDIA's libraries remain supplied binaries; the Linux ABI check covers the bundled ROCm libraries too.
+
+`D4R_BUNDLE_TEX` accepts the older flat directory for gfx1101, or a directory with per-target subdirectories (`gfx1100/`–`gfx1103/`, `gfx1200/`, `gfx1201/`, plus `gfx1200-fp8/` and `gfx1201-fp8/`). Build each texture set with `D4R_GPU_ARCH` and `D4R_ZLUDA_EMIT`; the `-fp8` sets also need `D4R_NATIVE_FP8=1`. `d4r_emit` targets those GPUs offline. Missing texture kernels fall back to ZLUDA on that target; the network-layer kernels are still included. Only the RX 7700 XT has been tested on real hardware.
+
 ## Checks
 
 - `scripts/check_environment.sh` lists the tools, GPU and Proton builds it finds.

docs/native-kernels.md

@@ -1,6 +1,6 @@
 # Native kernels
 
-The native kernels are RDNA3 (gfx11) code objects that ZLUDA runs in place of specific DLSS kernels. Each one has the same name, launch shape and parameter block as the PTX kernel it replaces, and reads the same buffers, including the network weights inside NVIDIA's DLL. No weights or NVIDIA code are stored here.
+The native kernels are RDNA3 (gfx11) and RDNA4 (gfx12) code objects that ZLUDA runs in place of specific DLSS kernels. Each one has the same name, launch shape and parameter block as the PTX kernel it replaces, and reads the same buffers, including the network weights inside NVIDIA's DLL. No weights or NVIDIA code are stored here.
 
 ## The override hook (patches/zluda/0004)
 
@@ -10,7 +10,9 @@ With `D4R_ZLUDA_NATIVE_DIR=DIR`, whenever NGX loads a PTX module and asks for ke
 |---|---|
 | `NAME_prep` kernel | launched first on the same stream, with the same parameters (typically re-lays out weights) |
 | `u32 d4r_prep_blocks` | grid size of the prep kernel (128 threads per block) |
-| `u32 d4r_prep_key_offset` | byte offset of a u64 in the parameter block; the prep is skipped while that value (the weights pointer) repeats |
+| `u32 d4r_prep_key_offset` | byte offset of a u64 in the parameter block; the prep is skipped while that value (the weights pointer) repeats (0 = no key) |
+| `u32 d4r_prep_key_at` | the same as offset + 1, so a key at offset 0 can be named (M's layers keep their weights pointer there) |
+| `u32 d4r_prep_key_slots` | N > 0: the kernel keeps one prepared image per key for up to N keys, so the prep runs once per weight set (the six enc3 tube blocks); keys beyond N share an overflow image and are prepared on every launch |
 | `u32 d4r_block_z` | replaces the launch's block z dimension (more waves per window) |
 | `u32 d4r_grid_x` | replaces the grid (persistent kernels) |
 
@@ -19,7 +21,7 @@ Kernels without a file in `DIR` are compiled from PTX as usual, so a partial set
 **Release directories.** A directory that holds `d4r-kernels.txt` (or `<gfx target>/d4r-kernels.txt`) is checked before use:
 - `kernels/tools/kernel_manifest.py DIR nvngx_dlss.dll...` writes the manifest. For each `NAME.hsaco` it records the FNV-1a 64 hash of the PTX module that defines `.entry NAME`, taken from each DLL given.
 - The CUDA bridge serves a kernel only after NGX has loaded a module with one of the listed hashes.
-- It picks the target folder from the GPU's KFD topology entry.
+- It picks the KFD GPU with the most SIMDs, then selects that GPU's target folder. `D4R_GPU_ARCH` overrides the choice. On gfx12, `D4R_ZLUDA_WMMA_FP8_NATIVE=1` selects the `<arch>-fp8` folder.
 
 The release is built this way, with hashes from DLSS 310.7.0 and 310.9.1. The PTX of every replaced kernel is identical in those two versions.
 
@@ -33,9 +35,27 @@ The release is built this way, with hashes from DLSS 310.7.0 and 310.9.1. The PT
   - `pwin_wide.h`: the deep layers, which have only 6–77 windows per frame. It uses 4·NG waves per window, split by token tile and channel group, so the GPU stays busy.
 - **Weights:** prep kernels expand the weights into WMMA operand images once.
 
-**DLSS 4.5, preset M (`kernels/m`, `rrlite_*`).**
-- **Network:** the Swin blocks of DLSS 4.5, whose weights are FP8. Their prep kernels expand the weights to f16 WMMA operands.
+**DLSS 4.5, presets L/M (`kernels/m`, `rrlite_*`).**
+- **Network:** the Swin blocks of DLSS 4.5, whose weights are FP8. Their prep kernels expand the weights to f16 WMMA operands, once per weight set.
 - **Template:** `swin_block.h` covers encoders, the tube-shaped enc3 and decoders.
+- **Output encoding:** the FP8 output is encoded two values at a time with packed 16-bit operations (`enc8x2`, exhaustively equal to the scalar encoder), and the 2×2 patch merge reads the rounded f16 values the codes decode to, from a row layout without LDS bank conflicts, instead of decoding the bytes again in every wave.
+- **Weight loads:** enc1 forms each weight tile's address in scalar registers (`SWIN_SCALAR_BLOAD`); the 8-wave layers are faster without it.
+
+**Preset L texture variants.** L uses unfolded `rrlite_enc0_4x4_*`, `rrlite_dec0_4x4`, and `rrlite_post_3_*` kernels. The L build keeps their full neural arithmetic and replaces surface stores with the existing native format-conversion functions. M's folded enc0 tail and dec0 head cannot be used for these variants. All four input flag combinations and all eight post variants are built, along with the shared downsample kernels. `all` and `tex` include these variants; `l` builds the shared Swin layers plus only the L texture set. L remains experimental; these replacements do not establish RTX parity or game performance.
+
+**Motion mitigation.** `[Kernels] NativeSwinEncoders = false` (`D4R_NATIVE_SWIN_ENCODERS=0`) bypasses only `rrlite_enc1_4x4` and `rrlite_enc2_4x4` in the native manifest. L/M then use the original translated PTX for those encoders, retaining native tube, decoder and texture kernels. Keep `PreferAccuracy = true` for the tested conservative path. It reduced the extra building-edge trails in a 64-frame moving Townfall capture at 853×480 → 2560×1440 on gfx1101; it does not recover detail absent from the low-resolution input or establish RTX parity. The default remains `true` for speed. Flat developer sets without manifests fall back entirely when this switch is off; add a manifest to keep selective acceleration.
+
+Initial validation: a gfx1101 fast L build completed two Ultra Performance harness frames (192×108 → 576×324, DLSS 310.7). Its finite, nonzero RGBA16F output matched the original translated L texture kernels byte for byte with the same native Swin layers. A gfx1201 FP8 accuracy L set compiled successfully; RDNA4 hardware and 4K game output have not been tested. Full packaging rebuilds missing L variants when given an older texture bundle.
+
+## RDNA4
+
+`kernels/common/wmma_layout.h` selects the WMMA operand and accumulator layout. gfx12 uses eight f16 values per lane in each K half; the accumulator rows are `i + 8·half`. The gfx11 path retains its original layout. `D4R_WMMA_LAYOUT=12` on gfx11 is a test shim: it exercises gfx12 indexing but executes gfx11 WMMA, so its replay output can be compared byte for byte with the existing gfx11 build.
+
+The gfx12 M variant can use native e4m3 FP8 WMMA with `D4R_NATIVE_FP8=1`. Its activations are requantized to e4m3 at operand load, and its prepared weights remain bytes. The matching ZLUDA and texture-tail variant uses `D4R_ZLUDA_WMMA_FP8_NATIVE=1` / `D4R_TEX_FP8=1`; the release bridge chooses the `-fp8` kernel folder. `NativeFp8` in d4r.ini controls that choice and defaults to on. The f16 widening path remains available.
+
+The gfx11 refactor and gfx12-layout shim matched the recorded gfx11 replays and 88-frame K/M harness images byte for byte on the RX 7700 XT. rocjitsu gfx1201 replay comparisons put the native K layers within 96–109 dB PSNR of gfx11, and M differed in 0.001–0.3% of e4m3 bytes. These are emulator results, not RDNA4 hardware results. rocjitsu's FP8 arithmetic is not hardware-checked; bit-exactness, speed and driver stability on an RDNA4 GPU remain unverified.
+
+For ZLUDA-compiled code, compare rocjitsu runs with other rocjitsu runs: the emulator does not reproduce the denormal-flush mode of those code objects, so a GPU byte comparison is not a valid arithmetic check. Captures with texture or surface objects must not be sent to the replay tools; the standalone `tail_check.hip` and the full D3D12 harness cover those tails.
 
 **Texture kernels (`kernels/tex`).**
 - **The problem:** some DLSS kernels are mostly texture sampling and scalar maths that ZLUDA already compiles well, apart from a few slow parts.
@@ -43,25 +63,37 @@ The release is built this way, with hashes from DLSS 310.7.0 and 310.9.1. The PT
   - surface stores become calls into `tex_common.h`;
   - the `roundf` idiom that forces ZLUDA into strict-FP mode is rewritten exactly;
   - for M's enc0 and dec0, a range of the kernel is replaced by a native tail or head.
-- **Build:** ZLUDA compiles the edited PTX with the HIP bitcode linked in (`D4R_ZLUDA_EXTRA_BC`), and the resulting code object is saved.
-- **Output redirect:** the K output kernel (`hiluma_engine_output_*`) can also write its result straight into the shim's output buffer instead of a CUDA array. See `D4R_SHIM_OUTPUT_DIRECT` in [architecture.md](architecture.md).
+- **Build:** ZLUDA's `d4r_emit` compiles the edited PTX for `D4R_GPU_ARCH` with the HIP bitcode linked in (`D4R_ZLUDA_EXTRA_BC`), and the resulting code object is saved. This is an offline per-target build.
+- **Every flag combination:** DLSS picks kernel variants by the game's settings (motion vectors at render or display resolution, HDR or LDR input, inverted or regular depth, ...). `kernels/build.sh` builds all of them: 24 K output kernels, 8 M post, 4 M downsample and 4 M enc0 kernels. enc0's native tail is spliced into each variant at its own line, with the registers read off the variant's PTX (`make_ptx.py` checks that the code after the cut matches the reference variant line for line).
+- **Wave64:** kernels without MMAs can be compiled as wave64 (`D4R_ZLUDA_WAVE64=1`, ZLUDA patch 0006): each wave runs two CUDA warps, and RDNA3 issues FP32 work for all 64 lanes at once. The post and K output kernels are built this way (post 0.89 → 0.77 ms), with bit-identical results.
+- **Output redirect:** the kernel that writes a preset's result can write it straight into the shim's output buffer instead of a CUDA array: `hiluma_engine_output_*` for K, `rrlite_downsample_kernel_*` for M (M's post kernel works at 3840×2160; the downsample produces the output). See `D4R_SHIM_OUTPUT_DIRECT` in [architecture.md](architecture.md).
 
 ## Numerics
 
-NVIDIA's tensor-core code rounds its f16 accumulator after every 16-deep step. The native kernels either do the same (bit-exact against the reference models) or keep f32 accumulators through a chain (`PWIN_F32ACC`, `SWIN_F32ACC`). The f32 version is more accurate than NVIDIA's own rounding, and its replayed output differs from the exact one by about 66 dB PSNR, which is invisible.
+The conservative kernels round their f16 accumulator after each MMA step (k16 for K, k32 for M). The fast kernels keep f32 accumulators through a chain (`PWIN_F32ACC`, `SWIN_F32ACC`); RDNA3 M also skips intermediate FP8 re-quantization (`SWIN_NO_Q8`). Wider arithmetic changes the network's intermediate values; it does not establish better image quality. Recorded fast-versus-conservative replays were close (about 66 dB PSNR), but that is not an RTX reference comparison or a guarantee for all scenes.
 
 The installed K set uses f32 accumulation for enc0–enc2 and dec0–dec2 and exact rounding for the deep layers. The M set uses the fast numerics throughout.
 
+**`[Kernels] PreferAccuracy = true`** (default `false`, environment `D4R_PREFER_ACCURACY=1`) selects a separate accuracy set for all K/M network layers and all texture/output variants, on every supported target, including gfx12 FP8. K disables `PWIN_F32ACC`; M restores FP8 quantization and rounds each k32 accumulator to f16. Texture builds use wave32 and retain the original denormal requirements; the existing exact round-half-away rewrite is retained. Native texture heads/tails already round their accumulators per k32 step. The bridge also disables `IgnoreDenormals`, translated f32 accumulator shadows, wave64 translation and NGX synchronization elision before ZLUDA loads, including when native kernels are off or fail their PTX hash check. Other interop and latency settings remain independent.
+
+The bridge selects `kernels/accuracy/<target>` (or `<target>-fp8`). Accuracy directories carry a `d4r-accuracy.txt` marker written only after a successful build. A marked flat developer directory also works. Missing accuracy variants fall back to translated PTX and are logged; the bridge never substitutes fast native binaries. Restart the game when changing the setting. The mode uses a separate `d4r-accuracy` JIT cache under the configured cache directory so older runtimes cannot reuse experimental fast-math results. The first accuracy run may compile new kernels.
+
+This mode aims to preserve NVIDIA's arithmetic, not to promise bit-identical RTX output: matrix accumulation order differs across hardware, and the shim still canonicalizes input formats and does not forward the optional transparency/current-color-bias masks. RTX parity requires identical captured inputs, library version, preset and temporal sequence on NVIDIA hardware.
+
 ## Building
 
 ```sh
 kernels/build.sh k          # DLSS 4 layers   (ROCm clang only)
 kernels/build.sh m          # DLSS 4.5 layers
+kernels/build.sh l          # L shared layers + unfolded textures (needs DLL and D4R_ZLUDA_EMIT)
 kernels/build.sh tex        # texture kernels (needs D4R_DLSS_DLL and D4R_ZLUDA_BUILD)
 kernels/build.sh all DIR    # everything into DIR (default kernels/out/native)
+D4R_PREFER_ACCURACY=1 kernels/build.sh all DIR/accuracy  # conservative versions of every family
 ```
 
-`D4R_GPU_ARCH` selects the target (default gfx1101). All kernels need gfx11 WMMA.
+`D4R_GPU_ARCH` selects one target for this standalone build (default gfx1101). The release script builds the network layers for gfx1100–gfx1103 and gfx1200–gfx1201 by default and puts each set in its own directory. The network kernels need gfx11 or gfx12 WMMA. Texture-kernel builds use `D4R_ZLUDA_EMIT` to generate code objects for the selected target without that GPU. Set `D4R_NATIVE_FP8=1` on gfx12 for the matching FP8 M and texture variants.
+
+The release includes both fast and accuracy network sets. Full releases also include accuracy texture sets: supply them in `D4R_BUNDLE_TEX/accuracy/<target>` or set `D4R_ZLUDA_EMIT` so packaging builds them. Build accuracy sets into a separate directory; the builder refuses to mark a directory containing unmarked older binaries as accurate.
 
 ## Validating a kernel
 
@@ -72,23 +104,33 @@ kernels/build.sh all DIR    # everything into DIR (default kernels/out/native)
    PREP_GRID=<d4r_prep_blocks> dump_runner NAME.hsaco NAME <capture> <out> 30
    ```
 
-   It runs the prep and main kernels, saves every buffer after the first launch, and times 30 launches.
+   It runs the prep and main kernels, saves every buffer after the first launch, and times 30 launches. The replay tool refuses captures with texture or surface objects; use the full harness for those kernels.
 3. **Compare against a reference.** `kernels/tools/pwin_model.py` (K) and `swin_model.py` (M) are numpy models of the layers, written stage by stage against `ptxsim.py`, a small vectorised PTX interpreter that runs NVIDIA's kernels on the CPU. A bit-exact variant should match the model within f16 rounding; after any change, compare the new build's buffers with the previous build's.
 4. **Check the whole pipeline.** Replay captured frames through the harness with both kernel sets and compare the outputs with `kernels/tools/psnr.py`. `D4R_CUDA_KERNEL_PROFILE=1` makes the bridge time every kernel; `kernels/tools/kprof.py` summarises the log per frame.
 
 ## Per-kernel cost
 
-GPU time per frame at Quality (1705×960 → 2560×1440, RX 7700 XT), from `D4R_CUDA_KERNEL_PROFILE`:
+Median GPU time per frame at Quality (1706×960 → 2560×1440, RX 7700 XT), from `D4R_CUDA_KERNEL_PROFILE` in the D3D12 harness (flags as in Townfall; frames 21–40). The profiler's per-launch events add a little to every kernel, so totals are slightly above the in-game cost. "Before" is release 0.1.1.
 
-| DLSS 4 (K) | ms | DLSS 4.5 (M) | ms |
-|---|---|---|---|
-| hiluma output (texture kernel) | 0.82 | enc3 tube (6 launches) | 1.42 |
-| dec0 | 0.30 | enc1 | 1.27 |
-| enc0 | 0.28 | dec1 | 1.18 |
-| dec1 | 0.24 | enc2 | 0.79 |
-| hiluma input (ZLUDA) | 0.24 | post (texture kernel) | 0.77 |
-| enc1 | 0.21 | dec2 | 0.69 |
-| enc2, dec2 | 0.10 each | enc0 (texture kernel) | 0.60 |
-| enc3, dec3, enc4, dec4, dec5 | 0.05–0.08 each | dec0 (texture kernel) | 0.46 |
-| NGX exposure and misc | 0.1 | downsample, NGX misc | 0.36 |
-| **total** | **2.8** | **total** | **7.5** |
+| DLSS 4 (K) | before | now | DLSS 4.5 (M) | before | now |
+|---|---|---|---|---|---|
+| hiluma output (texture kernel) | 0.92 | 0.89 | enc3 tube (6 launches) | 1.62 | 1.57 |
+| dec0 | 0.33 | 0.32 | enc1 | 1.41 | 1.27 |
+| enc0 | 0.31 | 0.31 | dec1 | 1.44 | 1.43 |
+| dec1 | 0.26 | 0.26 | enc2 | 0.87 | 0.77 |
+| hiluma input (ZLUDA) | 0.25 | 0.24 | post (texture kernel) | 0.88 | 0.77 |
+| enc1 | 0.23 | 0.22 | dec2 | 0.84 | 0.82 |
+| enc2, dec2 | 0.12, 0.10 | 0.12, 0.10 | enc0 (texture kernel) | 0.64 | 0.63 |
+| enc3, dec3, enc4, dec4, dec5 | 0.07–0.09 each | 0.07–0.09 each | dec0 (texture kernel) | 0.64 | 0.63 |
+| NGX exposure and misc | 0.1 | 0.1 | downsample, NGX misc | 0.32 | 0.33 |
+| **total** | **3.10** | **2.97** | **total** | **9.04** | **8.23** |
+
+Runs of the same build vary by about ±3% per kernel. M additionally skips its output copy now (direct output through the downsample kernel), which is not a kernel and not in the table. Games whose flags differ from Townfall's (motion vectors at render resolution, LDR input, regular depth) gain more: those kernel variants used to run as plain ZLUDA compiles (K 3.6–4.6 ms → 2.8–3.0 ms, M −0.5 to −0.7 ms per frame).
+
+## Experimental thin-feature motion coverage
+
+`[Interop] MotionVectorDilation = 1` or `2` (`D4R_MOTION_DILATION`) extends nearby foreground motion into pixels whose depth otherwise belongs to the background. The value selects the opaque-surface radius in render pixels. Both enabled values also fill sky gaps using the nearest foreground pixel within an eight-render-pixel circle (far depth 0 for reversed Z, 1 for normal Z). This is the wider sky fill from the comparison; it can strengthen trails in some frames and remains opt-in through `MotionVectorDilation`. This is a depth-guided approximation to better thin-feature motion coverage, not engine-side conservative rasterization. It improved the captured Townfall rooftop where the earlier native-encoder fallback did not resolve the artifact.
+
+The d4r-owned GPU kernel preserves the original depth/motion buffers and writes a separate RG16F velocity texture. The worker waits for its completion before NGX evaluates. No per-frame CPU readback is needed. It requires same-frame linear VRAM inputs, full-resolution unjittered motion vectors and zero subrect bases. Unsupported features or unavailable kernels keep the original vectors. The default is `0` (off); larger radii can affect disocclusion and unrelated foreground edges. The method does not recreate geometry missing from every input frame or establish RTX parity.
+
+`scripts/build_wine_nvcuda_bridge.sh` embeds the support kernels when `D4R_ROCM_DIR` points to ROCm clang. All six gfx110x/gfx120x targets compile; physical validation is currently gfx1101 only. A bridge built without the support kernels reports the feature unavailable rather than changing inputs. Optional GPU regressions cover forward/reversed depth, two radii, jitter, padded rows and preservation of input buffers; see `tests/test_motion_kernel_gpu.py`.

docs/performance.md

@@ -1,5 +1,7 @@
 # Performance
 
+All frame rates and kernel timings on this page were measured on an RX 7700 XT (gfx1101). RDNA4 (gfx1200/gfx1201) has compile and emulator validation only; no RDNA4 frame rate, latency, or kernel timing has been measured. Native FP8 may change both arithmetic and cost, so the gfx11 results cannot be projected to it.
+
 ## Method
 
 - **Hardware and settings:** Radeon RX 7700 XT (RDNA3, 54 CUs), Ryzen 9 5900XT, Linux 6.18. SILENT HILL Townfall at 2560×1440 output.
@@ -8,18 +10,20 @@
 - **Pairing:** runs are compared back-to-back in one session, because repeated runs of an identical setup vary by about 1%.
 - **Screen recording:** it costs a few fps, so recorded videos show slightly lower numbers than the table.
 - **Latency:** all DLSS numbers are same-frame (frame age 0). Every frame shows its own DLSS result, as with a native upscaler.
-- **Other upscaler modes:** Balanced, Performance and Ultra Performance use OptiScaler's render-ratio override (1.72, 2.0, 3.0) on top of the game's Quality mode.
+- **Upscaler modes:** every mode uses OptiScaler's render-ratio override (1.5, 1.72, 2.0, 3.0), so the result does not depend on the DLSS mode chosen in the game.
 
 ## Results
 
-| Mode (render resolution) | DLSS 3 CNN (E) | DLSS 4 (K) | DLSS 4.5 (M) | FSR 4 |
+| Mode (render resolution) | DLSS 3 CNN (E) | DLSS 4 (K) | DLSS 4.5 (M) | FSR 4\* |
 |---|---|---|---|---|
-| Quality (1705×960) | **72.4** | 69.4 | 51.5 | 76.0 |
-| Balanced (1488×837) | **80.5** | 76.6 | 59.6 | 84.5 |
-| Performance (1280×720) | **87.8** | 84.0 | 69.9 | 94.0 |
-| Ultra Performance (853×480) | 88.9 | **94.5** | 90.3 | 107.3 |
+| Quality (1705×960) | **71.7** | 67.9 | 49.3 | 76.0 |
+| Balanced (1488×837) | **80.8** | 75.9 | 58.0 | 84.5 |
+| Performance (1280×720) | **89.7** | 84.1 | 69.0 | 94.0 |
+| Ultra Performance (853×480) | 89.1 | **95.5** | 92.2 | 107.3 |
 | Native 2560×1440, no upscaling (TSR at 100%) | | 49.1 | | |
 
+\* FSR 4 and the native row are from 2026-09-27; the DLSS columns from 2026-09-29, when the machine ran about 2–4% slower overall (release 0.1.1 at Quality that day: E 71.2, K 67.9, M 48.4 fps, against 72.4, 69.4 and 51.5 two days earlier). The render resolution of every mode is set with OptiScaler's ratio override for whichever DLSS mode the game is set to (ratios 1.5, 1.72, 2.0 and 3.0).
+
 DLSS 4's cost is almost constant across modes (about 2.8 ms of GPU time per frame). Its network runs on a grid set by the output resolution, not the render resolution, so the gap to FSR 4 widens as the render resolution drops. The CNN gets more expensive at the 3× ratio, so K is the better choice at Ultra Performance.
 
 ## What each step contributed

docs/windows-game.md

@@ -0,0 +1,143 @@
+# Native Windows RDNA3 / RDNA4 game development package
+
+Validated hardware: Windows 11 x64 / Radeon RX 9070 XT / gfx1201. Native K and
+M transformer kernels pass standalone D3D12 + OptiScaler checks. Silent Hill 2
+K now renders real menu frames and M renders a saved gameplay level with
+GPU NaN/Inf checks. Both K and M pass 3840x2160 output on this machine.
+Cyberpunk 2077 2.31 also reaches saved gameplay with K and M at 3840x2160:
+all required native transformer layers execute and output checks pass.
+The [port notes](windows-rdna4-port.md) record session counts and limitations.
+Performance measurements and their controls are recorded in
+[windows-performance.md](windows-performance.md). This is a
+development package; no NVIDIA proprietary DLL is included.
+
+Experimental builds also accept gfx1100..gfx1103 (RDNA3), gfx1150..gfx1154
+(RDNA3.5) and gfx1200 (for example RX 9060/9060 XT). These targets are
+**compile-tested only**; no physical validation here has been performed.
+RX 9070 and other gfx1201 cards also remain untested. Use a package built for
+the GPU's actual HIP architecture: its `gpu-target.json`, host shim and ELF
+code objects must agree. Installation/runtime reject mixed targets. See
+[windows-gpu-support.md](windows-gpu-support.md) for separate-target build
+commands, the fork review and the first hardware diagnostic gate.
+
+Before changing game files, the runner inventories HIP devices without
+executing kernels. It rejects another architecture's package and a local
+D3D12 proxy identifying itself as vkd3d/Wine, producing one diagnostic ZIP.
+RX 9060 XT needs gfx1200: the old gfx1201-only package fails before DLSS
+initialization on it. Use native D3D12. On an untested GPU, start with
+synchronous K and output diagnostics. The ordinary launcher does not attach
+a debugger. `-CaptureExceptions` is an explicit developer option; games with
+anti-debug protection can refuse it.
+
+Numeric CUDA driver NGX initialization searches for `nvngx_dlss.dll` beside
+the calling d4r shim. The installer stages your local feature DLL in `d4r/`
+and the original NVIDIA core in `d4r/vendor/`. `Libraries.NvngxDlssPath` is
+the feature directory. Keeping the feature DLL only in `vendor/` causes
+`BAD00004` (FeatureNotFound) in numeric-ID games such as Cyberpunk.
+
+Build the shim/diagnostics, corrected ZLUDA, K/M native objects and patched
+OptiScaler using the scripts in `scripts/windows`, then run:
+
+```powershell
+.\scripts\windows\package-windows-game.ps1 -ArchivePath "$PWD\dist\windows-rdna4-game.zip"
+```
+
+The package pins TheRock 10.2.0a20260929 because its Windows external-memory
+mapping lifetime passes the repeated import/release test. Stable HIP 7.2 has
+a reproduced mapped-view leak; see `docs/windows-rdna4-port.md` for the source
+fix, exact dependency versions, validation and complete build commands.
+The ZIP contains only manifest-listed files, including a snapshot of committed
+d4r source. Private NVIDIA-derived objects, caches and diagnostic captures are
+excluded. Original FP8 M and the optional hardware-packing variant pass strict
+replay/network tests but do not establish a full-network speedup, so the game
+package retains its FP16-equivalent M baseline.
+
+Run a D3D12 game without anti-cheat using your locally supplied NVIDIA DLLs:
+
+```powershell
+.\dist\windows-rdna4-game\windows-game.ps1 -GameExe "D:\Games\SILENT HILL 2\SHProto\Binaries\Win64\SHProto-Win64-Shipping.exe" -NgxCore "C:\Users\Administrator\d4r\_nvngx.dll" -DlssDll "C:\Users\Administrator\d4r\nvngx_dlss.dll" -Preset 11
+```
+
+Use `-Preset 13` for M. The validated manifest requires DLSS 310.9.1 with the
+recorded SHA256; other feature DLLs require regenerating and validating the
+native manifest. Select an upscaler supported by OptiScaler in the game.
+The script installs `dxgi.dll`, an explicit DLSS configuration and a private
+`d4r` runtime directory, and captures stdout/stderr, exit code, loaded DLL paths,
+driver information and OptiScaler logs. Ordinary runs use a process job without
+an attached debugger. Add `-CaptureExceptions` to capture exception events and
+minidumps when reproducing a crash. A local
+ZIP bundle is printed at exit. `-RunSeconds 120` bounds a diagnostic run;
+the normal default allows you to play until you close the game.
+
+For the Silent Hill 2 startup black-screen investigation, add
+`-DiagnosticResolution 1280x720`. This backs up its two graphics configuration
+files, temporarily selects a window at that resolution, and restores the
+original bytes after the test. It does not change progression saves. First-use
+PTX compilation can be slow; `-CacheDirectory <existing-ZLUDA-cache>` reuses
+modules with matching GPU, codegen switches and ZLUDA binary fingerprint.
+
+Add `-ValidateOutput` to scan the entire RGBA16F DLSS output for NaN/Inf on
+the GPU before returning it to D3D12. Only two diagnostic counters (8 bytes)
+are read by the CPU; image data remains in VRAM. Validation failures are
+reported explicitly. Disable this diagnostic when measuring production speed.
+
+`-ProfileStages` records CPU completion time for seven interop/NGX stages,
+command-list split/replay, submitted state publication and suffix completion.
+The recording interval measures NGX calls on the recording thread, not Present.
+`-ProfileKernels` enables synchronizing HIP-event kernel profiling and changes
+frame scheduling; use it separately from a production FPS measurement.
+`-ProfileCommandHooks` samples one in 64 D3D12 interceptions to distinguish
+tracking/lock waits from driver calls. It is disabled in ordinary runs.
+`-ProfileCudaApi` aggregates existing CUDA API host durations without adding
+HIP events or synchronization. `-UncachedInteropLists` restores per-frame
+copy-list allocation for comparison with the normal safely reused lists.
+`-ProfileGpuBoundary` requires `-AsyncInterop` and measures D3D12 GPU intervals
+around the input/output copies and external fence. It reads 32 timestamp bytes
+only after existing completion, with no extra CPU wait. Its external interval
+includes HIP and scheduling; it is not an isolated transformer kernel time.
+The runner drains stdout/stderr concurrently as raw bytes; the earlier
+line-oriented PowerShell collector imposed large per-frame delays. The optional
+ZLUDA kernel profiler is for diagnostics only. Measurements and reproduction
+commands are in [windows-performance.md](windows-performance.md).
+
+`-AsyncInterop` enables the experimental K scheduling path. Submission queues
+the current frame's input/output dependencies and returns while a CUDA worker
+executes DLSS. Owned command memory is retained until GPU completion; feature
+release and resource reconfiguration wait for pending consumers. It passes
+the standalone queued-frame and Release/CreateFeature exact-output regression,
+but remains opt-in while game stability and performance are checked. It
+currently requires one D3D12 command queue. This switch is independent of
+profiling and output scans; omit those diagnostics for a production FPS run.
+
+Ordinary runs use OptiScaler info logging and suppress successful HIP/CUDA API
+messages. Errors, GPU identity, imports, frame records and native/translated
+kernel launches remain logged. `-VerboseRuntime` enables all API and frontend
+trace messages when investigating a failure. Numerical relaxation flags are
+cleared in the child process; FP16 rounding and subnormals stay enabled.
+
+The optional K output-store optimization is built locally from your NVIDIA DLL
+with `build-native-texture.ps1` and checked by `test-native-texture.ps1`;
+commands and measurements are in [windows-performance.md](windows-performance.md).
+Add `-LocalTextureKernels "C:\Users\Administrator\d4r\build\private-textures-gfx1201"`
+to the game command to install it. The object/manifest hashes and DLSS identity
+must match validation. These generated objects contain NVIDIA code, remain
+private, and are excluded from the distributable package. Ordinary launches
+inherit the game's resolution/window settings; the default argument is `-dx12`.
+
+Original files are backed up before replacement. Restore with:
+
+```powershell
+.\dist\windows-rdna4-game\windows-game.ps1 -Action restore -GameExe "D:\Games\SILENT HILL 2\SHProto\Binaries\Win64\SHProto-Win64-Shipping.exe"
+```
+
+Restore refuses to overwrite files changed since installation. Backups and
+diagnostic results are retained. The script does not change progression saves,
+system DLLs, driver settings or security settings. All runtime work is native
+Windows. Colour, depth, motion, exposure and output cross APIs through VRAM
+and shared D3D12 fences; the backend waits for the current frame and never
+uses a previous-frame fallback.
+
+OptiScaler source modifications are GPL-3.0 and exported under
+`source-patches/optiscaler`. ZLUDA source modifications are exported under
+`source-patches/zluda`; the LLVM gfx1154 backport is under
+`source-patches/zluda-llvm`. Dependency license texts and build hashes are included.

docs/windows-gpu-support.md

@@ -0,0 +1,189 @@
+# Experimental Windows RDNA3 / RDNA4 coverage
+
+The shim, diagnostics, native K/M kernels and game installer accept the
+targets below, with one package per actual ISA. **Only RX 9070 XT has physical
+validation here.** Community RX 9060 XT / gfx1200 logs now show K/M backend
+execution in Cyberpunk and Dawnwalker with all required native layers and
+finite output. Visual issues remain reported and are not resolved by those
+checks. All other targets are experimental and compile-tested; their DLSS,
+interop and performance remain unverified.
+
+| Family | Target | Examples / scope |
+| --- | --- | --- |
+| RDNA3 | gfx1100 | RX 7900 family; same-target workstation GPUs |
+| RDNA3 | gfx1101 | RX 7700 / 7700 XT / 7800 XT; same-target mobile/workstation GPUs |
+| RDNA3 | gfx1102 | RX 7600 / 7600 XT; same-target mobile/workstation GPUs |
+| RDNA3 | gfx1103 | RDNA3 integrated graphics |
+| RDNA3.5 | gfx1150, gfx1151, gfx1152, gfx1153, gfx1154 | Corresponding integrated graphics targets |
+| RDNA4 | gfx1200 | RX 9060 / 9060 XT; same-target GPUs |
+| RDNA4 | gfx1201 | RX 9070 / 9070 XT / 9070 GRE; same-target GPUs |
+
+Names and IDs follow [LLVM processors](https://llvm.org/docs/AMDGPUUsage.html#processors)
+and [ELF ABI](https://llvm.org/docs/AMDGPUUsage.html#amdgpu-elf-header-e-flags).
+`tools/windows/gpu-targets.json` supplies CMake-generated C++, PowerShell and
+the private texture builder. Selection uses HIP's real `gcnArchName` and the
+D3D12 adapter LUID, including systems with several GPUs sharing a target.
+Marketing names do not control eligibility.
+
+All listed RDNA3/RDNA3.5/RDNA4 targets with existing gfx11/gfx12 WMMA contracts
+are included. TBD RDNA4m gfx1170..gfx1172 and gfx12.5 gfx1250/gfx1251 are
+different ISA groups and excluded. Unknown future targets need explicit review.
+Check [AMD's Windows matrix](https://rocm.docs.amd.com/projects/radeon-ryzen/en/latest/docs/compatibility/compatibilityrad/windows/windows_compatibility.html)
+for driver/runtime eligibility. Compiler support cannot supply missing Windows
+driver support or external-memory APIs, especially on APUs.
+
+## Installation and diagnostics
+
+Use the actual target's archive: **gfx1200 for RX 9060 XT**, for example.
+Metadata, host target and every ELF target must agree. Runtime/scripts reject
+mixed targets and never rename targets, rewrite ELF IDs or set
+`HSA_OVERRIDE_GFX_VERSION`. Legacy metadata-free packages remain gfx1201 only.
+
+Before changing game files, `windows-game.ps1` now runs a read-only HIP
+inventory and checks the target and wave32. Inventory allocates no GPU buffers
+and runs no kernels; success is discovery, not validation. Failure prints one
+ZIP containing stdout/stderr, exit code, runtime/DLL and driver details. A local
+`d3d12.dll` identifying itself as vkd3d/Wine is rejected before install. Restore
+the game's native D3D12 installation; the script never loads or removes that
+proxy itself. Begin synchronously on an untested GPU:
+
+```powershell
+.\windows-game.ps1 -GameExe "D:\Games\Example\Game.exe" -NgxCore "C:\LocalDLLs\_nvngx.dll" -DlssDll "C:\LocalDLLs\nvngx_dlss.dll" -Preset 11 -ValidateOutput -CaptureExceptions
+```
+
+The local DLLs must match the documented NGX version and DLSS 310.9.1 SHA256
+in `package.json`. NVIDIA DLLs are never distributed. Retain the printed ZIP
+on failure. Async K and performance are subsequent gates; gfx1201-specific
+FP8, arithmetic and tuning experiments remain separate.
+
+## Issue #10 review, 2026-10-02
+
+Reviewed [comment 5952822115](https://github.com/countervolts/d4r/issues/10#issuecomment-5952822115),
+including all 32 attached text logs from eight attempts. HIP discovers
+**RX 9060 XT / gfx1200** successfully. Every attempt fails with the old shim's
+**"No selected gfx1201 device"** rejection; OptiScaler reports `support=false`
+/ `BAD00002`. None reaches `D4R_RUNTIME` initialization or native transformer
+execution. These logs do not demonstrate a K/M kernel crash. Use the gfx1200
+package instead of overriding architecture. Three attempts also load
+vkd3d-proton 3.1.0 from a local D3D12 proxy, outside this Windows backend.
+
+The new packages/preflight address the identified failures. They do not prove
+those games work on RX 9060 XT; a new native-D3D12 run on that GPU is required.
+Raw community logs and personal directory paths stay local.
+
+## Community retest, 2026-10-03
+
+[Comment 5962444244](https://github.com/countervolts/d4r/issues/10#issuecomment-5962444244)
+uses the corrected `52d9872` package / `3d62b98` binaries on RX 9060 XT
+(`gfx1200`, 16 GB). Cyberpunk K/M complete 4,365 / 1,751 session frames;
+Dawnwalker K/M complete 6,713 / 1,690. Every frame has a finite-output GPU
+check, all required native layers execute and all four sessions exit normally.
+The logs record zero backend failures, previous-frame outputs and CPU image
+copies. This is community backend coverage, not a reference-image comparison:
+the tester reports texture/LOD and motion-related artifacts. All four runs use
+the HDR network input/output path, unlike the local Cyberpunk LDR validation.
+
+Spider-Man 2 crashes with `0xc0000005` before any completed DLSS frame;
+its nonstandard launch is not evidence of a transformer kernel failure.
+Assassin's Creed exits with zero frames and Streamline logs repeated internal
+minidumps; its clean process exit is still inconclusive. Their causes remain
+open without a stack trace. The Witcher 3 log in
+[comment 5962808629](https://github.com/countervolts/d4r/issues/10#issuecomment-5962808629)
+selects the gfx1200 d4r shim instead of the original NGX core (its SHA256
+matches the shipped shim exactly), and the installer correctly refuses
+before modifying the game. It does not establish a Witcher runtime failure.
+
+## Build
+
+Prepare dependencies following `windows-rdna4-port.md`. Default: gfx1201.
+Select another target explicitly and keep separate directories:
+
+```powershell
+$arch = 'gfx1101'  # for example RX 7800 XT
+$diag = "$PWD\dist\windows-$arch-diagnostics"
+.\scripts\windows\build-windows-rdna4.ps1 -RuntimeProfile therock -GpuArch $arch -ZludaRoot "$PWD\dist\zluda-windows-final" -BuildDirectory "$PWD\build\windows-$arch" -InstallDirectory $diag
+.\scripts\windows\stage-native-k.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -PackageRoot $diag
+.\scripts\windows\stage-native-m.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -PackageRoot $diag
+.\scripts\windows\package-windows-game.ps1 -DiagnosticRoot $diag -GpuArch $arch -PackageRoot "$PWD\dist\windows-$arch-game" -ArchivePath "$PWD\dist\windows-$arch-game.zip"
+```
+
+Rebuild diagnostics for the new inventory executable. The package keeps strict
+K FP16 and M FP16-equivalent baselines; native M FP8 remains disabled. CMake:
+`-DD4R_GPU_ARCH=gfx1101`. Some source/executable names retain gfx1201/gfx12 for
+compatibility; actual targets follow the selected architecture.
+
+One command compiles all eleven targets without GPU workloads:
+
+```powershell
+.\scripts\windows\build-windows-gpu-matrix.ps1 -ZludaRoot "$PWD\dist\zluda-windows-final"
+```
+
+Add `-PackageGames -DlssDll "$PWD\nvngx_dlss.dll" -ArchiveDirectory "$PWD\dist\windows-gpu-archives"`
+to stage/archive all packages after committing source and preparing OptiScaler.
+Use `-GpuArchitectures gfx1100,gfx1101,gfx1102,gfx1200,gfx1201` for a subset.
+Per-target logs and `matrix-build.json` distinguish compilation from execution.
+The supplied DLL generates manifests locally and is never copied into archives.
+
+Rebuild LLVM/ZLUDA when updating from an older package: the old LLVM 22 backend
+does not recognize gfx1154 even though Clang 24 compiles native objects for it.
+`build-zluda-llvm.ps1` now applies the source backport in `patches/zluda-llvm`
+automatically; relink with `build-zluda-windows.ps1` afterward. Compiler base
+and patch hashes are recorded in the ZLUDA build metadata and included in game
+packages. The expected patched LLVM submodule appears modified in its parent
+checkout; its complete source delta is exported, not a binary target override.
+
+For a compilation gate requiring neither a GPU nor NVIDIA DLLs, run:
+
+```powershell
+.\scripts\windows\test-zluda-target-compilation.ps1 -ZludaRoot "$PWD\dist\zluda-windows-final"
+```
+
+All 33 public PTX compilations pass for the eleven targets: Driver API pattern,
+FP16 MMA strict reference and explicit WMMA modes, with correct ELF targets
+and wave32. The gfx12 strict F16 reference intentionally retains its existing
+double-precision helper; the separate WMMA mode checks code generation only.
+It does not enable relaxed arithmetic in game packages or validate numerical
+results on other GPUs. The original gfx1154 compiler abort is preserved locally
+in `build/llvm-gfx1154/before.log`; the regression results are in
+`test-results/windows-gpu-coverage/zluda-offline`.
+
+## Fork adaptation
+
+Reviewed [realdody's branch](https://github.com/realdody/d4r/tree/windows-rdna3)
+at `ae530b0f53ed5c0bb87558dd7c20f619e014e4fd` and the newer
+[`1897b4a` correction](https://github.com/realdody/d4r/commit/1897b4afa79832bd879234cfb37e7c1468db5cde).
+Earlier MSVC binary16 and generated `BOOL` fixes remain. The earlier forced
+gfx12 fragment layout is not used.
+
+The newer fork reports Windows gfx1101 `permlanex16` returning its own lane,
+duplicating even K values into odd slots. This port adapts that finding into
+`kernels/common/wave32_exchange.h`: a Windows/gfx11-only per-wave LDS exchange,
+with native gfx11 layout, generic x/y/z indexing, 2 KiB scratch (512 threads
+maximum), volatile accesses and wave fences/barriers protecting scratch reuse.
+It covers K/M fragment conversions and other K half-wave exchanges. gfx12 and
+Linux keep the original instruction path. The raw gfx11 probe uses its actual
+intrinsic/layout, without a forced gfx12 shim.
+
+The fork reports RX 7800 XT enc1/enc2 identity/nonzero/reference checks. Those
+are **the author's results**, not validation of this adaptation or full K/M
+here. LDS exchange, full network, translated CUDA, accuracy, interop and speed
+still need physical gfx11 testing. Credit: realdody <dodobozicek@gmail.com>.
+
+## Results and next hardware gate
+
+All eleven targets compile the complete MinGW host build and 19 device objects
+with TheRock `10.2.0a20260929`, HIP `7.17.26386`, Clang 24. Software target
+contracts inspect actual ELF objects without other-target GPU launches.
+RX 9070 XT passes all 20 CTest gates after these changes. K/M full-frame and
+package checks are recorded in `windows-rdna4-port.md`. Local logs are in
+`build/windows-gpu-coverage` and `test-results/windows-gpu-coverage`.
+
+The first hardware gate on another GPU needs no NVIDIA DLL:
+
+```powershell
+.\scripts\windows\test-windows-rdna4.ps1 -RuntimeProfile therock -PackageRoot "$PWD\dist\windows-gfx1101-diagnostics" -ZludaRoot "$PWD\dist\zluda-windows-final"
+```
+
+Success requires exit 0, correct actual target, exact HIP/PTX/WMMA/pixel checks
+and stable interop lifetime. Failure prints one ZIP. Full K/M reference,
+temporal, DLSS and game validation remain subsequent gates.

docs/windows-performance.md

@@ -0,0 +1,810 @@
+# Windows RDNA4 measurements
+
+Hardware: RX 9070 XT / gfx1201, Windows 11, display driver 32.0.31041.1004.
+Runtime: TheRock 10.2.0a20260929, HIP 7.17.26386; ZLUDA b0161a4 with
+`D4R_ZLUDA_WMMA=1`, FP8 widening, native FP8 disabled, f16 reference rounding.
+
+The original serializing profiler uses HIP events on the actual launch stream, including
+separate measurements of one-time weight preparation. It synchronizes each
+sampled launch and changes scheduling: these numbers are not application FPS.
+HIP occupancy predictions are not measured hardware occupancy. The native
+objects' actual wave32, LDS, VGPR/SGPR and spill counts are available in
+`llvm-readobj --notes` and can be attached to the JSON report. Memory bandwidth
+and WMMA utilization were not measured; no utilization is inferred from timing.
+
+The identical four-frame standalone D3D12/OptiScaler workloads output 512x288.
+Summing the measured main-kernel event times and dividing by four gives:
+
+| Preset | Fully translated, ms/frame | Native FP16-equivalent, ms/frame |
+| --- | ---: | ---: |
+| K | 157.745441 | 0.684675 |
+| M | 7.428538 | 1.566500 |
+
+Both rows include translated input/output tails. Native overrides cover all
+11 K layers and five M layers; M's tube layer executes six times per frame.
+M native and translated RGB are bit-exact, with max absolute/relative error 0.
+K native remains bit-exact against its independently validated native control;
+fully translated K has the previously documented arithmetic differences.
+All outputs are finite. These are reference-correct codegen settings, not
+measurements of a relaxed fast-math translated configuration.
+
+M's first profiled Evaluate includes about 0.9 seconds of host initialization;
+the four-sample CPU mean must not be treated as steady-state latency. Its
+median Evaluate host completion is 4.855 ms. GPU events and prep phases are
+reported separately in the raw JSON.
+
+The initial game runner used PowerShell line-oriented output redirection.
+It heavily throttled verbose stdout/stderr. The runner now concurrently drains
+raw byte streams using the same approach as the standalone diagnostics.
+
+| Silent Hill 2 stage, CPU mean | Initial capture, ms | Raw-byte capture with HIP profiler, ms |
+| --- | ---: | ---: |
+| Input fence wait | 5.641 | 0.833 |
+| Input conversion and CUDA array upload | 13.196 | 0.668 |
+| NGX Evaluate host call | 25.715 | 3.350 |
+| NGX all-stream completion | 1.846 | 0.033 |
+| Output CUDA array download | 4.288 | 0.167 |
+| Output conversion and shared fence signal | 1.500 | 0.033 |
+
+The first run additionally scanned every output (10.142 ms in that throttled
+capture); the kernel-profiling run disables that scan. Both use a temporary
+1280x720 window and current-frame VRAM interop. These are separate menu/game
+runs, not a controlled whole-game FPS comparison. No synchronization was
+removed to obtain the improvement. The raw-byte profiling run completed
+2345 K frames, with no d4r failure, CPU image copy or previous-frame output.
+
+Reproduce native or translated measurements with locally supplied NVIDIA DLLs
+at the repository root and the separately built profiling runtime:
+
+```powershell
+.\scripts\windows\profile-windows-rdna4.ps1 -Preset 11
+.\scripts\windows\profile-windows-rdna4.ps1 -Preset 11 -Translated
+.\scripts\windows\profile-windows-rdna4.ps1 -Preset 13
+.\scripts\windows\profile-windows-rdna4.ps1 -Preset 13 -Translated
+```
+
+`profile_report.py <results-directory> --output profile.json` summarizes raw
+CPU stages and every kernel's event time, launch geometry, registers, private
+memory, LDS and predicted blocks per multiprocessor. Add
+`--metadata-directory <directory-of-llvm-readobj-notes>` for native wave sizes
+and compiler spill counts. Reports and proprietary workload inputs remain local.
+
+Native FP8 remains a separate optional experiment; the package uses the
+validated FP16-equivalent baseline. Its later checks and timings are below.
+
+## K host synchronization and asynchronous submission
+
+Current priority is K performance. The user's synchronous 4K reading is
+49–51 FPS / 53% GPU utilization, versus approximately 80+ FPS / 100% with
+FSR4. These readings do not identify a CPU hardware limit or provide a
+controlled same-scene comparison. M/FP8 optimization is deferred.
+
+ZLUDA candidate `a1c506f` adds `D4R_ZLUDA_PROFILE_API`, enabled by the game
+runner's `-ProfileCudaApi`. It measures existing API host durations, aggregates
+per thread and logs every two seconds without HIP events or additional waits.
+The synchronous control shows five `cuCtxSynchronize` calls per frame with
+about 3.9 ms aggregate host time. CUDA launch host overhead is approximately
+0.46 ms per frame. Cached copy-list setup is only 0.011 ms median. These host
+measurements include GPU waits and must not be added to GPU kernel times.
+
+| K/4K command submission, CPU median | Synchronous | First async | Async without CUDA mutex |
+| --- | ---: | ---: | ---: |
+| Entire submission, ms | 10.512 | 2.676 | 0.181 |
+| DLSS boundary, ms | 10.083 | 2.604 | 0.124 |
+| Consumer suffix completion, ms | 0.350 | 0.005 | 0.004 |
+
+The first async game run completes 9481 finite frames; the user reports
+60 FPS / 63% GPU. The mutex-free run crashes during feature recreation after
+366 finite frames, so its submission timing is not a successful gameplay
+benchmark. The standalone burst/recreate reproducer subsequently identifies
+caller-owned GPU backing freed after ReleaseFeature returns. Waiting for
+pending D3D12 consumers in ReleaseFeature fixes the reproducer: all twelve
+sync/async test frames pass, and corresponding RGB outputs are exact. Game
+stability and FPS must be checked separately on the fixed build.
+
+The fixed game retest on 2026-10-02 completes 16941 K/4K frames, all finite,
+including feature recreation and color-format reimports, with zero backend
+failures. The user reports 62 FPS / 63% GPU. Median CPU submission is
+0.197 ms, NGX host completion on the worker is 4.400 ms and output validation
+is 0.380 ms. The 10.162 ms median worker input wait includes queued game
+rendering. These results establish a stable five-minute test of this fix,
+not a conclusion about the remaining GPU idle time or FSR4 performance.
+
+`-AsyncInterop` is opt-in. It queues input/output fences on the submission
+thread, executes CUDA on a worker and retires owned command memory by GPU
+completion. Three copy slots bound outstanding ownership. The queue consumes
+the output of the same frame, with no CPU image copy. The worker's input wait
+includes its lead behind submitted rendering and is not a CPU submission
+stall. Neither that duration nor the NGX recording interval measures Present
+latency. Reconfiguration waits for all consumers; ordinary submission does not
+hold the CUDA mutex. The experimental runtime currently supports one queue.
+
+```powershell
+.\scripts\windows\test-async-k.ps1
+```
+
+The test generates fresh synchronous controls, submits three frames without
+intermediate CPU completion, and repeats while releasing/recreating the DLSS
+feature with work pending. Results include exact RGB comparisons, finite GPU
+scans, stdout/stderr, exceptions and runtime provenance. To include the local
+validated output-store optimization, pass its combined native directory with
+`-NativeRoot`; proprietary objects remain local.
+
+`-ProfileGpuBoundary` (requires `-AsyncInterop`) adds D3D12 timestamps before
+and after the VRAM input/output copies. Resolved results are read only once
+the copy slot's existing fence completes; 32 diagnostic bytes per frame are
+read and no new completion wait is added. `external_span_ms` spans HIP and
+host/driver scheduling across the external fence. `between_boundaries_ms`
+also includes game rendering, and `input_interval_ms` is not Present timing.
+These queries obey the queue's timestamp ordering and add GPU commands; they
+are disabled for an ordinary FPS run. GPU clock behavior during idle can
+affect intervals; no hardware occupancy is inferred. Timestamp semantics:
+[Microsoft D3D12 timing](https://learn.microsoft.com/en-us/windows/win32/direct3d12/timing).
+
+The timestamp-enabled 4K burst/recreation gate completes twelve frames, all
+finite; all six async RGB outputs still match their fresh controls exactly.
+Each async mode emits all three timing records, including the final frame
+at feature release. Results: `test-results/k-async-gpu-boundary-4k`.
+All sixteen CTest gates pass on the rebuilt shim. Reproduce with:
+
+```powershell
+.\scripts\windows\test-async-k.ps1 -ProfileGpuBoundary
+```
+
+The 2026-10-02 game timestamp run completes 9384 K/4K frames without backend
+errors (`test-results/silent-hill2-k-gpu-boundary-4k`). Median GPU intervals:
+input copies 0.061 ms, external span 5.586 ms, output copy 0.208 ms, entire
+boundary 5.856 ms, between boundaries 9.512 ms. Median input interval is
+15.383 ms. Output scans and serializing kernel events are off. This locates
+the remaining time outside the small D3D12 copies; it does not prove whether
+that time is game GPU work, CPU waits or context scheduling.
+
+### Present and GPU-busy capture
+
+`capture-presentmon.ps1` attaches the official PresentMon 2.6.0 x64 CLI to
+one running game process. It verifies SHA256
+`b2a706bc6ad475749e3b7e3409263aa1e6906d45bdcf993f6dbc0f660188f1af`,
+uses a unique ETW session and stops at its time bound/process exit, without
+replacing any existing trace or installing a service. It saves CSV, console
+output, driver version, target PID and the HWS registry setting. The default
+downloads the pinned CLI into `.tools` if needed. Run elevated when ETW access
+requires it; no driver or security setting is changed.
+
+```powershell
+.\scripts\windows\capture-presentmon.ps1 -Seconds 30
+python .\tools\windows\profile_report.py <capture-directory> --output profile.json
+```
+
+Keep the game foreground and use the same scene/quality setting for comparison.
+The report separates swapchains and preserves missing metrics. Present FPS
+comes from present intervals, rather than NGX recording timestamps. GPU
+busy/wait are ETW estimates, not a hardware sensor percentage; CPU busy is
+frame work/wait attribution before Present, not physical CPU utilization.
+HWS and cross-API context attribution can affect accuracy. Metric definitions:
+[PresentMon console documentation](https://github.com/GameTechDev/PresentMon/blob/v2.6.0/README-ConsoleApplication.md).
+
+The initial 60-second trace captures 2172 presents during a changing
+Independent Flip / Composed Flip session, with mean present interval
+27.593 ms (36.241 FPS), mean GPU busy 18.759 ms and GPU wait 2.303 ms.
+This is not a controlled capture of the user's 62 FPS scene and is excluded
+from speedup conclusions. Its containing game run completes 9627 K/4K frames
+without backend errors; ordinary recording intervals later return to about
+16 ms. A stable foreground-scene capture is needed before optimizing from
+the ETW metrics. Raw files: `test-results/silent-hill2-k-presentmon-4k`.
+
+The following 30-second capture starts after the user confirms the same
+foreground scene is ready. All 1902 presents use Hardware Independent Flip
+and SyncInterval 0; input/output resolution remains 2259x1271 -> 3840x2160.
+The mean present interval is 15.730 ms (**63.574 FPS**). PresentMon estimates
+mean GPU busy/wait at 14.504/1.110 ms, GPU time at 15.614 ms and CPU wait at
+0.052 ms. These frame-attributed metrics do not equal a hardware utilization
+sensor and do not establish a physical CPU limit. FSR4 has not been captured
+in this same scene, so its previously reported 80+ FPS remains a user reading.
+Raw files: `test-results/silent-hill2-k-steady-profile-4k`.
+
+The containing ten-minute diagnostic completes 36591 K frames, with zero
+backend failures, CPU image copies, previous-frame outputs or recorded game
+crashes. It deliberately stops at its time bound. Output scans and serializing
+kernel events are off. Whole-run median boundary timestamps measure input
+copies 0.063 ms, external span 5.399 ms and output copies 0.212 ms. The external
+span includes HIP execution and host/driver scheduling, not only transformer
+arithmetic. These medians cover the whole run; the PresentMon numbers above
+come from the explicitly selected 30-second scene.
+
+### Deferred kernel events and batched inputs
+
+The new optional `-ProfileKernelsDeferred` samples every seventeenth launch
+by default (`-KernelProfileEvery`). Its thread-local pool is bounded to 64
+event pairs. It queries ready events before selected launches and after the
+application's existing context synchronization, reusing a pair only after
+reading its old timestamps. It adds no explicit event/stream/device completion
+wait; timestamps, queries and logging still add overhead. Stream capture and
+legacy default-stream launches are skipped by default. `-ProfileLegacyStream`
+allows the latter explicitly, retaining their legacy cross-stream ordering.
+This opt-in is necessary to profile this DLSS K binary, whose measured launches
+use the legacy default stream. `completion_ms` now denotes collection lag for
+deferred records. The report separates serializing/deferred records and retains
+invalid timestamps and skipped samples outside kernel timing statistics.
+
+HIP 7.17.26386 reports some negative elapsed times in the tiny queued PTX
+diagnostic, including intervals exceeding one microsecond. These are logged as
+`D4R_KERNEL_PROFILE_INVALID`; they are not clamped or used as valid GPU samples.
+The cause is not established, and nonnegative samples alone cannot prove a
+runtime's timestamp accuracy. The diagnostic verifies all 516 PTX launches
+across primary/created contexts and individual/queued submission, checks final
+device output and guards, accounts for every collected event and verifies safe
+default-stream skipping. Results: `test-results/zluda-deferred-profile-final-ptx`.
+The same diagnostic with stable HIP SDK 7.2 also verifies all 516 outputs and
+reproduces negative intervals (`test-results/zluda-deferred-profile-stable-ptx`).
+A native K dec4 sample is negative too; this is not confined to tiny PTX kernels.
+Do not infer hardware/kernel utilization or a speedup from these event samples.
+HIP event semantics: [AMD event management](https://rocm.docs.amd.com/projects/HIP/en/latest/reference/hip_runtime_api/modules/event_management.html).
+
+The opt-in `-BatchInputCopies` (`D4R_BATCH_INPUT_COPIES=1`) enqueues native
+pixel decoding and device-to-array input copies on the shared HIP/ZLUDA legacy
+default stream. One all-stream completion remains before NGX can use any input
+on its own streams. The independent D3D12 input-fence check, NGX completion and
+same-frame output dependency remain. Output transfers retain their existing
+completion. Every image stays in VRAM. The ZLUDA implementation adds
+`cuMemcpy2DAsync_v2` using `hipMemcpyParam2DAsync`; format-80 array emulation
+is explicitly rejected because its synchronous implementation uses temporary
+host storage. The Windows path supplies canonical FP16/FP32 arrays instead.
+This experiment remains disabled by default. Correctness checks pass, but the
+controlled game comparison below shows only a small frame-time difference.
+
+```powershell
+.\scripts\windows\test-deferred-kernel-profile.ps1
+.\scripts\windows\test-async-k.ps1 -BatchInputCopies -ProfileGpuBoundary -ZludaRoot "$PWD\dist\zluda-windows-deferred-profile"
+```
+
+The rebuilt Windows shim and new ZLUDA pass all seventeen CTest gates,
+including the added asynchronous pitched FP16/FP32 array-copy test and
+padding checks. Completed boundary timestamps are also read before reuse if
+the copy fence advances between the first completion scan and slot selection;
+this prevents diagnostic records being overwritten without adding a wait.
+
+The depth/stencil and packed-format burst/recreation gates pass 24 full K/4K
+frames. All twelve candidate RGB outputs match fresh unbatched synchronous
+controls exactly, with finite RGBA. The former also enables deferred legacy
+events; invalid timestamps remain separate. These short correctness fixtures
+include initialization and do not establish steady-state FPS or a copy-stage
+speedup. Raw results: `test-results/k-batch-deferred-4k` and
+`test-results/k-batch-packed-4k`. ZLUDA candidate source is `67127dd`, isolated
+in `dist/zluda-windows-deferred-profile`; patches 0016/0017 and all preceding
+patches apply to the clean pinned base. The public prerelease is unchanged.
+After committing/rebuilding ZLUDA with a clean worktree, the twelve-frame
+four-way K/4K gate passes again with exact candidate RGB and finite RGBA
+(`test-results/k-batch-committed-source-4k`). This warms the committed runtime's
+JIT cache before the game comparison.
+
+The following user-confirmed 30-second foreground-scene capture contains 1925
+presents at **64.265 FPS**, versus the previous unbatched run's 63.574 FPS.
+Both use 2259x1271 -> 3840x2160, Independent Flip and SyncInterval 0, without
+kernel-event profiling, output scans or attached debuggers. Mean present time
+is 15.561 versus 15.730 ms, a 0.169 ms difference. This single pair does not
+establish repeatability or a large performance improvement. The containing
+ten-minute candidate run completes 35673 K frames with zero backend failures,
+CPU image copies, previous-frame outputs or recorded game crashes. Whole-run
+median input preparation drops from 0.482 to 0.332 ms; median external span
+is 5.438 versus 5.399 ms. Whole-run stages are not the selected PresentMon
+window. No GPU utilization conclusion is drawn from ETW estimates. Results:
+`test-results/silent-hill2-k-batch-inputs-4k`. The K performance gate stays open.
+
+### AMD driver telemetry
+
+A separate read-only diagnostic polls the installed driver's ADLX base C
+interfaces for device-wide usage, core/VRAM clocks, board power and temperatures.
+It adds no work to game queues and changes no tuning setting. Build it locally
+using the official SDK pinned at `32b5a740d42295c5dfe9026b9f52683da0f3af91`:
+
+```powershell
+.\scripts\windows\build-gpu-telemetry.ps1
+.\scripts\windows\capture-presentmon.ps1 -Seconds 30 -GpuTelemetry
+```
+
+The SDK retains its own AMD license; SDK headers, implementations, driver DLLs
+and the SDK-dependent executable are excluded from public game packages.
+The probe is independently written and uses the documented C ABI, including
+with LLVM-MinGW. It loads `amdadlx64.dll` only from Windows System32 and
+releases acquired interfaces before ADLX termination/unload. GPU selection
+must match exactly one device (`-GpuName` selects a different name substring).
+The process exits at its bound or target process exit. The capture retains
+CSV, raw stderr, exit/timeout, driver DLL version/hash and SDK build identity.
+No optional telemetry failure is silently converted to a valid measurement.
+
+CSV records independent host UTC/QPC timestamps and the driver's raw sample
+timestamp. On this installed ADLX 1.5.0.124, the latter is uptime despite the
+SDK description of epoch time; do not interpret it as UTC. Unsupported metrics
+are blank, zero utilization/clock remains valid, and the report uses each
+driver timestamp once. Device-wide usage is distinct from PresentMon's
+per-process frame attribution and is not measured WMMA occupancy. ADLX API:
+[AMD GPU metrics sample](https://gpuopen.com/manuals/adlx/adlx-c__perf_g_p_u_metrics/).
+
+The initial idle-device smoke test captures six valid polls, selects the real
+RX 9070 XT and terminates ADLX successfully. Board power/core clock/usage
+and temperatures are available; `GPUPower` is unsupported and stays absent
+from the report. The first poll costs roughly 10 ms for initialization;
+subsequent polls are about 0.08 ms at 500 ms intervals. This idle smoke test
+does not establish game clocks, utilization or an overhead-free benchmark.
+Results: `test-results/gpu-telemetry-smoke`.
+
+The subsequent combined-capture smoke test collects 246 presents and ten
+ADLX polls and exits cleanly (`test-results/gpu-telemetry-capture-smoke`).
+It includes startup/scene changes and is excluded from FPS comparisons.
+After the user confirms the foreground comparison scene, the 30-second run
+with detailed d4r stages/boundary timestamps disabled records **64.661 FPS**
+over 1935 presents. ADLX's sixty polls report mean usage **93.183%**, core
+clock **3124.667 MHz**, board power **316.717 W**, temperature 57 C and hotspot
+82.9 C. Mean PresentMon GPU busy/wait is 14.551/0.800 ms. All presents retain
+Independent Flip / SyncInterval 0. This records high device-wide driver usage
+in this run; the user identified the earlier 63% reading as Task Manager.
+It is not the same measurement as ADLX device-wide usage. Do not infer a physical CPU
+limit, isolated shader occupancy or a FSR4 speed comparison. Query overhead
+is median 0.094 ms per 500 ms poll, with a roughly 12 ms first query. Results:
+`test-results/silent-hill2-k-telemetry-unprofiled-4k`. The game stability
+summary completes 37187 K frames with zero backend failures, CPU image copies,
+previous-frame outputs or recorded crashes, stopping at its ten-minute bound.
+Remaining K work now prioritizes transformer/output-kernel cost; frame
+scheduling remains measured.
+
+### Rejected packed FP16 accumulator experiment
+
+An isolated `D4R_K_F16_WMMA` enc1 build uses gfx12's documented packed FP16
+C/D intrinsic, retaining the current operand/accumulator lane mapping. It
+removes most conversion instructions (1066 static F16/F32 conversions become
+two) and reduces main-kernel VGPRs from 218 to 201; LDS remains 24576 bytes,
+with zero spills. These are compiler/ISA counts, not GPU time or occupancy.
+The candidate is rejected before moving to another layer: its real replay
+changes 1187603 half codes in the output allocation, max absolute difference
+0.0390625 from the validated baseline. All values are finite. Against the
+current twelve-window NumPy reference, PSNR drops from baseline 93.7527 dB
+to 69.2377 dB; max absolute error rises from 0.0078125 to 0.0390625.
+The original comparison incorrectly selected a pre-normalization-fix saved
+output (78.7702 dB); a fresh replay matches `k-real-strict-dual-reference/enc1`
+exactly and supplies the corrected baseline here. The rejection still holds.
+Two independent PTX windows give 70.4022/70.08 dB. These pass the existing
+coarse reference thresholds but do not satisfy this optimization's baseline
+preservation gate. No game installation or performance claim follows.
+
+A public 64-matrix primitive reproducer isolates the difference to direct
+instruction arithmetic. The packed intrinsic and the layout adapter agree
+bit-for-bit. Direct FP16 output differs from FP32 WMMA rounded once to FP16
+even after one K16 step (1674 of 4096 elements, max absolute 0.00390625).
+After eight steps, 2874 elements differ, max absolute 0.0625. This rules out
+the adapter packing as the source of this fixture's difference; the hardware
+internal accumulation/rounding cause is not established. Production K keeps
+its existing FP32-instruction/FP16-rounding baseline and the experimental flag
+is off in every normal build. Results: `test-results/k-f16-wmma-enc1` and
+`test-results/wmma-f16-arithmetic-compare-fixed`.
+The finished `test-wmma-f16.ps1` repeats the same 64-case result. Rebuilding
+every ordinary K/M object leaves all sixteen byte-identical; all seventeen
+CTest gates pass and enc1 still compiles for gfx1101 (compile-only). Logs:
+`test-results/wmma-f16-script-check` and `test-results/k-f16-baseline-ctest.log`.
+
+```powershell
+.\scripts\windows\test-wmma-f16.ps1
+.\scripts\windows\build-native-k-f16-wmma.ps1 -Layer enc1
+```
+
+The first command compiles and runs the public arithmetic diagnostic without
+NVIDIA inputs. Its success means the reproducer executed correctly, not that
+the two instruction paths are equivalent; JSON retains `baselineEquivalent`
+and `gameOptimizationAccepted`. The second compiles the rejected experiment
+in a separate directory and generates no override manifest. Intrinsic API:
+[Clang gfx12 FP16 WMMA](https://clang.llvm.org/docs/AMDGPUBuiltinReference.html#builtin-amdgcn-wmma-f16-16x16x16-f16-w32-gfx12).
+
+### Controlled native replay timings
+
+`native_replay_probe` now supports `--benchmark-timing events|dispatch` and
+`--benchmark-batch 1..256`. Dispatch mode uses the documented
+`hipExtModuleLaunchKernel` start/stop events, with work-item dimensions derived
+from each module's launch metadata and flags zero. It does not change wave
+order or arithmetic. Control/candidate can have different native block/grid
+metadata; each launch uses its own geometry. The default remains event markers.
+
+Each AB/BA sample restores captured inputs outside timing, submits a series,
+waits for its end event, and logs both the event interval and independent host
+launch-to-completion interval divided by launch count. These are isolated
+diagnostics: batch intervals include dispatch/queue gaps and are not exclusive
+instruction execution times, frame latency, or game FPS. Zero, negative,
+nonfinite or event intervals exceeding the enclosing host interval are retained
+as `D4R_REPLAY_PROFILE_INVALID` and excluded. Reports keep timing sources,
+batch counts and log files separate, retaining host statistics and invalid
+samples; no partially valid pair is accepted by the benchmark runner.
+
+The new runner first executes fresh ordinary control/candidate replays and
+compares every allocation's SHA256 before timing. It stops at any mismatch,
+then checks the control output again after every benchmark. A successful
+timing run does not replace the independent NumPy/PTX correctness gates.
+Captured weights/IR and GPU outputs remain private.
+
+```powershell
+.\scripts\windows\benchmark-native-replay.ps1 -Module "$PWD\build\native-k-gfx1201\dltss_pwin_enc1_layer.hsaco" -Candidate "$PWD\build\native-k-gfx1201\dltss_pwin_enc1_layer.hsaco" -FixtureDirectory "$PWD\test-results\ngx-k-fp16-fenced-capture\replay\replay-000001-dltss_pwin_enc1_layer"
+```
+
+The same-module enc1 control has all exact allocations and 32 valid pairs per
+mode. Median per-launch times at the captured 512x288 workload are 0.054850 ms
+with separate single-launch events and 0.089700 ms with single-launch dispatch
+events. At batch 64 they converge to 0.048213/0.048017 ms. Thus dispatch events
+alone do not remove timing overhead; use the batch/host cross-check. All eleven
+K same-module controls pass sixteen pairs at batch 64 in both modes, including
+dec4. This does not fix or explain the previously recorded negative ZLUDA
+timestamps. Results: `test-results/k-replay-timing-methods` and
+`test-results/k-all-layers-batch-profile`.
+
+One-frame standalone K/4K HDR captures provide actual enc0/enc1/dec0 inputs.
+Each harness run uses the eleven validated native layers and private output
+tail, passes the finite-output scan and completes without network fallback. At this
+synthetic resolution enc1 uses a 61x35 grid; this workload is distinct from
+the game's 2259x1271 input. Experiments retain F32 WMMA instructions and strict
+FP16 rounding. `build-native-k-experiment.ps1` builds separate objects and no
+override manifest; no experimental object is installed by the game runner.
+
+CU/WGP and non-temporal load/store/both on enc1 preserve every allocation but
+do not improve its paired 4K timings. Position-only token tiling also preserves
+outputs: enc1 with two tiles and shared V spills four VGPRs and is about 9%
+slower; one-tile shared V has no measured gain. The two-tile shared-V enc0 run
+initially has a roughly 4% lower median paired ratio amid scheduling variance.
+A repeat with 64 AB/BA pairs at batch 64 gives control/candidate medians
+0.297060/0.282559 ms (events) and 0.299988/0.285011 ms (dispatch), with median
+paired ratios 0.950863/0.952240. A 512x288 replay independently gives ratios
+0.950396/0.952582. Both tested dec0 shared-V variants are slower or indistinguishable.
+
+Enc0 now selects shared V for the gfx12 strict baseline, preserving the existing
+two-token tiles, F32 WMMA and FP16 rounding. Gfx11 and arithmetic experiments
+retain their prior selection. `D4R_K_PRIVATE_V` restores the previous enc0 for
+comparison; the build helper exposes `-PrivateValues`. The exact CMake-produced
+object also passes a fresh 4K replay and 32 paired batch-64 samples, with ratios
+0.955418/0.948494. Its ELF hash differs from the experimental object; `.text`
+and `.rodata` match exactly, and numerical validation is repeated on the actual
+object. The other fifteen ordinary K/M objects remain byte-identical.
+Gfx1101 enc0/enc1 compile-only controls retain identical code and constant
+sections to their original sources; no RDNA3 hardware execution is claimed.
+
+Full K controls/candidates complete sixteen 4K frames, and twelve more with
+the game's 2259x1271 input and 3840x2160 output. Every compared RGB half code
+matches, all RGBA scans are finite. Results: `test-results/k-enc0-shared-full-4k`,
+`k-enc0-shared-full-game-input`, `k-4k-enc0-pos2-shared-repeat`,
+`k-enc0-pos2-shared-small` and `k-enc0-default-shared-4k`. The saving is about
+0.015 ms for enc0 in the isolated 4K replay, not 5% application FPS.
+
+The separate `D4R_K_PACKED_ACC` experiment stores already-rounded accumulators
+as half vectors between steps, widening for each unchanged F32 WMMA and
+rounding its F32 output back to half. The ordinary `acc8v` alias remains F32.
+Enc0/enc1/dec0 4K replays match every allocation exactly. Enc1 main VGPRs fall
+218 -> 195, LDS remains 24576 bytes and spills stay zero, but paired GPU time
+does not improve; enc0 is about 4% slower and dec0 about 1% slower. Combining
+packed storage with enc1 two-tile shared V is about 13% slower. The flag remains
+off. Smaller register counts alone are not evidence of an optimization.
+Raw results are under `test-results/k-4k-*-packed-acc` and
+`test-results/k-4k-enc1-pos2-shared-packed`.
+
+A direct recompilation of the private output tail's optimized LLVM IR with
+TheRock Clang 24 fails in instruction selection with `unsupported library
+call operation` (driver exit 1, backend exit 70, exception 0xE0000046). The generated crash
+reproducer/log are retained locally; strict FP is not removed to work around
+this. The working LLVM 22 output object remains selected. The HDR tail's
+metadata is wave32, 133 VGPRs, 6912 LDS bytes and no spills; LDR uses 128 VGPRs
+with the same LDS and no spills. These are static resource counts, not actual
+occupancy. Compiler scheduling hints are documented in the
+[LLVM AMDGPU backend guide](https://llvm.org/docs/AMDGPUUsage.html#llvm-ir-attributes).
+
+### Private output-tail WGP experiment
+
+ZLUDA source `fb0adc8714f57647320f0601533ec50331e378b0`, patch 0018, adds
+opt-in `D4R_ZLUDA_WGP=1`. It changes only CU/WGP placement for gfx11/gfx12,
+preserving wave size and arithmetic. The ordinary runtime stays in CU mode.
+The offline texture builder accepts `-ShaderMode wgp` and verifies `-cumode`
+on the emitted kernel before accepting the object. The compiled output-tail
+objects carry this mode independently of the runtime's environment variable.
+These NVIDIA-derived objects remain private; the public package excludes them.
+
+The matched K/4K comparison uses identical public network objects and the
+existing validated CU output kernels as control. Both HDR and LDR complete
+eight frames per variant; repeating with candidate-first order completes
+sixteen frames per variant. All 96 control/candidate frames are finite, and
+all 48 candidate RGB outputs exactly match their fresh controls. Every frame
+has a native output-tail hit and a valid serializing event measurement.
+
+| Output-tail median, ms | CU, first run | WGP, first run | CU, reverse order | WGP, reverse order |
+| --- | ---: | ---: | ---: | ---: |
+| LDR | 1.864650 | 1.703100 | 1.840260 | 1.687500 |
+| HDR | 2.012301 | 1.883395 | 2.016020 | 1.869250 |
+
+These are separate-process serializing diagnostic measurements, not paired
+resident-module timings or game FPS. Unchanged network kernels also vary
+between processes. The repeat indicates a candidate worth testing in the
+game; it does not establish a frame-rate improvement. Results:
+`test-results/k-output-wgp-reference-4k-clean-env` and
+`k-output-wgp-reference-4k-reverse`.
+The game-sized WGP async burst/recreate gate completes twelve frames, all
+finite; all six async RGB outputs match fresh sync controls exactly, with
+no CPU image copy or aged output (`k-output-wgp-async-game-input`).
+The exact CMake shared-V enc0 and WGP output tails together also pass twelve
+game-sized async/burst/recreation frames, with all six async RGB images exact
+(`k-enc0-shared-wgp-async-game-input`). Default selection of private output
+objects remains CU until a game comparison establishes the candidate's effect.
+
+### Combined K game check
+
+The private WGP outputs are rebuilt from clean ZLUDA source `fb0adc8`; both
+object hashes match the earlier validated experimental objects. A final
+2259x1271 -> 3840x2160 comparison completes twelve finite control/candidate
+frames, all six candidate RGB outputs exact, with all expected native hits and
+valid timings (`k-output-wgp-clean-source-final-4k`). This supplies clean build
+provenance without changing the tested binary identities.
+
+The subsequent Silent Hill 2 game check combines the exact CMake shared-V enc0
+and those private WGP outputs. Package source is clean `68b551b`; the runtime
+remains the validated `67127dd` ZLUDA build with its warm cache, while the
+offline output objects carry WGP placement. DLSS is the locally provided
+310.9.1.0 DLL. Async interop and batched GPU inputs are enabled; debugger,
+serializing/deferred kernel profiling, detailed stage/boundary timestamps and
+output scans are disabled.
+
+After the user confirms the stationary foreground comparison scene, thirty
+seconds capture 1954 presents: **65.250 FPS**, mean/median/p95 frame intervals
+15.326/15.318/16.306 ms. Every present uses Hardware Independent Flip with
+SyncInterval 0. PresentMon mean GPU busy/wait is 14.403/0.807 ms. Sixty unique
+ADLX polls record **93.500%** mean device usage, **3118.400 MHz** core,
+2505 MHz VRAM and **318.817 W** board power, with 57 C temperature and 82.9 C
+hotspot. Median telemetry query time is 0.083 ms per 500 ms poll.
+
+The preceding accepted scene capture was 64.661 FPS: the new result is only
+0.589 FPS higher, about 0.91%, with mean frame interval 0.139 ms shorter.
+This is one historical control and one candidate capture, without a fresh
+repeated game A/B. It does not establish a reproducible gain or isolate enc0
+from output placement. WGP remains an optional experiment, and the overall
+K performance gate stays open.
+
+The full bounded run completes **35664 K frames** and **427968 native
+launches**, including feature recreation and color-format reimports. There
+are zero backend errors, CPU image copies, previous-frame outputs or recorded
+crashes. `diagnostic_timeout` denotes the intentional ten-minute stop.
+Numerical game scans are off, so the game's zero reported nonfinite count is
+not a NaN/Inf validation result; use the separate finite/exact-frame gates.
+Raw capture, environment, package identities and summary remain in
+`test-results/silent-hill2-k-vshare-wgp-4k` and its diagnostic ZIP.
+
+```powershell
+.\scripts\windows\build-zluda-windows.ps1 -InstallRoot "$PWD\dist\zluda-windows-wgp-experiment"
+.\scripts\windows\build-native-texture.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -ZludaRoot "$PWD\dist\zluda-windows-wgp-experiment" -ShaderMode wgp -OutputDirectory "$PWD\build\private-textures-wgp-gfx1201"
+.\scripts\windows\build-native-texture.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -ZludaRoot "$PWD\dist\zluda-windows-wgp-experiment" -ShaderMode wgp -Kernel hiluma_engine_output_depthreg_mvhi_ldr_max_v2_rel -OutputDirectory "$PWD\build\private-textures-wgp-gfx1201"
+.\scripts\windows\test-native-texture.ps1 -TextureRoot "$PWD\build\private-textures-wgp-gfx1201" -ControlTextureRoot "$PWD\build\private-textures-gfx1201" -ZludaRoot "$PWD\dist\zluda-windows-deferred-profile" -OutputResolution 3840x2160
+```
+
+The compiler placement unit passes, all seventeen hardware/software CTest gates
+pass on the clean source build with ordinary CU defaults, and the six ZLUDA
+gates also pass with WGP explicitly enabled. All seventeen patches (0002..0018)
+apply to the clean pinned base `ee2f25a`. The numerical benchmark's rejected
+F16 candidate stops after its two ordinary reference replays with differing
+output and no timing samples (`k-replay-rejected-f16-gate-fixed`).
+
+### Diagnostic environment and input dimensions
+
+Some PowerShell/.NET hosts preserve removed process variables as empty strings.
+Presence-based Rust diagnostic flags then select deferred profiling, while
+empty codegen values can disable WMMA. The probe/game child environments now
+omit empty `D4R_` keys; the probe restores codegen defaults for null or empty
+values. Disabled WGP is also omitted so older runtimes keep their warm cache.
+The texture test requires valid timing records instead of silently accepting
+a profiling-free comparison. The initial candidate attempt times out during
+NGX initialization with no accepted frame; later pre-fix comparisons have
+correct RGB but no accepted GPU timings. Their logs remain in
+`k-output-wgp-reference-small`, `k-output-wgp-reference-small-fixed` and
+`k-output-wgp-reference-4k-fixed`; they are excluded from the table above.
+
+The D3D12 harness now supports `--ngx-input-resolution WIDTHxHEIGHT`, exposed
+as `-NgxInputResolution` in the probe runner and `-InputResolution` in the
+texture/async gates. Output defaults and limits remain unchanged. This allows
+the game's non-aligned input dimensions without inventing a larger output.
+An initial attempt using 4518x2542 output is rejected by the harness's existing
+range guard before GPU initialization; it is not a GPU or enc0 failure.
+
+### Rejected K input-rounding overrides
+
+The remaining translated K input is about 0.34-0.40 ms in the earlier
+game-sized standalone profile. The private builder now exposes the existing
+surface-store/rounding recipe for both tested input variants. These inputs
+have **zero surface stores**: the actual change replaces eight round-half-away
+idioms with the existing recipe, avoiding constrained arithmetic lowering.
+It is not a native surface-store saving. Build metadata records the recipe's
+counts and refuses a no-op override.
+
+CU and WGP input objects are built from the same clean `fb0adc8` ZLUDA source.
+Both retain accuracy mode and wave32; native FP8 remains disabled. The full
+pipeline control and candidate share all thirteen other code objects,
+including the same private WGP output tails. Only the two input overrides
+are added. Four frames per variant at 512x288, then eight per variant at
+2259x1271 -> 3840x2160 in CU mode, then eight in WGP mode with candidate-first
+order complete **80 finite control/candidate frames**. All forty candidate
+RGB images match exactly. Every expected input launch and valid serializing
+timing is checked; raw logs and module SHA256 values are retained.
+
+| Input median, ms | Translated, CU test | Native CU | Translated, WGP test | Native WGP |
+| --- | ---: | ---: | ---: | ---: |
+| LDR / regular depth / high-resolution MV | 0.350050 | 0.359425 | 0.351000 | 0.358800 |
+| HDR / inverted depth / low-resolution MV | 0.512300 | 0.556350 | 0.412650 | 0.552100 |
+
+These are separate-process diagnostic measurements with eight samples each;
+other unchanged kernels also vary between processes. Neither input candidate
+shows a saving. Both are rejected for game installation and default packaging;
+the validated game configuration keeps translated input. Results:
+`test-results/k-input-native-stores-small` (initial filename; no stores were
+replaced), `k-input-rounding-cu-game-input`,
+`k-input-rounding-wgp-game-input-reverse` and
+`k-input-rounding-experiment-summary.json`.
+
+The comparison script's `-KernelStage input` selects the input hit/timing gate.
+`-CommonTextureRoot` holds output objects identical in both processes; staged
+object hashes are recorded. The ordinary output gate now accepts a translated
+control when its object is absent, while still requiring a native candidate.
+Rebuilding the two original WGP output objects preserves their SHA256 values;
+the default translated-output comparison also passes four finite frames and
+two exact candidate RGB images
+(`k-output-recipe-translated-control-regression`). Input objects remain private
+and are excluded by the game's existing output-only installation allowlist.
+
+Reproduce the WGP experiment with locally built, validated output tails:
+
+```powershell
+foreach ($kernel in @('hiluma_engine_input_depthinv_mvlo_hdr_v2_rel', 'hiluma_engine_input_depthreg_mvhi_ldr_v2_rel')) {
+    .\scripts\windows\build-native-texture.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -ZludaRoot "$PWD\dist\zluda-windows-wgp-experiment" -ShaderMode wgp -Kernel $kernel -OutputDirectory "$PWD\build\private-input-textures-wgp-gfx1201"
+}
+.\scripts\windows\test-native-texture.ps1 -KernelStage input -TextureRoot "$PWD\build\private-input-textures-wgp-gfx1201" -CommonTextureRoot "$PWD\build\private-textures-wgp-gfx1201" -ZludaRoot "$PWD\dist\zluda-windows-deferred-profile" -InputResolution 2259x1271 -OutputResolution 3840x2160 -Iterations 8 -CandidateFirst -OutputDirectory "$PWD\test-results\k-input-rounding-wgp-retest"
+```
+
+Further game coverage (HIP kernel profiling disabled, GPU output checks enabled):
+M at 1920x1080 completes 4833 frames, all finite. K at 3840x2160 completes
+4685 frames, all finite. Neither has a backend failure, CPU image copy or an
+aged-frame fallback. Their mean interop/NGX CPU stages total about 13.6 and
+10.1 ms respectively; these are different scenes, not a K/M comparison.
+
+The no-debugger K/4K run completes 5774 frames, with 5774 finite output scans
+and 63514 native transformer launches. The user reports 38-42 FPS. Additional
+CPU measurements exclude replay/publication as the main cause of low FPS:
+
+| CPU measurement | Mean, ms | Median, ms |
+| --- | ---: | ---: |
+| Command split setup | 0.284 | 0.275 |
+| State replay within split setup | 0.025 | 0.023 |
+| Submitted texture state publication | 0.029 | 0.029 |
+| Consumer suffix completion | 0.405 | 0.409 |
+| NGX recording interval | 27.474 | 28.466 |
+
+Recording intervals are between NGX recording calls on the same thread, not
+DXGI Present times. The interval includes rendering and application scheduling.
+A second K/4K run with the attached debugger completes 3567 frames without
+failure, with median recording interval 30.091 ms. The runs include different
+menu/game states; this does not establish a precise debugger FPS speedup.
+Its sparse debugger events also rule out a per-kernel OutputDebugString storm.
+An earlier control that stayed on the startup screen completed zero DLSS frames
+and is excluded. Normal game launches now omit the attached debugger; crash
+diagnostics remain available with `-CaptureExceptions`.
+
+The 2961-frame 4K K profile identifies the translated output tail as its largest
+kernel: 4.401670 ms mean GPU time, versus about 2.875 ms for all eleven native
+transformer layers combined. Ported upstream's texture-store recipe to native
+Windows. It retains accuracy mode, wave32, FP16 subnormals and f16 reference
+accumulation; native FP8 remains disabled.
+
+Both output variants execute on every checked frame and match the previous
+RGB output bit-exactly: eight consecutive 512x288 frames per variant, then
+three consecutive 3840x2160 frames per variant. All RGBA components are finite.
+Identical synthetic inputs and public NGX flags give these 4K event times
+(three samples each; host initialization and readback are outside the event):
+
+| K output variant | Translated mean, ms | Native stores mean, ms |
+| --- | ---: | ---: |
+| LDR, regular depth, high-resolution MV | 2.907823 | 1.803233 |
+| HDR, inverted depth, low-resolution MV | 2.978547 | 2.088347 |
+
+A subsequent game run executes 6221 frames with the native HDR output tail,
+zero backend failures and mean tail GPU time 1.801918 ms. The earlier game
+profile's 4.401670 ms is from a different scene/scheduling state; use the
+identical-input table for the controlled comparison, not an inferred FPS
+multiplier. This profiled game run did not enable the NaN scan.
+
+Reproduce the private texture build and full-frame comparisons:
+
+```powershell
+.\scripts\windows\build-native-texture.ps1 -DlssDll "$PWD\nvngx_dlss.dll"
+.\scripts\windows\build-native-texture.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -Kernel hiluma_engine_output_depthreg_mvhi_ldr_max_v2_rel
+.\scripts\windows\test-native-texture.ps1 -OutputResolution 3840x2160 -Iterations 3
+```
+
+These objects contain code derived from the locally supplied NVIDIA PTX. They
+stay in ignored local directories and are excluded from the public package.
+The installer accepts them with `-LocalTextureKernels` only after validation,
+checking the exact DLL identity, object and manifest hashes. M retains its
+independently validated FP16-equivalent baseline.
+
+The following no-debugger K/4K run completes 9982 frames and 9982 finite GPU
+scans. Kernel event profiling is disabled; the validated private output tail
+executes natively. The user reports 43-47 FPS. Mean NGX recording interval is
+21.867 ms. No backend failure, CPU image copy or previous-frame output occurs.
+
+Random one-in-64 hook samples then identify contention in the shared routing
+table used by every intercepted D3D12 command. Per-thread weak lookup caches
+now bypass that table on repeated calls to a list. Table mutations invalidate
+cache generations; Reset/address reuse cannot route to a previous suffix, and
+thread caches do not retain game resources. The per-list and submission locks
+remain. A GPU regression resets and re-splits the exact same list with changed
+root constants twice per iteration, checking both prefix and suffix results.
+
+| Sampled command-hook access | Shared table | Per-thread weak cache |
+| --- | ---: | ---: |
+| Busiest recording thread, mean us/call | 0.203 | 0.066 |
+| Estimated access + capture across threads, ms/s | 977.528 | 233.606 |
+
+The latter estimate includes lock waits summed across concurrently running
+threads, not CPU execution time or serial frame latency. The two game runs
+have different thread populations (31 versus 29); there is no identical frame
+trace. Generated forwarding methods have separate driver/capture markers;
+special-case hooks contribute access samples but lack those body markers.
+
+With the cache, 4770 complete K/4K frames and finite GPU scans pass, with zero
+backend failures, CPU image copies or previous-frame output. Median NGX
+recording interval is 19.473 ms. The user reports 49-51 FPS and 53% GPU usage
+in the previously measured scene. These user readings are separate from the
+sampled hook estimates and do not establish isolated WMMA utilization.
+All sixteen CTest gates pass, including 64 same-object Reset/split checks in
+each physical command-backend test and the software indirect-root regression.
+Use `-ProfileCommandHooks -ProfileStages` to collect samples; the sampler is
+disabled for ordinary play.
+
+Native M FP8 was subsequently compiled and tested on this gfx1201. ISA confirms
+`v_wmma_f32_16x16x16_fp8_fp8`, with packed e4m3 weight images; exact FP16
+attention/PV remains. All logical outputs and merges in forty recorded launches
+match the previously NumPy-validated FP16 baseline byte-for-byte. Full DLSS RGB
+also matches exactly: four 512x288 LDR frames and four 3840x2160 HDR frames,
+all RGBA finite. This establishes correctness for those fixtures, not a speedup.
+
+The original software operand packing makes the FP8 network slower in the
+four-frame 4K test: sum of five main-kernel event means (tube weighted six times)
+is 27.789640 ms versus FP16's 23.550875 ms. A paired in-process replay confirms
+the regression without relying solely on separate process timings. Both modules
+and prepared weights remain resident, 512 launches warm the GPU, then sixty-four
+AB/BA pairs time the same restored input using HIP events. Diagnostic CPU input
+restoration is outside the timed event and is unrelated to game VRAM interop.
+
+Added opt-in `D4R_FP8_HW_PACK`: native packed conversion of already quantised
+operands replaces software exponent/subnormal reconstruction. It preserves all
+254 finite OCP e4m3 encodings, including signed zero, in an exhaustive GPU check.
+All forty recorded launches still match exactly. Four complete 4K HDR frames
+also match FP16 RGB exactly with finite RGBA. ISA confirms both packed converts
+and FP8 WMMA. The baseline build's sixteen objects remain byte-identical.
+
+| Paired replay median, ms | FP16 (software-packing test) | FP8 software packing | FP16 (hardware-packing test) | FP8 hardware packing |
+| --- | ---: | ---: | ---: | ---: |
+| enc1 | 0.175450 | 0.217050 | 0.175100 | 0.177700 |
+| enc2 | 0.141350 | 0.164550 | 0.132550 | 0.131550 |
+| enc3 tube | 0.108000 | 0.112000 | 0.107600 | 0.099850 |
+| dec2 | 0.129900 | 0.146000 | 0.143250 | 0.140850 |
+| dec1 | 0.169650 | 0.194450 | 0.154250 | 0.156550 |
+
+These are two paired tests on the captured 512x288 workload. Medians avoid large
+isolated scheduling outliers; raw per-pair ratios are retained. They identify a
+substantial cost in software packing but do not establish an overall FPS gain.
+The separately run four-frame 4K hardware-packing network mean is 24.603463 ms;
+it still does not beat the measured FP16 total. Default packaging retains FP16.
+Hardware counters for bandwidth or WMMA utilization were not measured.
+
+Reproduce the isolated build, encoding/replay checks and paired timings using
+the existing private validated captures; no source editing is required:
+
+```powershell
+.\scripts\windows\build-native-m-fp8.ps1 -DlssDll "$PWD\nvngx_dlss.dll" -HardwarePacking
+.\scripts\windows\test-native-m-fp8.ps1 -ModuleDirectory "$PWD\build\native-m-fp8-hardware-pack" -CaptureDirectory "$PWD\test-results\ngx-m-exact-pv-capture\captures" -BaselineValidation "$PWD\test-results\m-native-exact-pv-temporal-reference\summary.json" -Benchmark
+```
+
+Omit `-HardwarePacking` to compile the original FP8 experiment. The replay
+validator stops at the first mismatch, retains GPU stdout/stderr/exit code and
+hashes every compared logical output. `profile_report.py` accepts replay logs
+and reports per-pair GPU time and candidate/control ratios. This experiment is
+not installed by the game runner or substituted silently for the baseline.
+The conversion builtin is listed in the official
+[Clang AMDGPU builtin reference](https://clang.llvm.org/docs/AMDGPUBuiltinReference.html#builtin-amdgcn-cvt-pk-fp8-f32).
+
+References: [HIP events](https://rocmdocs.amd.com/projects/HIP/en/develop/doxygen/html/group___event.html),
+[HIP occupancy API](https://rocm.docs.amd.com/projects/HIP/en/latest/doxygen/html/group___occupancy.html).

docs/windows-quick-test.txt

@@ -0,0 +1,62 @@
+D4R WINDOWS QUICK TEST -- READ THIS FIRST
+
+This is an EXPERIMENTAL Windows 11 D3D12 build for RDNA3 / RDNA3.5 / RDNA4.
+Only RX 9070 XT has been tested here. Other cards may fail.
+The package detects the real GPU automatically. Do not rename any DLLs.
+
+1. EXTRACT THE WHOLE ZIP to a normal writable folder, e.g. Desktop\d4r-test.
+   Do not run it inside the ZIP. Do not copy files into the game manually.
+   Already installed an older test? Close the game and run RESTORE-GAME.cmd
+   from that old package first, then extract this version to a new folder.
+
+2. CLOSE YOUR GAME. If your distributor included the validated NVIDIA DLL pair,
+   it is selected automatically; you only need to choose the game executable.
+   Otherwise have these TWO ORIGINAL NVIDIA files ready:
+   _nvngx.dll       version 32.0.16.1714
+   nvngx_dlss.dll   version 310.9.1
+   The xdfnx-dev public test ZIP does not include them. Other versions are rejected.
+   The small _nvngx.dll inside this package is d4r, NOT the original NVIDIA file.
+
+3. DOUBLE-CLICK START-K.cmd. Pick the actual game .exe when asked.
+   If asked, pick your two NVIDIA DLLs. These choices are remembered for the next run.
+   Next time, press Enter for the same game, or C to choose another.
+   If Steam/Epic is needed, keep it open. Choose a D3D12 game without anti-cheat.
+   If a launcher opens, type C and choose the real game executable next time.
+
+4. IN THE GAME, select DLSS if available, otherwise FSR/XeSS for OptiScaler.
+   Leave frame generation OFF. First launch can compile for several minutes.
+   Keep the console open. Play for 30 seconds, check the image, then EXIT THE GAME.
+   START-M.cmd tests DLSS 4.5 preset M; START-K.cmd tests DLSS 4 preset K.
+
+5. SEND THE SINGLE ZIP shown as "SEND THIS ONE FILE" in the console.
+   It is also in the results folder. Tell us your GPU, game, K/M and image result:
+   "correct image", "black screen", "crash" or "visual corruption".
+   A passing backend log does not automatically prove the image is correct.
+   Logs can contain local paths. Review them before posting them publicly.
+
+TO UNINSTALL: close the game and double-click RESTORE-GAME.cmd.
+Original replaced files are backed up and restored; changed files are preserved.
+Restore the previous game BEFORE choosing a different game for the next test.
+Always launch tests through START-K/M.cmd, not the usual game shortcut.
+
+IF SOMETHING FAILS:
+- Wrong NVIDIA DLL: select the exact versions above on the next run.
+- No supported GPU / HIP failure: send the ZIP; do not spoof the GPU architecture.
+- vkd3d/Wine detected: use the game's original native Windows D3D12 files.
+- Access denied: extract to a writable folder. Do not disable system security.
+- Do not start a second test while the first game is running.
+
+No HIP SDK installation is required; the isolated runtime is included.
+This uses synchronous interop and output checks. No debugger is attached by default.
+Exit codes and loaded DLLs are logged. A debugger is only an explicit developer option.
+Your local DLSS DLL is staged beside the d4r shim automatically.
+It is a correctness diagnostic, NOT an FPS benchmark or an optimized daily-use build.
+Colour, depth, motion and output remain in VRAM in the runtime fast path.
+
+Runtime files are in files/. Cache/settings are created in work/.
+Licenses are in licenses/. Corresponding sources are a SEPARATE release download;
+testers do not need to download or build them. NVIDIA DLLs, when supplied by a
+distributor, keep their separate license and are not part of the GPL source archive.
+
+Report: https://github.com/countervolts/d4r/issues/10
+Source: https://github.com/xdfnx-dev/d4r

kernels/build.sh

@@ -1,32 +1,62 @@
 #!/usr/bin/env bash
-# Builds the native RDNA3 replacements for DLSS kernels into one directory that ZLUDA serves them from
+# Builds the native replacements for DLSS kernels into one directory that ZLUDA serves them from
 # (D4R_ZLUDA_NATIVE_DIR, or NativeKernelDirFast in d4r.ini). See docs/native-kernels.md.
 #
-# usage: kernels/build.sh [all|k|m|tex] [OUT_DIR]      (default: all, kernels/out/native)
+# usage: kernels/build.sh [all|k|l|m|tex] [OUT_DIR]      (default: all, kernels/out/native)
 #
 #   D4R_ROCM_DIR    ROCm installation with clang and the HIP device libraries (default /opt/rocm)
-#   D4R_GPU_ARCH    target GPU (default gfx1101; the kernels need RDNA3 / gfx11 WMMA)
+#   D4R_GPU_ARCH    target GPU (default gfx1101): an RDNA3 (gfx110x) or RDNA4 (gfx120x) target; the kernels
+#                   need wave32 WMMA, and kernels/common/wmma_layout.h picks the register layout per target
+#   D4R_NATIVE_FP8  1 (RDNA4 only): the variant for ZLUDA's native FP8 WMMA (d4r.ini [Kernels] NativeFp8):
+#                   M's layers on FP8 WMMA with NVIDIA's e4m3 activations, texture tails matching ZLUDA's
+#                   D4R_ZLUDA_WMMA_FP8_NATIVE lowering. K is the same in both variants.
+#   D4R_PREFER_ACCURACY  1: preserve FP8 quantization, per-MMA f16 rounding and denormal handling;
+#                   build a separate set (default output: kernels/out/native/accuracy).
 # Texture kernels (tex) are NVIDIA's PTX with parts replaced, so they additionally need:
 #   D4R_DLSS_DLL    nvngx_dlss.dll 310.7 (its PTX is extracted into kernels/extracted/, never committed)
-#   D4R_ZLUDA_BUILD a ZLUDA build with patches/zluda applied (directory holding libcuda.so)
+#   D4R_ZLUDA_EMIT  ZLUDA's d4r_emit (patches/zluda applied; `cargo build --release -p ptx --example d4r_emit`),
+#                   which compiles PTX offline for D4R_GPU_ARCH (no GPU of that kind needed)
+#   D4R_DLSS_PTX_DIR optional pre-extracted PTX directory for parallel per-target builds
 set -euo pipefail
 
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 WHAT="${1:-all}"
-OUT="$(realpath -m "${2:-$HERE/out/native}")"
+ACCURACY="${D4R_PREFER_ACCURACY:-0}"
+[[ "$ACCURACY" == 0 || "$ACCURACY" == 1 ]] || { echo "D4R_PREFER_ACCURACY must be 0 or 1" >&2; exit 2; }
+DEFAULT_OUT="$HERE/out/native"
+[[ "$ACCURACY" == 1 ]] && DEFAULT_OUT+="/accuracy"
+OUT="$(realpath -m "${2:-$DEFAULT_OUT}")"
 ROCM="${D4R_ROCM_DIR:-/opt/rocm}"
 ARCH="${D4R_GPU_ARCH:-gfx1101}"
+FP8="${D4R_NATIVE_FP8:-0}"
+case "$ARCH" in
+    gfx11[0-9][0-9]) [[ "$FP8" == 1 ]] && { echo "D4R_NATIVE_FP8 needs an RDNA4 (gfx12) target" >&2; exit 2; } ;;
+    gfx12[0-9][0-9]) ;;
+    *) echo "unsupported D4R_GPU_ARCH $ARCH (RDNA3 gfx110x or RDNA4 gfx120x)" >&2; exit 2 ;;
+esac
 CLANG="$ROCM/lib/llvm/bin/clang++"
 [[ -x "$CLANG" ]] || { echo "clang++ not found in $ROCM/lib/llvm/bin (set D4R_ROCM_DIR)" >&2; exit 2; }
 mkdir -p "$OUT"
+case "$WHAT" in all|k|l|m|tex) ;; *) echo "unknown kernel family: $WHAT" >&2; exit 2 ;; esac
+# Never certify a directory containing older fast binaries as an accuracy set.
+if [[ "$ACCURACY" == 1 ]] && compgen -G "$OUT/*.hsaco" >/dev/null &&
+    { [[ ! -f "$OUT/d4r-accuracy.txt" ]] || [[ "$(cat "$OUT/d4r-accuracy.txt")" != 1 ]]; }; then
+    echo "accuracy kernels need an empty output directory or an existing accuracy set: $OUT" >&2
+    exit 2
+fi
+rm -f "$OUT/d4r-accuracy.txt"
 
 # HIP source -> raw code object (the kernel name inside matches the DLSS kernel it replaces)
 build_hip() {
-    local src="$1" name
+    local src="$1" extra="${2:-}" name
     name="$(basename "$src" .hip)"
-    local tmp
+    local tmp flags
     tmp="$(mktemp -d)"
-    (cd "$tmp" && "$CLANG" -x hip --offload-arch="$ARCH" --offload-device-only -O3 -I"$ROCM/include" \
+    # per-kernel compiler flags from a "// d4r-build-flags: ..." line in the source
+    flags="$(sed -n 's|^// d4r-build-flags: *||p' "$src")"
+    [[ "$ACCURACY" == 1 ]] && extra+=" -DD4R_ACCURACY"
+    # shellcheck disable=SC2086
+    (cd "$tmp" && "$CLANG" -x hip --offload-arch="$ARCH" --offload-device-only -O3 $flags $extra -I"$ROCM/include" \
         --rocm-path="$ROCM" --rocm-device-lib-path="$ROCM/amdgcn/bitcode" -I"$(dirname "$src")" \
         -o "$OUT/$name.hsaco" "$src" -save-temps=cwd --no-gpu-bundle-output)
     printf '%-52s %s\n' "$name" "$(grep -hE '^; (NumVgprs|ScratchSize|Occupancy)' "$tmp"/*.s | tail -3 | paste -sd' ')"
@@ -37,25 +67,59 @@ if [[ "$WHAT" == all || "$WHAT" == k ]]; then
     echo "== DLSS 4 (preset K) transformer layers"
     for src in "$HERE"/k/dltss_pwin_*.hip; do build_hip "$src"; done
 fi
-if [[ "$WHAT" == all || "$WHAT" == m ]]; then
-    echo "== DLSS 4.5 (preset M) Swin layers"
-    for src in "$HERE"/m/rrlite_*.hip; do build_hip "$src"; done
+if [[ "$WHAT" == all || "$WHAT" == m || "$WHAT" == l ]]; then
+    echo "== DLSS 4.5 (presets L/M) shared Swin layers"
+    for src in "$HERE"/m/rrlite_*.hip; do build_hip "$src" "$([[ "$FP8" == 1 ]] && echo -DD4R_FP8_WMMA)"; done
 fi
-if [[ "$WHAT" == all || "$WHAT" == tex ]]; then
+if [[ "$WHAT" == all || "$WHAT" == tex || "$WHAT" == l ]]; then
     echo "== texture kernels (NVIDIA PTX with native parts)"
     : "${D4R_DLSS_DLL:?set D4R_DLSS_DLL to nvngx_dlss.dll (310.7)}"
-    : "${D4R_ZLUDA_BUILD:?set D4R_ZLUDA_BUILD to a ZLUDA build with patches/zluda}"
-    PTX_DIR="$HERE/extracted/ptx"
-    python3 "$HERE/tools/extract_dlss_ptx.py" "$D4R_DLSS_DLL" "$PTX_DIR"
-    PTXLOAD="$HERE/out/tools/ptxload"
-    mkdir -p "$(dirname "$PTXLOAD")"
-    cc -O2 -o "$PTXLOAD" "$HERE/tools/ptxload.c" -ldl
-    # kernel : source of its native part
-    for pair in hiluma_engine_output_depthinv_mvhi_hdr_max_v2_rel:sust_only rrlite_post_3_2_mvhi_hdr_folded:sust_only \
-        rrlite_downsample_kernel_static_hdr:sust_only rrlite_enc0_4x4_mvhi_hdr_folded:enc0_tail \
-        rrlite_dec0_4x4_folded:dec0_head; do
-        D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" D4R_DLSS_PTX_DIR="$PTX_DIR" D4R_PTXLOAD="$PTXLOAD" \
-            "$HERE/tex/build_tex.sh" "${pair%%:*}" "${pair#*:}" "$OUT"
+    : "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT to d4r_emit from a ZLUDA build with patches/zluda}"
+    PTX_DIR="${D4R_DLSS_PTX_DIR:-$HERE/extracted/ptx}"
+    if [[ -z "${D4R_DLSS_PTX_DIR:-}" ]]; then
+        python3 "$HERE/tools/extract_dlss_ptx.py" "$D4R_DLSS_DLL" "$PTX_DIR"
+    elif [[ ! -d "$PTX_DIR" ]]; then
+        echo "missing extracted PTX directory: $PTX_DIR" >&2; exit 2
+    fi
+    # kernel : source of its native part [: w64 = compiled as wave64 (two CUDA warps per wave; RDNA3 issues
+    # FP32 work for all 64 lanes at once). Only for kernels without MMAs; measured faster, bit-identical:
+    # post 0.89 -> 0.77 ms, hiluma output 0.905 -> 0.887 ms (downsample was slower as wave64).]
+    # Every flag combination DLSS selects (motion vectors hi/lo, HDR/LDR, depth inverted/regular, ...) gets
+    # the same treatment, so games other than the one measured run the native versions too.
+    specs=()
+    if [[ "$WHAT" != l ]]; then
+        specs+=(rrlite_dec0_4x4_folded:dec0_head)
+        for mv in mvhi mvlo; do
+            for range in hdr ldr; do
+                specs+=("rrlite_enc0_4x4_${mv}_${range}_folded:enc0_tail")
+                for v in 3_1 3_2; do specs+=("rrlite_post_${v}_${mv}_${range}_folded:sust_only:w64"); done
+                for depth in depthinv depthreg; do
+                    for kind in "" _max; do
+                        [[ -z "$kind" ]] && specs+=("hiluma_engine_output_${depth}_${mv}_${range}_v1_rel:sust_only:w64")
+                        specs+=("hiluma_engine_output_${depth}_${mv}_${range}${kind}_v2_rel:sust_only:w64")
+                    done
+                done
+            done
+        done
+    fi
+    # L's unfolded variants keep their complete neural arithmetic. The folded M
+    # GEMM heads/tails are incompatible; substitute only the surface stores.
+    specs+=(rrlite_dec0_4x4:sust_only)
+    for mv in mvhi mvlo; do
+        for range in hdr ldr; do
+            specs+=("rrlite_enc0_4x4_${mv}_${range}:sust_only")
+            for v in 3_1 3_2; do specs+=("rrlite_post_${v}_${mv}_${range}:sust_only:w64"); done
+        done
+    done
+    for mode in static dynamic; do
+        for range in hdr ldr; do specs+=("rrlite_downsample_kernel_${mode}_${range}:sust_only"); done
+    done
+    for spec in "${specs[@]}"; do
+        IFS=: read -r kernel src mode <<< "$spec"
+        D4R_PREFER_ACCURACY="$ACCURACY" \
+            D4R_ZLUDA_WAVE64="$([[ "$mode" == w64 && "$ACCURACY" == 0 ]] && echo 1 || echo 0)" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" \
+            D4R_TEX_FP8="$FP8" D4R_DLSS_PTX_DIR="$PTX_DIR" "$HERE/tex/build_tex.sh" "$kernel" "$src" "$OUT"
     done
 fi
+[[ "$ACCURACY" == 1 ]] && printf '1\n' > "$OUT/d4r-accuracy.txt"
 echo "native kernels in $OUT"

kernels/common/hip_device_minimal.h

@@ -0,0 +1,43 @@
+#pragma once
+
+// Device-only Windows HIP build shim. The diagnostic/kernel code objects are
+// compiled with -nogpuinc/-nogpulib; no host HIP or C++ standard library is
+// needed. Definitions below mirror public HIP qualifiers and LLVM AMDGPU
+// work-item/grid builtins. The normal Linux HIP include path is untouched.
+#define __device__ __attribute__((device))
+#define __host__ __attribute__((host))
+#define __global__ __attribute__((global))
+#define __shared__ __attribute__((shared))
+#define __constant__ __attribute__((constant))
+#define __forceinline__ __attribute__((always_inline)) inline
+#define __launch_bounds__(n) __attribute__((amdgpu_flat_work_group_size(1, n)))
+
+struct d4r_device_dim3 { unsigned x, y, z; };
+// Public HIP vector storage ABI used by M's packed global/LDS accesses.
+struct alignas(8) uint2 { unsigned x, y; };
+struct alignas(16) uint4 { unsigned x, y, z, w; };
+static_assert(sizeof(uint2) == 8 && alignof(uint2) == 8, "HIP uint2 ABI");
+static_assert(sizeof(uint4) == 16 && alignof(uint4) == 16, "HIP uint4 ABI");
+__device__ __forceinline__ d4r_device_dim3 d4r_thread_idx() {
+    return {__builtin_amdgcn_workitem_id_x(), __builtin_amdgcn_workitem_id_y(),
+        __builtin_amdgcn_workitem_id_z()};
+}
+__device__ __forceinline__ d4r_device_dim3 d4r_block_idx() {
+    return {__builtin_amdgcn_workgroup_id_x(), __builtin_amdgcn_workgroup_id_y(),
+        __builtin_amdgcn_workgroup_id_z()};
+}
+__device__ __forceinline__ d4r_device_dim3 d4r_grid_dim() {
+    return {__builtin_amdgcn_grid_size_x() / __builtin_amdgcn_workgroup_size_x(),
+        __builtin_amdgcn_grid_size_y() / __builtin_amdgcn_workgroup_size_y(),
+        __builtin_amdgcn_grid_size_z() / __builtin_amdgcn_workgroup_size_z()};
+}
+#define threadIdx d4r_thread_idx()
+#define blockIdx d4r_block_idx()
+#define gridDim d4r_grid_dim()
+__device__ __attribute__((convergent, always_inline)) inline void __syncthreads() {
+    // Match HIP __work_group_barrier: execution rendezvous alone does not
+    // publish LDS stores or prevent loads moving across the barrier.
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "workgroup");
+    __builtin_amdgcn_s_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");
+}

kernels/common/wave32_exchange.h

@@ -0,0 +1,31 @@
+#pragma once
+#include <stdint.h>
+
+// Windows gfx11 portability workaround informed by realdody/d4r 1897b4a.
+// The reported permlanex16 failure duplicates the even K value into the odd
+// slot. Retain the native gfx11 layout and exchange through per-wave LDS.
+// Other platforms and gfx12 keep their original instruction sequence.
+__attribute__((device, always_inline)) static inline uint32_t d4r_wave32_other_half(uint32_t value)
+{
+#if defined(D4R_WINDOWS_GFX11_LANE_EXCHANGE) && defined(__GFX11__)
+    __attribute__((shared)) volatile uint32_t scratch[512];
+    const uint32_t tid = __builtin_amdgcn_workitem_id_x() + __builtin_amdgcn_workgroup_size_x() *
+        (__builtin_amdgcn_workitem_id_y() + __builtin_amdgcn_workgroup_size_y() * __builtin_amdgcn_workitem_id_z());
+    // All native K/M launch bounds are <=512. Do not silently corrupt LDS if
+    // this helper is later reused by a larger workgroup.
+    if (tid >= 512) __builtin_trap();
+    scratch[tid] = value;
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+    __builtin_amdgcn_wave_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+    const uint32_t result = scratch[tid ^ 16u];
+    // Protect scratch reuse by the next exchange in this wave. No block-wide
+    // rendezvous: different waves can reach GEMMs in a different order.
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+    __builtin_amdgcn_wave_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+    return result;
+#else
+    return __builtin_amdgcn_permlanex16(value, value, 0x76543210u, 0xfedcba98u, false, false);
+#endif
+}

kernels/common/wmma_backend.h

@@ -0,0 +1,57 @@
+#pragma once
+#include <stdint.h>
+#include "wave32_exchange.h"
+
+// d4r's gfx11 interface: each lane supplies 16 FP16 K operands. The 8 FP32
+// outputs have logical row 2*i + lane/16 and logical column lane%16.
+// gfx12 consumes only 8 FP16 operands per lane and emits 8 consecutive rows
+// per lane group. Keep the old interface for K/M/texture call sites until their
+// surrounding fragments are ported and validated individually.
+using d4r_wmma_h16 = _Float16 __attribute__((ext_vector_type(16)));
+using d4r_wmma_h8 = _Float16 __attribute__((ext_vector_type(8)));
+using d4r_wmma_f8 = float __attribute__((ext_vector_type(8)));
+
+__attribute__((device, always_inline)) static inline float d4r_wmma_other_half(float value)
+{
+    const uint32_t bits = __builtin_bit_cast(uint32_t, value);
+    const uint32_t swapped = d4r_wave32_other_half(bits);
+    return __builtin_bit_cast(float, swapped);
+}
+
+__attribute__((device, always_inline)) static inline d4r_wmma_f8
+d4r_wmma_legacy_layout(d4r_wmma_h16 a, d4r_wmma_h16 b, d4r_wmma_f8 c)
+{
+#if defined(__gfx1200__) || defined(__gfx1201__)
+    const uint32_t group = __builtin_amdgcn_mbcnt_lo(~0u, 0u) >> 4;
+    d4r_wmma_h8 a12, b12;
+    d4r_wmma_f8 c12;
+    d4r_wmma_f8 remote_c;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        remote_c[i] = d4r_wmma_other_half(c[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        a12[i] = a[8u * group + i];
+        b12[i] = b[8u * group + i];
+        const uint32_t row = 8u * group + i;
+        const uint32_t old_index = row >> 1;
+        c12[i] = (row & 1u) == group ? c[old_index] : remote_c[old_index];
+    }
+    const d4r_wmma_f8 d12 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a12, b12, c12);
+    d4r_wmma_f8 result, remote_d;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        remote_d[i] = d4r_wmma_other_half(d12[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const uint32_t row = 2u * i + group;
+        const uint32_t new_index = row & 7u;
+        result[i] = (row >> 3) == group ? d12[new_index] : remote_d[new_index];
+    }
+    return result;
+#elif defined(__GFX11__)
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#else
+#error "d4r WMMA backend requires a validated gfx11 or gfx12 target"
+#endif
+}

kernels/common/wmma_layout.h

@@ -0,0 +1,346 @@
+// WMMA register layouts of RDNA3 (gfx11) and RDNA4 (gfx12) for the native kernels (wave32, 16x16x16 f16 -> f32).
+// Every layout fact the kernels depend on is here; the kernel sources are written against these helpers.
+//
+// Notation: lane l, half h = l >> 4, r = l & 15. A operand lane l = row r of A, B operand lane l = column r
+// of B, D lane l = column r of D.
+//   gfx11: an operand holds all 16 K values of its row/column (8 dwords, the same data in both halves);
+//          D VGPR i is row 2i + h.
+//   gfx12: an operand holds 8 of the 16 K values (4 dwords), half h the K values 8h .. 8h+7 in order;
+//          D VGPR i is row i + 8h, which is the gfx12 B-operand layout of the next GEMM (no exchange).
+// A and B only have to agree on which K each slot holds; "half h = K 8h .. 8h+7" makes a gfx12 operand the
+// lower or upper 16 bytes of the gfx11 one, so row images and weight images serve both layouts.
+//
+// D4R_WMMA_LAYOUT selects the layout (default: the target's). D4R_WMMA_LAYOUT=12 on a gfx11 target is the
+// layout shim: the gfx12 register layout with every WMMA done by the gfx11 instruction on operands and
+// accumulators converted around it, in the gfx11 K order. Its results are bit-identical to the gfx11 build,
+// so the gfx12 indexing can be validated on RDNA3 hardware.
+//
+// Only clang builtins are used (the texture tails compile with -nogpuinc -nogpulib).
+#pragma once
+#include <stdint.h>
+#include "wave32_exchange.h"
+
+#ifndef D4R_WMMA_LAYOUT
+#if defined(__GFX12__)
+#define D4R_WMMA_LAYOUT 12
+#else
+#define D4R_WMMA_LAYOUT 11
+#endif
+#endif
+#if D4R_WMMA_LAYOUT == 12 && !defined(__GFX12__)
+#define D4R_WMMA_SHIM 1
+#endif
+#if D4R_WMMA_LAYOUT == 11 && defined(__GFX12__)
+#error "the gfx11 WMMA layout does not exist on gfx12"
+#endif
+
+#define WM_FN __attribute__((device, always_inline)) static inline
+
+typedef _Float16 wm_f16;
+typedef _Float16 wm_hv2 __attribute__((ext_vector_type(2)));
+typedef _Float16 wm_h8 __attribute__((ext_vector_type(8)));
+typedef _Float16 wm_h16 __attribute__((ext_vector_type(16)));
+typedef float wm_f8v __attribute__((ext_vector_type(8)));
+typedef uint32_t wm_u8v __attribute__((ext_vector_type(8)));
+typedef uint32_t wm_u4v __attribute__((ext_vector_type(4)));
+typedef int wm_i2v __attribute__((ext_vector_type(2)));
+
+// ---------------------------------------------------------------- lanes
+WM_FN uint32_t wm_lane()
+{
+    return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
+}
+WM_FN uint32_t wm_half()
+{
+    return wm_lane() >> 4;
+}
+// value of the lane with the same index in the other half of the wave
+WM_FN uint32_t wm_other_half(uint32_t v)
+{
+    return d4r_wave32_other_half(v);
+}
+WM_FN float wm_other_half_f(float v)
+{
+    return __builtin_bit_cast(float, wm_other_half(__builtin_bit_cast(uint32_t, v)));
+}
+WM_FN uint32_t wm_pack2(wm_f16 lo, wm_f16 hi)
+{
+    return __builtin_bit_cast(uint32_t, (wm_hv2){lo, hi});
+}
+WM_FN wm_f16 wm_lo16(uint32_t v)
+{
+    return __builtin_bit_cast(wm_hv2, v)[0];
+}
+WM_FN wm_f16 wm_hi16(uint32_t v)
+{
+    return __builtin_bit_cast(wm_hv2, v)[1];
+}
+
+// ---------------------------------------------------------------- accumulator rows
+// row (of the 16-row tile) that accumulator VGPR i holds in a lane of half h (prep code: logical lanes)
+WM_FN int wm_acc_row_h(int i, int h)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return i + 8 * h;
+#else
+    return 2 * i + h;
+#endif
+}
+// ... in this lane
+WM_FN int wm_acc_row(int i)
+{
+    return wm_acc_row_h(i, (int)wm_half());
+}
+// rows are the K dimension of the next GEMM: VGPRs 2j, 2j + 1 are adjacent K values in gfx12 (and 16 bytes
+// of this lane's 8 rows are contiguous), rows 2i + h are interleaved between the halves in gfx11
+#if D4R_WMMA_LAYOUT == 12
+#define WM_ACC_CONTIGUOUS 1
+#else
+#define WM_ACC_CONTIGUOUS 0
+#endif
+
+// ---------------------------------------------------------------- f16 operands
+#if D4R_WMMA_LAYOUT == 12
+typedef wm_u4v wm_op; // this half's 8 K values
+#define WM_OP_DWORDS 4
+#else
+typedef wm_u8v wm_op; // all 16 K values
+#define WM_OP_DWORDS 8
+#endif
+
+// operand from 16 contiguous halves (K 0..15) in memory (16-byte aligned); gfx12 reads only this half's 8
+WM_FN wm_op wm_op_load(const wm_f16* p)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return *(const wm_u4v*)(p + 8 * wm_half());
+#else
+    const wm_u4v a = *(const wm_u4v*)p, b = *(const wm_u4v*)(p + 8);
+    return (wm_u8v){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+#endif
+}
+
+// operand slot `idx` of an image of 32-byte slots (16 halves each, K in order): gfx12 reads its 16-byte half
+WM_FN wm_op wm_op_image(const wm_u8v* __restrict__ img, int idx)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return ((const wm_u4v*)img)[2 * idx + (int)wm_half()];
+#else
+    return img[idx];
+#endif
+}
+
+// element (K value) of an operand that holds accumulator row wm_acc_row(i) of the same lane
+WM_FN wm_f16 wm_op_acc_elem(const wm_op& v, int i)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return (i & 1) ? wm_hi16(v[i >> 1]) : wm_lo16(v[i >> 1]);
+#else
+    const int c = 2 * i + (int)wm_half();
+    return (c & 1) ? wm_hi16(v[c >> 1]) : wm_lo16(v[c >> 1]);
+#endif
+}
+
+// the f16 accumulator values own[i] (row wm_acc_row(i) of this lane's column) as an operand of the next GEMM
+// (the rows become K; gfx11 fetches the other parity from lane l ^ 16)
+WM_FN wm_op wm_op_from_acc(const wm_f16 own[8])
+{
+#if D4R_WMMA_LAYOUT == 12
+    return (wm_u4v){wm_pack2(own[0], own[1]), wm_pack2(own[2], own[3]), wm_pack2(own[4], own[5]), wm_pack2(own[6], own[7])};
+#else
+    const uint32_t hf = wm_half();
+    uint32_t mine[4], theirs[4];
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+    {
+        mine[j] = wm_pack2(own[2 * j], own[2 * j + 1]);
+        theirs[j] = wm_other_half(mine[j]);
+    }
+#ifndef PWIN_NO_PERM
+    // v_perm_b32(theirs, mine): bytes 0-3 = mine, 4-7 = theirs. Even j: the low halves, odd j: the high
+    // halves, in the order (mine, theirs) for hf = 0 and (theirs, mine) for hf = 1.
+    const uint32_t sel_lo = hf ? 0x01000504u : 0x05040100u, sel_hi = hf ? 0x03020706u : 0x07060302u;
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 8; ++j)
+        r[j] = __builtin_amdgcn_perm(theirs[j >> 1], mine[j >> 1], (j & 1) ? sel_hi : sel_lo);
+    return r;
+#else
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 8; ++j)
+    {
+        const wm_f16 a = (j & 1) ? wm_hi16(mine[j >> 1]) : wm_lo16(mine[j >> 1]);
+        const wm_f16 b = (j & 1) ? wm_hi16(theirs[j >> 1]) : wm_lo16(theirs[j >> 1]);
+        r[j] = hf ? wm_pack2(b, a) : wm_pack2(a, b);
+    }
+    return r;
+#endif
+#endif
+}
+
+WM_FN wm_op wm_op_from_f8(wm_f8v d)
+{
+    wm_f16 h[8];
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        h[i] = (wm_f16)d[i];
+    return wm_op_from_acc(h);
+}
+
+// Stores an operand back as a row of 16 halves (K 0..15). In gfx11 both halves hold the whole row and the
+// lanes of half 0 store it; in gfx12 each half stores its 8 values. `count` (8 or 16) limits the row to its
+// first `count` halves; `ok` is the lane's own condition.
+WM_FN void wm_op_store(wm_f16* p, const wm_op& v, bool ok = true, int count = 16)
+{
+#if D4R_WMMA_LAYOUT == 12
+    if (ok && (wm_half() == 0 || count > 8))
+        *(wm_u4v*)(p + 8 * wm_half()) = v;
+#else
+    if (ok && wm_half() == 0)
+    {
+        *(wm_u4v*)p = (wm_u4v){v[0], v[1], v[2], v[3]};
+        if (count > 8)
+            *(wm_u4v*)(p + 8) = (wm_u4v){v[4], v[5], v[6], v[7]};
+    }
+#endif
+}
+
+// 8 halves of an f16 vector in accumulator row order: out[i] = v[wm_acc_row(i)], v at p (16 halves, 16-byte
+// aligned)
+WM_FN void wm_acc_vec_load(const wm_f16* p, wm_f16 out[8])
+{
+#if D4R_WMMA_LAYOUT == 12
+    const wm_u4v a = *(const wm_u4v*)(p + 8 * wm_half());
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        out[i] = (i & 1) ? wm_hi16(a[i >> 1]) : wm_lo16(a[i >> 1]);
+#else
+    const uint32_t hf = wm_half();
+    const wm_u4v a = *(const wm_u4v*)p, b = *(const wm_u4v*)(p + 8);
+    const wm_u8v v = (wm_u8v){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        out[i] = hf ? wm_hi16(v[i]) : wm_lo16(v[i]);
+#endif
+}
+
+// ---------------------------------------------------------------- the WMMA itself
+#if D4R_WMMA_LAYOUT == 12 && defined(__GFX12__)
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(__builtin_bit_cast(wm_h8, a), __builtin_bit_cast(wm_h8, b), c);
+}
+#ifdef D4R_K_F16_WMMA
+// Rejected K experiment: direct packed F16 changes the arithmetic on gfx1201.
+// The public WMMA reproducer and enc1 replay are in tools/windows and docs.
+// Experimental gfx12 packed FP16 C/D. K's strict baseline enters each K16
+// step with an already FP16-rounded accumulator. Keep the existing logical
+// eight-row contract; only this instruction/register representation changes.
+WM_FN wm_f8v wm_mma_f16_step(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    const wm_h8 hc = __builtin_convertvector(c, wm_h8);
+    const wm_h8 hd = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(
+        __builtin_bit_cast(wm_h8, a), __builtin_bit_cast(wm_h8, b), hc);
+    return __builtin_convertvector(hd, wm_f8v);
+}
+#endif
+#else
+WM_FN wm_f8v wm_mma11(const wm_u8v& a, const wm_u8v& b, wm_f8v c)
+{
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(wm_h16, a), __builtin_bit_cast(wm_h16, b), c);
+}
+#if D4R_WMMA_LAYOUT == 12
+// layout shim: gfx12 operand halves -> one gfx11 operand in K order (half 0's 8 K, then half 1's)
+WM_FN wm_u8v wm_shim_operand(const wm_u4v& v)
+{
+    const bool hf = wm_half() != 0;
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+    {
+        const uint32_t o = wm_other_half(v[j]);
+        r[j] = hf ? o : v[j];
+        r[4 + j] = hf ? v[j] : o;
+    }
+    return r;
+}
+// gfx12 accumulator (VGPR i = row i + 8h) <-> gfx11 accumulator (VGPR i = row 2i + h)
+WM_FN wm_f8v wm_shim_acc_to11(const wm_f8v& c)
+{
+    const bool hf = wm_half() != 0;
+    wm_f8v o, r;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        o[i] = wm_other_half_f(c[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+    {
+        // row 2i + h lives in half (2i + h) >> 3, VGPR (2i + h) & 7
+        const float h0 = i < 4 ? c[2 * i] : o[2 * i - 8];
+        const float h1 = i < 4 ? o[2 * i + 1] : c[2 * i - 7];
+        r[i] = hf ? h1 : h0;
+    }
+    return r;
+}
+WM_FN wm_f8v wm_shim_acc_to12(const wm_f8v& d)
+{
+    const bool hf = wm_half() != 0;
+    wm_f8v o, r;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        o[i] = wm_other_half_f(d[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+    {
+        // row i + 8h lives in half i & 1, VGPR (i >> 1) + 4h
+        const float h0 = (i & 1) ? o[i >> 1] : d[i >> 1];
+        const float h1 = (i & 1) ? d[(i >> 1) + 4] : o[(i >> 1) + 4];
+        r[i] = hf ? h1 : h0;
+    }
+    return r;
+}
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_shim_acc_to12(wm_mma11(wm_shim_operand(a), wm_shim_operand(b), wm_shim_acc_to11(c)));
+}
+// ZLUDA's K order: its gfx11 helpers put K pair p (K 2p, 2p + 1) in operand dword p, its gfx12 helpers put
+// in half h, dword j the pair (j & 1) * 4 + 2h + (j >> 1) (see zluda_ptx_impl.cpp)
+WM_FN wm_u8v wm_shim_operand_zluda(const wm_u4v& v)
+{
+    const bool hf = wm_half() != 0;
+    wm_u4v o;
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+        o[j] = wm_other_half(v[j]);
+    wm_u8v r;
+#pragma unroll
+    for (int p = 0; p < 8; ++p)
+    {
+        const int hp = (p & 3) >> 1, jp = 2 * (p & 1) + (p >> 2);
+        r[p] = (hp != 0) == hf ? v[jp] : o[jp];
+    }
+    return r;
+}
+WM_FN wm_f8v wm_mma_zluda(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_shim_acc_to12(wm_mma11(wm_shim_operand_zluda(a), wm_shim_operand_zluda(b), wm_shim_acc_to11(c)));
+}
+#else
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_mma11(a, b, c);
+}
+#endif
+#endif
+#if D4R_WMMA_LAYOUT == 12 && defined(__GFX12__)
+// operands whose K slots follow ZLUDA's gfx12 order: the same instruction (the order only matters to the shim)
+WM_FN wm_f8v wm_mma_zluda(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_mma(a, b, c);
+}
+#endif
+
+WM_FN wm_f8v wm_splat8(float v)
+{
+    return (wm_f8v){v, v, v, v, v, v, v, v};
+}

kernels/k/dltss_pwin_dec0_layer.hip

@@ -1,3 +1,10 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
+#ifndef D4R_K_POS_MT
+#define D4R_K_POS_MT 2
+#elif !defined(__GFX12__) || !defined(D4R_K_FP16_BASELINE)
+#error "experimental position token tiling requires gfx12 strict K baseline"
+#endif
 #include "pwin_pos.h"
-POS_DECODER(dltss_pwin_dec0_layer, 2, 32, 64, 40, 48, 2)
+POS_DECODER(dltss_pwin_dec0_layer, 2, 32, 64, 40, 48, D4R_K_POS_MT)

kernels/k/dltss_pwin_dec1_layer.hip

@@ -1,3 +1,5 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
 #include "pwin_layer.h"
 PWIN_DECODER(dltss_pwin_dec1_layer, 2, 64, 64)

kernels/k/dltss_pwin_dec2_layer.hip

@@ -1,3 +1,5 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
 #include "pwin_layer.h"
 PWIN_DECODER(dltss_pwin_dec2_layer, 2, 64, 96)

kernels/k/dltss_pwin_enc0_layer.hip

@@ -1,3 +1,19 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
+#ifndef D4R_K_POS_MT
+#define D4R_K_POS_MT 2
+#elif !defined(__GFX12__) || !defined(D4R_K_FP16_BASELINE)
+#error "experimental position token tiling requires gfx12 strict K baseline"
+#endif
+// Reuse V across the window's waves. Paired gfx1201 replays retain every half
+// code and reduce enc0 time by about 5%; gfx11 and arithmetic experiments keep
+// their previous path. D4R_K_PRIVATE_V supplies the original A/B control.
+#if defined(__GFX12__) && defined(D4R_K_FP16_BASELINE) && !defined(D4R_K_PRIVATE_V) && \
+    !defined(D4R_K_PACKED_ACC) && !defined(D4R_K_F16_WMMA)
+#ifndef POS_VSHARE
+#define POS_VSHARE
+#endif
+#endif
 #include "pwin_pos.h"
-POS_ENCODER(dltss_pwin_enc0_layer, 2, 32, 64, 16, 2)
+POS_ENCODER(dltss_pwin_enc0_layer, 2, 32, 64, 16, D4R_K_POS_MT)

kernels/k/dltss_pwin_enc1_layer.hip

@@ -1,3 +1,16 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
+#ifdef D4R_K_ENC1_POS_MT
+#if !defined(__GFX12__) || !defined(D4R_K_FP16_BASELINE)
+#error "experimental enc1 token tiling requires gfx12 strict K baseline"
+#endif
+#if D4R_K_ENC1_POS_MT != 1 && D4R_K_ENC1_POS_MT != 2 && D4R_K_ENC1_POS_MT != 4
+#error "enc1 token tiles per wave must be 1, 2 or 4"
+#endif
+#include "pwin_pos.h"
+POS_ENCODER(dltss_pwin_enc1_layer, 2, 64, 64, 0, D4R_K_ENC1_POS_MT)
+#else
 #include "pwin_layer.h"
 PWIN_ENCODER_POS(dltss_pwin_enc1_layer, 2, 64, 64, true)
+#endif

kernels/k/dltss_pwin_enc2_layer.hip

@@ -1,3 +1,5 @@
+// CU mode: the waves of a window share one L0 cache (1-6% faster here; slower in the wide layers)
+// d4r-build-flags: -mcumode
 #define PWIN_F32ACC
 #include "pwin_layer.h"
 PWIN_ENCODER(dltss_pwin_enc2_layer, 2, 64, 96)

kernels/k/pwin_common.h

@@ -1,20 +1,38 @@
-// Native RDNA3 (gfx11 wave32 WMMA) kernels for the DLSS 4 (preset K) DltssPaddedWinLayer blocks.
+// Native kernels (gfx11 / gfx12 wave32 WMMA) for the DLSS 4 (preset K) DltssPaddedWinLayer blocks.
 // Semantics: kernels/tools/pwin_model.py (verified against the PTX interpreter, kernels/tools/ptxsim.py).
 //
 // Layout conventions (transposed WMMA, as in the preset M kernels): a GEMM Y = X W is computed as
-// Y^T = W^T X^T with the weights as the WMMA A operand (lane l: output channel l & 15, 16 K values)
-// and the activations as the B operand (lane l: token l & 15, 16 K values). The f32 result then has
-// lane l = token l & 15 and VGPR i = output channel 2i + (l >> 4) of the 16-channel tile ("D^T").
+// Y^T = W^T X^T with the weights as the WMMA A operand (lane l: output channel l & 15) and the activations
+// as the B operand (lane l: token l & 15). The f32 result then has lane l = token l & 15 and VGPR i =
+// output channel wm_acc_row(i) of the 16-channel tile ("D^T"); ../common/wmma_layout.h has the per-target
+// register layouts (gfx11: channel 2i + (l >> 4); gfx12: channel i + 8 (l >> 4)).
 #pragma once
+#ifdef D4R_DEVICE_ONLY_MINIMAL
+#include "../common/hip_device_minimal.h"
+#else
 #include <hip/hip_runtime.h>
+#endif
 #include <stdint.h>
-
+#include "../common/wmma_layout.h"
+#if defined(D4R_ACCURACY) && defined(PWIN_F32ACC)
+#undef PWIN_F32ACC
+#endif
 #pragma clang fp contract(off)
 
 typedef _Float16 half_t;
 typedef _Float16 hv2 __attribute__((ext_vector_type(2)));
 typedef _Float16 h16 __attribute__((ext_vector_type(16)));
 typedef float f8v __attribute__((ext_vector_type(8)));
+#ifdef D4R_K_PACKED_ACC
+#if !defined(__GFX12__) || !defined(D4R_K_FP16_BASELINE) || defined(D4R_K_F16_WMMA)
+#error "packed accumulator storage requires gfx12 strict F32 WMMA baseline"
+#endif
+// Keep already-rounded values packed between steps. Every matrix instruction
+// still uses F32 C/D and rounds once to half afterwards, like the baseline.
+typedef _Float16 acc8v __attribute__((ext_vector_type(8)));
+#else
+typedef f8v acc8v;
+#endif
 typedef uint32_t u8v __attribute__((ext_vector_type(8)));
 typedef uint32_t u4v __attribute__((ext_vector_type(4)));
 
@@ -23,32 +41,88 @@ __device__ __forceinline__ uint32_t lane_id()
     return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
 }
 
-__device__ __forceinline__ f8v wmma(u8v a, u8v b, f8v c)
+typedef wm_op op_t; // WMMA operand: u8v (gfx11) or u4v (gfx12)
+
+// The CUDA fragment reduction rounds products and every tree node to f16.
+// A float sum of the squared halves changes rsqrt and can amplify differences
+// through attention. The strict baseline preserves its channel grouping.
+template <int C>
+__device__ __forceinline__ half_t l2_sum(const half_t* row)
 {
-    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(h16, a), __builtin_bit_cast(h16, b), c);
+#ifdef D4R_K_FP16_BASELINE
+    static_assert(C % 32 == 0 && C <= 160, "audited K channel counts");
+    hv2 partial[4];
+#pragma unroll
+    for (int t = 0; t < 4; ++t)
+    {
+        hv2 halves[2];
+#pragma unroll
+        for (int parity = 0; parity < 2; ++parity)
+        {
+            hv2 pairs[C / 32];
+#pragma unroll
+            for (int group = 0; group < C / 32; ++group)
+            {
+                const hv2 a = *(const hv2*)(row + 2 * t + 8 * parity + 32 * group);
+                const hv2 b = *(const hv2*)(row + 2 * t + 8 * parity + 32 * group + 16);
+                pairs[group] = (a * a) + (b * b);
+            }
+            hv2 value = pairs[0];
+            if constexpr (C >= 64) value = value + pairs[1];
+            if constexpr (C == 96) value = value + pairs[2];
+            if constexpr (C >= 128) value = value + (pairs[2] + pairs[3]);
+            if constexpr (C == 160) value = value + pairs[4];
+            halves[parity] = value;
+        }
+        partial[t] = halves[0] + halves[1];
+    }
+    const hv2 sum = (partial[0] + partial[1]) + (partial[2] + partial[3]);
+    return sum[0] + sum[1];
+#else
+    float sum = 0;
+#pragma unroll
+    for (int c = 0; c < C; ++c) sum += (float)(half_t)(row[c] * row[c]);
+    return (half_t)sum;
+#endif
 }
 
 // one k16 step with the f16 accumulator of NVIDIA's f16 wmma (rounded after the step)
 // PWIN_F32ACC: keep the accumulator in f32 through the chain (rounded to f16 where the values are used)
-__device__ __forceinline__ f8v mma16(u8v a, u8v b, f8v c)
+__device__ __forceinline__ acc8v mma16(const op_t& a, const op_t& b, acc8v c)
 {
-    f8v d = wmma(a, b, c);
-#ifndef PWIN_F32ACC
+#ifdef D4R_K_PACKED_ACC
+    const f8v fc = __builtin_convertvector(c, f8v);
+    const f8v d = wm_mma(a, b, fc);
+    return __builtin_convertvector(d, acc8v);
+#elif defined(D4R_K_F16_WMMA)
+#if !defined(__GFX12__) || D4R_WMMA_LAYOUT != 12 || !defined(D4R_K_FP16_BASELINE)
+#error "experimental packed F16 WMMA requires gfx12 native layout and strict K baseline"
+#endif
+    return wm_mma_f16_step(a, b, c);
+#else
+    f8v d = wm_mma(a, b, c);
+#if !defined(PWIN_F32ACC) || defined(D4R_K_FP16_BASELINE)
 #pragma unroll
     for (int i = 0; i < 8; ++i)
         d[i] = (float)(half_t)d[i];
 #endif
     return d;
+#endif
 }
 
-__device__ __forceinline__ f8v splat8(float v)
+__device__ __forceinline__ acc8v splat8(float v)
 {
-    return (f8v){v, v, v, v, v, v, v, v};
+#ifdef D4R_K_PACKED_ACC
+    const half_t h = (half_t)v;
+    return (acc8v){h, h, h, h, h, h, h, h};
+#else
+    return (acc8v){v, v, v, v, v, v, v, v};
+#endif
 }
 
 __device__ __forceinline__ uint32_t other_half(uint32_t v)
 {
-    return __builtin_amdgcn_permlanex16(v, v, 0x76543210u, 0xfedcba98u, false, false);
+    return wm_other_half(v);
 }
 
 __device__ __forceinline__ uint32_t pack2(half_t lo, half_t hi)
@@ -65,50 +139,24 @@ __device__ __forceinline__ half_t hi16(uint32_t v)
     return __builtin_bit_cast(hv2, v)[1];
 }
 
-// D^T f16 values (own[i] = channel 2i + hf of token l & 15) -> B operand of the same token (all 16
-// channels in order): the other parity comes from lane l ^ 16
-__device__ __forceinline__ u8v operand_from_dt(const half_t own[8])
+// D^T f16 values (own[i] = channel wm_acc_row(i) of token l & 15) -> operand of the same token (the
+// channels become K)
+__device__ __forceinline__ op_t operand_from_dt(const half_t own[8])
 {
-    const uint32_t hf = lane_id() >> 4;
-    uint32_t mine[4], theirs[4];
-#pragma unroll
-    for (int j = 0; j < 4; ++j)
-    {
-        mine[j] = pack2(own[2 * j], own[2 * j + 1]);
-        theirs[j] = other_half(mine[j]);
-    }
-#ifndef PWIN_NO_PERM
-    // v_perm_b32(theirs, mine): bytes 0-3 = mine, 4-7 = theirs. Even j: the low halves, odd j: the high
-    // halves, in the order (mine, theirs) for hf = 0 and (theirs, mine) for hf = 1.
-    const uint32_t sel_lo = hf ? 0x01000504u : 0x05040100u, sel_hi = hf ? 0x03020706u : 0x07060302u;
-    u8v r;
-#pragma unroll
-    for (int j = 0; j < 8; ++j)
-        r[j] = __builtin_amdgcn_perm(theirs[j >> 1], mine[j >> 1], (j & 1) ? sel_hi : sel_lo);
-    return r;
+    return wm_op_from_acc(own);
+}
+
+__device__ __forceinline__ op_t operand_from_f8(acc8v d)
+{
+#ifdef D4R_K_PACKED_ACC
+    const half_t h[8] = {d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]};
+    return wm_op_from_acc(h);
 #else
-    u8v r;
-#pragma unroll
-    for (int j = 0; j < 8; ++j)
-    {
-        const half_t a = (j & 1) ? hi16(mine[j >> 1]) : lo16(mine[j >> 1]);
-        const half_t b = (j & 1) ? hi16(theirs[j >> 1]) : lo16(theirs[j >> 1]);
-        r[j] = hf ? pack2(b, a) : pack2(a, b);
-    }
-    return r;
+    return wm_op_from_f8(d);
 #endif
 }
 
-__device__ __forceinline__ u8v operand_from_f8(f8v d)
-{
-    half_t h[8];
-#pragma unroll
-    for (int i = 0; i < 8; ++i)
-        h[i] = (half_t)d[i];
-    return operand_from_dt(h);
-}
-
-// element c (0..15) of a B operand (16 halves)
+// element c (0..15) of a row of 16 halves (VALU code; not a WMMA operand)
 __device__ __forceinline__ half_t op_get(const u8v& v, int c)
 {
     return (c & 1) ? hi16(v[c >> 1]) : lo16(v[c >> 1]);
@@ -145,6 +193,58 @@ __device__ __forceinline__ void gstore_row16(half_t* dst, const u8v& v)
     gstore16(dst + 8, (u4v){v[4], v[5], v[6], v[7]});
 }
 
+// WMMA operands from rows of 16 channels: LDS, global (PWIN_NT_LOAD as above), prep images (32-byte slots)
+__device__ __forceinline__ op_t op_lds(const half_t* p)
+{
+    return wm_op_load(p);
+}
+__device__ __forceinline__ op_t op_gload(const half_t* p)
+{
+#if D4R_WMMA_LAYOUT == 12
+#ifdef PWIN_NT_LOAD
+    return __builtin_nontemporal_load((const u4v*)(p + 8 * wm_half()));
+#else
+    return *(const u4v*)(p + 8 * wm_half());
+#endif
+#else
+    return gload_row16(p);
+#endif
+}
+__device__ __forceinline__ op_t op_img(const u8v* __restrict__ img, int idx)
+{
+    return wm_op_image(img, idx);
+}
+// operand -> slot idx of an operand image in LDS (read back with op_img)
+__device__ __forceinline__ void op_img_store(u8v* img, int idx, const op_t& v)
+{
+#if D4R_WMMA_LAYOUT == 12
+    ((u4v*)img)[2 * idx + wm_half()] = v;
+#else
+    if (!wm_half())
+        img[idx] = v;
+#endif
+}
+// operand (16 channels of this lane's token) -> row of `count` (8 or 16) channels in global memory, if ok
+__device__ __forceinline__ void op_gstore(half_t* dst, const op_t& v, bool ok, int count = 16)
+{
+#if D4R_WMMA_LAYOUT == 12
+    if (ok && (wm_half() == 0 || count >= 16))
+        gstore16(dst + 8 * wm_half(), v);
+#else
+    if (!wm_half() && ok)
+    {
+        gstore16(dst, (u4v){v[0], v[1], v[2], v[3]});
+        if (count >= 16)
+            gstore16(dst + 8, (u4v){v[4], v[5], v[6], v[7]});
+    }
+#endif
+}
+// operand -> row of 16 channels in LDS
+__device__ __forceinline__ void op_store(half_t* dst, const op_t& v)
+{
+    wm_op_store(dst, v);
+}
+
 __device__ __forceinline__ void block_sync()
 {
     __syncthreads();
@@ -162,27 +262,3 @@ __device__ __forceinline__ int frag_offset(int k, int n)
 {
     return 64 * (n & 7) + 16 * ((k & 7) >> 1) + 8 * (n >> 3) + 4 * (k >> 3) + 2 * (k & 1);
 }
-
-// A-operand image of a K x N weight: tile (kt, nt) = 16 lanes x 8 dwords (lane = output channel)
-struct WDesc
-{
-    int src;       // byte offset of the fragments in the weight buffer
-    int kt_stride; // bytes between k tiles
-    int nt_stride; // bytes between n tiles
-    int KT, NT;    // tiles
-    int dst;       // u8v index of the image in the prep buffer
-};
-
-// weight operand tile `tile` of an A-operand image (16 lanes x 32 bytes): lanes l and l + 16 need the same
-// row, so each half-wave loads one 16-byte half and the halves are swapped across (halves L2 traffic)
-__device__ __forceinline__ u8v wtile(const u8v* __restrict__ img, int tile)
-{
-    const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
-    const u4v mine = ((const u4v*)&img[tile * 16 + m])[hf];
-    u4v other;
-#pragma unroll
-    for (int j = 0; j < 4; ++j)
-        other[j] = other_half(mine[j]);
-    const u4v lo = hf ? other : mine, hi = hf ? mine : other;
-    return (u8v){lo[0], lo[1], lo[2], lo[3], hi[0], hi[1], hi[2], hi[3]};
-}

kernels/k/pwin_layer.h

@@ -72,14 +72,10 @@ __device__ __forceinline__ half_t wload(const uint8_t* w, int byte)
     return *(const half_t*)(w + byte);
 }
 
-// channels 16 nt + 2i + hf (i = 0..7) of an f16 vector at byte offset `byte` (16 nt already added)
-__device__ __forceinline__ void dvec8(const uint8_t* w, int byte, uint32_t hf, half_t out[8])
+// channels 16 nt + wm_acc_row(i) (i = 0..7) of an f16 vector at byte offset `byte` (16 nt already added)
+__device__ __forceinline__ void dvec8(const uint8_t* w, int byte, half_t out[8])
 {
-    const u4v a = *(const u4v*)(w + byte), b = *(const u4v*)(w + byte + 16);
-    const u8v v = (u8v){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
-#pragma unroll
-    for (int i = 0; i < 8; ++i)
-        out[i] = hf ? hi16(v[i]) : lo16(v[i]);
+    wm_acc_vec_load((const half_t*)(w + byte), out);
 }
 
 __device__ __forceinline__ u8v img_tile(const uint8_t* w, int src, int n)
@@ -131,7 +127,7 @@ template <class L> __device__ void pwin_prep_at(const PwinParams& p, u8v* img, u
 #pragma unroll
     for (int i = 0; i < 8; ++i)
     {
-        const int key = 16 * kt + 2 * i + (lane >> 4), r = q & 15, c = key & 15;
+        const int key = 16 * kt + wm_acc_row_h(i, lane >> 4), r = q & 15, c = key & 15;
         const int off = 512 * (4 * kt + qt) + 64 * (r & 7) + 16 * ((c & 7) >> 1) + 2 * (c & 1) + 4 * (r >> 3) + 8 * (c >> 3);
         v[i] = wload(w, L::BIAS + 8192 * h + off);
     }
@@ -167,7 +163,7 @@ __device__ __forceinline__ half_t gelu(half_t x)
 }
 
 // gelu of 8 accumulator values as 4 packed f16 pairs (the same f16 operations as gelu(), element-wise)
-__device__ __forceinline__ void gelu8(const f8v& d, half_t g[8])
+__device__ __forceinline__ void gelu8(const acc8v& d, half_t g[8])
 {
     const hv2 k1 = (hv2)(half_t)__builtin_bit_cast(float, 0x3ED306EBu), k2 = (hv2)(half_t)__builtin_bit_cast(float, 0x3DA60DD6u);
 #pragma unroll
@@ -209,6 +205,45 @@ __device__ uint64_t g_stamp[16384][16];
 __device__ float g_dbg[12][64][160];
 #define DBG_ON (blockIdx.x == DBX && blockIdx.y == DBY)
 #endif
+// Softmax row sum over the 64 keys of this lane's query: this lane's 32 accumulator rows (summed into rs as
+// the scores are computed), plus the other half's. The layout shim (gfx12 layout on gfx11) instead redoes the
+// sum in gfx11's order (keys 16 kt + 2 i + h, then the other parity), which the f32 result depends on, so
+// that its output stays bit-identical to the gfx11 build.
+#ifdef D4R_WMMA_SHIM
+constexpr bool kShimRowSum = true;
+#else
+constexpr bool kShimRowSum = false;
+#endif
+__device__ __forceinline__ float row_sum_total(float rs, const half_t (&e)[4][8])
+{
+    if constexpr (!kShimRowSum)
+        return rs + __builtin_bit_cast(float, other_half(__builtin_bit_cast(uint32_t, rs)));
+    // this lane: keys 16 kt + 8 h + j; the other half: keys 16 kt + 8 (1 - h) + j
+    const bool hf = wm_half() != 0;
+    float s[2] = {0.0f, 0.0f};
+#pragma unroll
+    for (int kt = 0; kt < 4; ++kt)
+    {
+        half_t lo[8], hi[8];
+#pragma unroll
+        for (int j = 0; j < 8; j += 2)
+        {
+            const uint32_t own = pack2(e[kt][j], e[kt][j + 1]), other = other_half(own);
+            const uint32_t l = hf ? other : own, u = hf ? own : other;
+            lo[j] = lo16(l), lo[j + 1] = hi16(l), hi[j] = lo16(u), hi[j + 1] = hi16(u);
+        }
+#pragma unroll
+        for (int par = 0; par < 2; ++par)
+#pragma unroll
+            for (int i = 0; i < 8; ++i)
+            {
+                const int key = 2 * i + par;
+                s[par] += (float)(key < 8 ? lo[key] : hi[key - 8]);
+            }
+    }
+    return s[0] + s[1];
+}
+
 // The Swin core for one window: x0 rows of this wave's 16 tokens are in act (f16, C per token).
 // On return act holds the block output y. hb: normalised input rows; K/V of one head at a time in kl/vt.
 template <class L>
@@ -224,17 +259,7 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
 
     STAMP(1);
     // ---- L2 norm: h1 = x0 * (rsqrt(sum x0^2) * gamma1) -> hb
-    float ss = 0.0f;
-    for (int kt = 0; kt < KT; ++kt)
-    {
-        const u8v xv = lds_row16(arow + 16 * kt);
-#pragma unroll
-        for (int c = 0; c < 16; ++c)
-        {
-            const half_t v = op_get(xv, c);
-            ss += (float)(half_t)(v * v);
-        }
-    }
+    const half_t ss = l2_sum<C>(arow);
     const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss);
     for (int kt = (int)hf; kt < KT; kt += 2)
     {
@@ -258,14 +283,14 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
         block_sync(); // V of all 64 tokens is computed by every wave from hb
     STAMP(2);
     // ---- attention, one head at a time; the output projection accumulates head by head
-    f8v acc[KT];
+    acc8v acc[KT];
 #pragma unroll
     for (int nt = 0; nt < KT; ++nt)
 {
 
     half_t vv_[8];
 
-    dvec8(w, L::BO + 32 * nt, hf, vv_);
+    dvec8(w, L::BO + 32 * nt, vv_);
 
 #pragma unroll
 
@@ -277,7 +302,7 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
     for (int h = 0; h < H; ++h)
     {
         // position-only attention: V^T A operands straight from D = h1 Wv (lane = dim, 16 keys per tile)
-        u8v vtop[2][4];
+        op_t vtop[2][4];
         if constexpr (L::POS)
         {
             // each wave computes the V^T operands of its own key tile; shared through LDS, one slot per head
@@ -285,61 +310,57 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
             {
-                f8v d = splat8(0.0f);
+                acc8v d = splat8(0.0f);
                 for (int ks = 0; ks < KT; ++ks)
-                    d = mma16(lds_row16(&hb[16 * wv + m][16 * ks]), img[(L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m], d);
-                const u8v vo = operand_from_f8(d);
-                if (!hf)
-                    vsh[h][nt][wv][m] = vo;
+                    d = mma16(op_lds(&hb[16 * wv + m][16 * ks]), op_img(img, (L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m), d);
+                op_img_store(&vsh[h][nt][0][0], wv * 16 + m, operand_from_f8(d));
             }
             block_sync();
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
 #pragma unroll
                 for (int kt = 0; kt < 4; ++kt)
-                    vtop[nt][kt] = vsh[h][nt][kt][m];
+                    vtop[nt][kt] = op_img(&vsh[h][nt][0][0], kt * 16 + m);
         }
-        u8v pop[4];
+        op_t pop[4];
         if constexpr (L::POS)
         {
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
-                pop[kt] = ((const u8v*)bias)[((h * 4 + wv) * 4 + kt) * 16 + m];
+                pop[kt] = op_img((const u8v*)bias, ((h * 4 + wv) * 4 + kt) * 16 + m);
         }
         else
         {
-        u8v qop[2];
+        op_t qop[2];
         // six independent chains (Q, K, V x 2 n-tiles), each over kt in order
-        f8v dq[6];
+        acc8v dq[6];
 #pragma unroll
         for (int u = 0; u < 6; ++u)
             dq[u] = splat8(0.0f);
         for (int kt = 0; kt < KT; ++kt)
         {
-            const u8v row = lds_row16(hrow + 16 * kt);
+            const op_t row = op_lds(hrow + 16 * kt);
 #pragma unroll
             for (int u = 0; u < 6; ++u)
-                dq[u] = mma16(img[(L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)) * 16 + m], row, dq[u]);
+                dq[u] = mma16(op_img(img, (L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)) * 16 + m), row, dq[u]);
         }
 #pragma unroll
         for (int j = 0; j < 3; ++j)
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
             {
-                const f8v d = dq[2 * j + nt];
+                const acc8v d = dq[2 * j + nt];
                 if (j == 0)
                     qop[nt] = operand_from_f8(d);
                 else if (j == 1)
                 {
-                    const u8v kop = operand_from_f8(d);
-                    if (!hf)
-                        store_row16(&kl[16 * wv + m][16 * nt], kop);
+                    op_store(&kl[16 * wv + m][16 * nt], operand_from_f8(d));
                 }
                 else
                 {
 #pragma unroll
                     for (int i = 0; i < 8; ++i)
-                        vt[16 * nt + 2 * i + hf][16 * wv + m] = (half_t)d[i];
+                        vt[16 * nt + wm_acc_row(i)][16 * wv + m] = (half_t)d[i];
                 }
             }
         block_sync();
@@ -349,7 +370,7 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
         for (int kt = 0; kt < 4; ++kt)
         {
             const u4v bi = bias[((h * 4 + wv) * 4 + kt) * 32 + l];
-            f8v d;
+            acc8v d;
 #pragma unroll
             for (int i = 0; i < 4; ++i)
             {
@@ -358,15 +379,16 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
             }
 #pragma unroll
             for (int ds = 0; ds < 2; ++ds)
-                d = mma16(lds_row16(&kl[16 * kt + m][16 * ds]), qop[ds], d);
+                d = mma16(op_lds(&kl[16 * kt + m][16 * ds]), qop[ds], d);
 #pragma unroll
             for (int i = 0; i < 8; ++i)
             {
                 e[kt][i] = softmax_e((half_t)d[i]);
-                rs += (float)e[kt][i];
+                if constexpr (!kShimRowSum)
+                    rs += (float)e[kt][i];
             }
         }
-        rs += __builtin_bit_cast(float, other_half(__builtin_bit_cast(uint32_t, rs)));
+        rs = row_sum_total(rs, e);
         const half_t rc = (half_t)__builtin_amdgcn_rcpf((float)(half_t)rs);
 #pragma unroll
         for (int kt = 0; kt < 4; ++kt)
@@ -378,31 +400,31 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
             pop[kt] = operand_from_dt(pv);
         }
         }
-        u8v oop[2];
+        op_t oop[2];
 #pragma unroll
         for (int nt = 0; nt < 2; ++nt)
         {
-            f8v d = splat8(0.0f);
+            acc8v d = splat8(0.0f);
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
             {
                 if constexpr (L::POS)
                     d = mma16(vtop[nt][kt], pop[kt], d);
                 else
-                    d = mma16(lds_row16(&vt[16 * nt + m][16 * kt]), pop[kt], d);
+                    d = mma16(op_lds(&vt[16 * nt + m][16 * kt]), pop[kt], d);
             }
             oop[nt] = operand_from_f8(d);
 #ifdef PWIN_DBG
             if (DBG_ON)
                 for (int i = 0; i < 8; ++i)
-                    g_dbg[2 + h][16 * wv + m][16 * nt + 2 * i + hf] = (float)(half_t)d[i];
+                    g_dbg[2 + h][16 * wv + m][16 * nt + wm_acc_row(i)] = (float)(half_t)d[i];
 #endif
         }
 #pragma unroll
         for (int nt = 0; nt < KT; ++nt)
 #pragma unroll
             for (int ks = 0; ks < 2; ++ks)
-                acc[nt] = mma16(img[(L::T_WO + (2 * h + ks) * KT + nt) * 16 + m], oop[ks], acc[nt]);
+                acc[nt] = mma16(op_img(img, (L::T_WO + (2 * h + ks) * KT + nt) * 16 + m), oop[ks], acc[nt]);
         if constexpr (!L::POS)
             block_sync(); // kl / vt are reused by the next head
     }
@@ -411,43 +433,41 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
     if (DBG_ON)
         for (int nt = 0; nt < KT; ++nt)
             for (int i = 0; i < 8; ++i)
-                g_dbg[1][16 * wv + m][16 * nt + 2 * i + hf] = acc[nt][i];
+                g_dbg[1][16 * wv + m][16 * nt + wm_acc_row(i)] = acc[nt][i];
 #endif
     STAMP(3);
     // ---- residual; MLP init acc = x1 + b2, MLP input m = x1 * g2 (-> hb)
 #pragma unroll
     for (int nt = 0; nt < KT; ++nt)
     {
-        const u8v xv = lds_row16(arow + 16 * nt);
+        const op_t xv = op_lds(arow + 16 * nt);
         half_t mv[8], g2[8], b2[8];
-        dvec8(w, L::G2 + 32 * nt, hf, g2);
-        dvec8(w, L::B2 + 32 * nt, hf, b2);
+        dvec8(w, L::G2 + 32 * nt, g2);
+        dvec8(w, L::B2 + 32 * nt, b2);
 #pragma unroll
         for (int i = 0; i < 8; ++i)
         {
-            const half_t x1 = (half_t)acc[nt][i] + op_get(xv, 2 * i + hf);
+            const half_t x1 = (half_t)acc[nt][i] + wm_op_acc_elem(xv, i);
             mv[i] = x1 * g2[i];
             acc[nt][i] = (float)(half_t)(x1 + b2[i]);
         }
-        const u8v mo = operand_from_dt(mv);
-        if (!hf)
-            store_row16(hrow + 16 * nt, mo);
+        op_store(hrow + 16 * nt, operand_from_dt(mv));
     }
     __builtin_amdgcn_wave_barrier();
 
     // ---- MLP: C/8 chunks of 32 hidden
     for (int c = 0; c < L::NMLP; ++c)
     {
-        u8v gop[2];
+        op_t gop[2];
 #pragma unroll
         for (int hn = 0; hn < 2; ++hn)
         {
-            f8v d;
+            acc8v d;
 {
 
     half_t vv_[8];
 
-    dvec8(w, L::B1 + 64 * c + 32 * hn, hf, vv_);
+    dvec8(w, L::B1 + 64 * c + 32 * hn, vv_);
 
 #pragma unroll
 
@@ -457,7 +477,7 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
 
 }
             for (int kt = 0; kt < KT; ++kt)
-                d = mma16(img[(L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m], lds_row16(hrow + 16 * kt), d);
+                d = mma16(op_img(img, (L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m), op_lds(hrow + 16 * kt), d);
             half_t g[8];
             gelu8(d, g);
             gop[hn] = operand_from_dt(g);
@@ -466,16 +486,14 @@ __device__ void pwin_core(const PwinParams& p, const u8v* __restrict__ img, cons
         for (int nt = 0; nt < KT; ++nt)
 #pragma unroll
             for (int ks = 0; ks < 2; ++ks)
-                acc[nt] = mma16(img[(L::T_W2 + (c * 2 + ks) * KT + nt) * 16 + m], gop[ks], acc[nt]);
+                acc[nt] = mma16(op_img(img, (L::T_W2 + (c * 2 + ks) * KT + nt) * 16 + m), gop[ks], acc[nt]);
     }
     STAMP(4);
     // y -> act (rows of this wave)
 #pragma unroll
     for (int nt = 0; nt < KT; ++nt)
     {
-        const u8v yo = operand_from_f8(acc[nt]);
-        if (!hf)
-            store_row16(arow + 16 * nt, yo);
+        op_store(arow + 16 * nt, operand_from_f8(acc[nt]));
     }
     __builtin_amdgcn_wave_barrier();
 }
@@ -530,31 +548,29 @@ __device__ void pwin_encoder(const PwinParams& p0, const u8v* __restrict__ img,
         if constexpr (L::CIN != 0)
         {
             const half_t* xin = (const half_t*)p.in + (size_t)(Y * p.W + X) * L::CIN;
-            u8v xe[L::KE];
+            op_t xe[L::KE];
 #pragma unroll
             for (int ks = 0; ks < L::KE; ++ks)
-                xe[ks] = gload_row16(xin + 16 * ks);
+                xe[ks] = op_gload(xin + 16 * ks);
 #pragma unroll
             for (int nt = 0; nt < C / 16; ++nt)
             {
-                f8v d;
+                acc8v d;
                 {
                     half_t vv_[8];
-                    dvec8(p0.w, 32 * nt, hf, vv_);
+                    dvec8(p0.w, 32 * nt, vv_);
 #pragma unroll
                     for (int i = 0; i < 8; ++i)
                         d[i] = (float)vv_[i];
                 }
 #pragma unroll
                 for (int ks = 0; ks < L::KE; ++ks)
-                    d = mma16(img[(L::T_EMB + ks * (C / 16) + nt) * 16 + m], xe[ks], d);
+                    d = mma16(op_img(img, (L::T_EMB + ks * (C / 16) + nt) * 16 + m), xe[ks], d);
                 half_t xo[8];
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     xo[i] = hmax((half_t)d[i], (half_t)0.0f);
-                const u8v xv = operand_from_dt(xo);
-                if (!hf)
-                    store_row16(&act[tok][16 * nt], xv);
+                op_store(&act[tok][16 * nt], operand_from_dt(xo));
             }
         }
         else
@@ -587,12 +603,12 @@ __device__ void pwin_encoder(const PwinParams& p0, const u8v* __restrict__ img,
     const uint8_t* w = p.w;
     for (int vt = wv; vt < L::PMT; vt += 4)
     {
-        f8v d;
+        acc8v d;
 {
 
     half_t vv_[8];
 
-    dvec8(w, L::PMB + 32 * vt, hf, vv_);
+    dvec8(w, L::PMB + 32 * vt, vv_);
 
 #pragma unroll
 
@@ -606,18 +622,11 @@ __device__ void pwin_encoder(const PwinParams& p0, const u8v* __restrict__ img,
         {
             const int sub = ks / L::KT, dy = sub >> 1, dx = sub & 1;
             const int src = 8 * (2 * my + dy) + 2 * mx + dx;
-            d = mma16(img[(L::T_PM + ks * L::PMT + vt) * 16 + m], lds_row16(&act[src][16 * (ks % L::KT)]), d);
+            d = mma16(op_img(img, (L::T_PM + ks * L::PMT + vt) * 16 + m), op_lds(&act[src][16 * (ks % L::KT)]), d);
         }
-        const u8v mo = operand_from_f8(d);
         // allocated column 16 vt + c belongs to original warp g, local column 16 u + c
         const int g = vt / (L::NPA / 16), u = vt % (L::NPA / 16), valid = L::NPW - 16 * u;
-        half_t* dst = mout + L::NPW * g + 16 * u;
-        if (!hf && minb)
-        {
-            gstore16(dst, (u4v){mo[0], mo[1], mo[2], mo[3]});
-            if (valid >= 16)
-                gstore16(dst + 8, (u4v){mo[4], mo[5], mo[6], mo[7]});
-        }
+        op_gstore(mout + L::NPW * g + 16 * u, operand_from_f8(d), minb, valid);
     }
 }
 
@@ -744,12 +753,12 @@ __device__ void pwin_decoder(const PwinParams& p, const u8v* __restrict__ img, c
         for (int nt = 0; nt < C / 16; ++nt)
         {
             const int gnt = wv * (C / 16) + nt; // expand output column tile
-            f8v d;
+            acc8v d;
 {
 
     half_t vv_[8];
 
-    dvec8(p.w, L::EXPB + 32 * gnt, hf, vv_);
+    dvec8(p.w, L::EXPB + 32 * gnt, vv_);
 
 #pragma unroll
 
@@ -759,15 +768,13 @@ __device__ void pwin_decoder(const PwinParams& p, const u8v* __restrict__ img, c
 
 }
             for (int ks = 0; ks < L::KL; ++ks)
-                d = mma16(img[(L::T_EXP + ks * L::EXPN + gnt) * 16 + m], gload_row16(xl + 16 * ks), d);
-            const u8v sv = gload_row16(sk + 16 * nt);
+                d = mma16(op_img(img, (L::T_EXP + ks * L::EXPN + gnt) * 16 + m), op_gload(xl + 16 * ks), d);
+            const op_t sv = op_gload(sk + 16 * nt);
             half_t xo[8];
 #pragma unroll
             for (int i = 0; i < 8; ++i)
-                xo[i] = (half_t)d[i] + op_get(sv, 2 * i + hf);
-            const u8v xv = operand_from_dt(xo);
-            if (!hf)
-                store_row16(&act[t][16 * nt], xv);
+                xo[i] = (half_t)d[i] + wm_op_acc_elem(sv, i);
+            op_store(&act[t][16 * nt], operand_from_dt(xo));
         }
     }
     block_sync();
@@ -783,12 +790,12 @@ __device__ void pwin_decoder(const PwinParams& p, const u8v* __restrict__ img, c
 #pragma unroll
         for (int nt = 0; nt < L::NOUTA / 16; ++nt)
         {
-            f8v d;
+            acc8v d;
 {
 
     half_t vv_[8];
 
-    dvec8(q.w, L::HB + 32 * nt, hf, vv_);
+    dvec8(q.w, L::HB + 32 * nt, vv_);
 
 #pragma unroll
 
@@ -799,15 +806,8 @@ __device__ void pwin_decoder(const PwinParams& p, const u8v* __restrict__ img, c
 }
 #pragma unroll
             for (int kt = 0; kt < L::KT; ++kt)
-                d = mma16(img[(L::T_HEAD + kt * (L::NOUTA / 16) + nt) * 16 + m], lds_row16(&act[tok][16 * kt]), d);
-            const u8v ho = operand_from_f8(d);
-            const int valid = L::NOUT - 16 * nt;
-            if (!hf && inb)
-            {
-                gstore16(hout + 16 * nt, (u4v){ho[0], ho[1], ho[2], ho[3]});
-                if (valid >= 16)
-                    gstore16(hout + 16 * nt + 8, (u4v){ho[4], ho[5], ho[6], ho[7]});
-            }
+                d = mma16(op_img(img, (L::T_HEAD + kt * (L::NOUTA / 16) + nt) * 16 + m), op_lds(&act[tok][16 * kt]), d);
+            op_gstore(hout + 16 * nt, operand_from_f8(d), inb, L::NOUT - 16 * nt);
         }
     }
     else if (inb)

kernels/k/pwin_pos.h

@@ -8,6 +8,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
                          half_t (*act)[L::C], half_t (*hb)[L::C])
 {
     constexpr int C = L::C, KT = L::KT, H = L::H, NW = 4 / MT;
+    static_assert(MT == 1 || MT == 2 || MT == 4, "position token tiles per wave");
     static_assert(L::POS, "position-only attention layers");
     const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
     const int wv = threadIdx.z;
@@ -19,18 +20,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
     {
         const half_t* arow = act[16 * (wv * MT + mi) + m];
         half_t* hrow = hb[16 * (wv * MT + mi) + m];
-        float ss = 0.0f;
-#pragma unroll
-        for (int kt = 0; kt < KT; ++kt)
-        {
-            const u8v xv = lds_row16(arow + 16 * kt);
-#pragma unroll
-            for (int c = 0; c < 16; ++c)
-            {
-                const half_t v = op_get(xv, c);
-                ss += (float)(half_t)(v * v);
-            }
-        }
+        const half_t ss = l2_sum<C>(arow);
         const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss);
         for (int kt = (int)hf; kt < KT; kt += 2)
         {
@@ -48,12 +38,12 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
     STAMP(2);
 
     // ---- attention: O = P V per head (P from the table), output projection accumulated head by head
-    f8v acc[MT][KT];
+    acc8v acc[MT][KT];
 #pragma unroll
     for (int nt = 0; nt < KT; ++nt)
     {
         half_t vv[8];
-        dvec8(w, L::BO + 32 * nt, hf, vv);
+        dvec8(w, L::BO + 32 * nt, vv);
 #pragma unroll
         for (int mi = 0; mi < MT; ++mi)
 #pragma unroll
@@ -67,7 +57,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
 #endif
     for (int h = 0; h < H; ++h)
     {
-        u8v vtop[2][4];
+        op_t vtop[2][4];
 #ifdef POS_VSHARE
 #pragma unroll
         for (int nt = 0; nt < 2; ++nt)
@@ -75,46 +65,44 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
             for (int mi = 0; mi < MT; ++mi)
             {
                 const int kt = wv * MT + mi;
-                f8v d = splat8(0.0f);
+                acc8v d = splat8(0.0f);
 #pragma unroll
                 for (int ks = 0; ks < KT; ++ks)
-                    d = mma16(lds_row16(&hb[16 * kt + m][16 * ks]), img[(L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m], d);
-                const u8v vo = operand_from_f8(d);
-                if (!hf)
-                    vsh[h][nt][kt][m] = vo;
+                    d = mma16(op_lds(&hb[16 * kt + m][16 * ks]), op_img(img, (L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m), d);
+                op_img_store(&vsh[h][nt][0][0], kt * 16 + m, operand_from_f8(d));
             }
         block_sync();
 #pragma unroll
         for (int nt = 0; nt < 2; ++nt)
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
-                vtop[nt][kt] = vsh[h][nt][kt][m];
+                vtop[nt][kt] = op_img(&vsh[h][nt][0][0], kt * 16 + m);
 #else
 #pragma unroll
         for (int nt = 0; nt < 2; ++nt)
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
             {
-                f8v d = splat8(0.0f);
+                acc8v d = splat8(0.0f);
 #pragma unroll
                 for (int ks = 0; ks < KT; ++ks)
-                    d = mma16(lds_row16(&hb[16 * kt + m][16 * ks]), img[(L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m], d);
+                    d = mma16(op_lds(&hb[16 * kt + m][16 * ks]), op_img(img, (L::T_QKV + ((h * 3 + 2) * KT + ks) * 2 + nt) * 16 + m), d);
                 vtop[nt][kt] = operand_from_f8(d);
             }
 #endif
-        u8v oop[MT][2];
+        op_t oop[MT][2];
 #pragma unroll
         for (int mi = 0; mi < MT; ++mi)
         {
             const int qt = wv * MT + mi;
-            u8v pop[4];
+            op_t pop[4];
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
-                pop[kt] = ((const u8v*)bias)[((h * 4 + qt) * 4 + kt) * 16 + m];
+                pop[kt] = op_img((const u8v*)bias, ((h * 4 + qt) * 4 + kt) * 16 + m);
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
             {
-                f8v d = splat8(0.0f);
+                acc8v d = splat8(0.0f);
 #pragma unroll
                 for (int kt = 0; kt < 4; ++kt)
                     d = mma16(vtop[nt][kt], pop[kt], d);
@@ -126,7 +114,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
 #pragma unroll
             for (int ks = 0; ks < 2; ++ks)
             {
-                const u8v wo = img[(L::T_WO + (2 * h + ks) * KT + nt) * 16 + m];
+                const op_t wo = op_img(img, (L::T_WO + (2 * h + ks) * KT + nt) * 16 + m);
 #pragma unroll
                 for (int mi = 0; mi < MT; ++mi)
                     acc[mi][nt] = mma16(wo, oop[mi][ks], acc[mi][nt]);
@@ -139,24 +127,22 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
     for (int nt = 0; nt < KT; ++nt)
     {
         half_t g2[8], b2[8];
-        dvec8(w, L::G2 + 32 * nt, hf, g2);
-        dvec8(w, L::B2 + 32 * nt, hf, b2);
+        dvec8(w, L::G2 + 32 * nt, g2);
+        dvec8(w, L::B2 + 32 * nt, b2);
 #pragma unroll
         for (int mi = 0; mi < MT; ++mi)
         {
             const int tok = 16 * (wv * MT + mi) + m;
-            const u8v xv = lds_row16(&act[tok][16 * nt]);
+            const op_t xv = op_lds(&act[tok][16 * nt]);
             half_t mv[8];
 #pragma unroll
             for (int i = 0; i < 8; ++i)
             {
-                const half_t x1 = (half_t)acc[mi][nt][i] + op_get(xv, 2 * i + hf);
+                const half_t x1 = (half_t)acc[mi][nt][i] + wm_op_acc_elem(xv, i);
                 mv[i] = x1 * g2[i];
                 acc[mi][nt][i] = (float)(half_t)(x1 + b2[i]);
             }
-            const u8v mo = operand_from_dt(mv);
-            if (!hf)
-                store_row16(&hb[tok][16 * nt], mo);
+            op_store(&hb[tok][16 * nt], operand_from_dt(mv));
         }
     }
     __builtin_amdgcn_wave_barrier();
@@ -165,13 +151,13 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
     // ---- MLP: C/8 chunks of 32 hidden; each weight operand serves the MT tiles
     for (int c = 0; c < L::NMLP; ++c)
     {
-        u8v gop[MT][2];
+        op_t gop[MT][2];
 #pragma unroll
         for (int hn = 0; hn < 2; ++hn)
         {
             half_t bb[8];
-            dvec8(w, L::B1 + 64 * c + 32 * hn, hf, bb);
-            f8v d[MT];
+            dvec8(w, L::B1 + 64 * c + 32 * hn, bb);
+            acc8v d[MT];
 #pragma unroll
             for (int mi = 0; mi < MT; ++mi)
 #pragma unroll
@@ -180,10 +166,10 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
 #pragma unroll
             for (int kt = 0; kt < KT; ++kt)
             {
-                const u8v w1 = img[(L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m];
+                const op_t w1 = op_img(img, (L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m);
 #pragma unroll
                 for (int mi = 0; mi < MT; ++mi)
-                    d[mi] = mma16(w1, lds_row16(&hb[16 * (wv * MT + mi) + m][16 * kt]), d[mi]);
+                    d[mi] = mma16(w1, op_lds(&hb[16 * (wv * MT + mi) + m][16 * kt]), d[mi]);
             }
 #pragma unroll
             for (int mi = 0; mi < MT; ++mi)
@@ -198,7 +184,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
 #pragma unroll
             for (int ks = 0; ks < 2; ++ks)
             {
-                const u8v w2 = img[(L::T_W2 + (c * 2 + ks) * KT + nt) * 16 + m];
+                const op_t w2 = op_img(img, (L::T_W2 + (c * 2 + ks) * KT + nt) * 16 + m);
 #pragma unroll
                 for (int mi = 0; mi < MT; ++mi)
                     acc[mi][nt] = mma16(w2, gop[mi][ks], acc[mi][nt]);
@@ -211,9 +197,7 @@ __device__ void pos_core(const PwinParams& p, const u8v* __restrict__ img, const
 #pragma unroll
         for (int nt = 0; nt < KT; ++nt)
         {
-            const u8v yo = operand_from_f8(acc[mi][nt]);
-            if (!hf)
-                store_row16(&act[16 * (wv * MT + mi) + m][16 * nt], yo);
+            op_store(&act[16 * (wv * MT + mi) + m][16 * nt], operand_from_f8(acc[mi][nt]));
         }
     block_sync();
 }
@@ -239,29 +223,27 @@ __device__ void pos_encoder(const PwinParams& p0, const u8v* __restrict__ img, c
         if constexpr (L::CIN != 0)
         {
             const half_t* xin = (const half_t*)p.in + (size_t)(Y * p.W + X) * L::CIN;
-            u8v xe[L::KE];
+            op_t xe[L::KE];
 #pragma unroll
             for (int ks = 0; ks < L::KE; ++ks)
-                xe[ks] = gload_row16(xin + 16 * ks);
+                xe[ks] = op_gload(xin + 16 * ks);
 #pragma unroll
             for (int nt = 0; nt < C / 16; ++nt)
             {
                 half_t vv[8];
-                dvec8(p0.w, 32 * nt, hf, vv);
-                f8v d;
+                dvec8(p0.w, 32 * nt, vv);
+                acc8v d;
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     d[i] = (float)vv[i];
 #pragma unroll
                 for (int ks = 0; ks < L::KE; ++ks)
-                    d = mma16(img[(L::T_EMB + ks * (C / 16) + nt) * 16 + m], xe[ks], d);
+                    d = mma16(op_img(img, (L::T_EMB + ks * (C / 16) + nt) * 16 + m), xe[ks], d);
                 half_t xo[8];
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     xo[i] = hmax((half_t)d[i], (half_t)0.0f);
-                const u8v xv = operand_from_dt(xo);
-                if (!hf)
-                    store_row16(&act[tok][16 * nt], xv);
+                op_store(&act[tok][16 * nt], operand_from_dt(xo));
             }
         }
         else
@@ -296,8 +278,8 @@ __device__ void pos_encoder(const PwinParams& p0, const u8v* __restrict__ img, c
     for (int vt = wv; vt < L::PMT; vt += NW)
     {
         half_t vv[8];
-        dvec8(p.w, L::PMB + 32 * vt, hf, vv);
-        f8v d;
+        dvec8(p.w, L::PMB + 32 * vt, vv);
+        acc8v d;
 #pragma unroll
         for (int i = 0; i < 8; ++i)
             d[i] = (float)vv[i];
@@ -305,17 +287,10 @@ __device__ void pos_encoder(const PwinParams& p0, const u8v* __restrict__ img, c
         {
             const int sub = ks / L::KT, dy = sub >> 1, dx = sub & 1;
             const int src = 8 * (2 * my + dy) + 2 * mx + dx;
-            d = mma16(img[(L::T_PM + ks * L::PMT + vt) * 16 + m], lds_row16(&act[src][16 * (ks % L::KT)]), d);
+            d = mma16(op_img(img, (L::T_PM + ks * L::PMT + vt) * 16 + m), op_lds(&act[src][16 * (ks % L::KT)]), d);
         }
-        const u8v mo = operand_from_f8(d);
         const int g = vt / (L::NPA / 16), u = vt % (L::NPA / 16), valid = L::NPW - 16 * u;
-        half_t* dst = mout + L::NPW * g + 16 * u;
-        if (!hf && minb)
-        {
-            gstore16(dst, (u4v){mo[0], mo[1], mo[2], mo[3]});
-            if (valid >= 16)
-                gstore16(dst + 8, (u4v){mo[4], mo[5], mo[6], mo[7]});
-        }
+        op_gstore(mout + L::NPW * g + 16 * u, operand_from_f8(d), minb, valid);
     }
     STAMP(8);
 }
@@ -333,10 +308,10 @@ __device__ void pos_decoder(const PwinParams& p, const u8v* __restrict__ img, co
         const int W2 = p.W / 2, H2 = p.H / 2, ly = m >> 2, lx = m & 3;
         const int LX = mirror((8 * bx - p.sx) / 2 + lx, W2), LY = mirror((8 * by - p.sy) / 2 + ly, H2);
         const half_t* xl = (const half_t*)p.in + (size_t)(LY * W2 + LX) * L::CL;
-        u8v xls[L::KL];
+        op_t xls[L::KL];
 #pragma unroll
         for (int ks = 0; ks < L::KL; ++ks)
-            xls[ks] = gload_row16(xl + 16 * ks);
+            xls[ks] = op_gload(xl + 16 * ks);
         for (int q = wv; q < 4; q += NW)
         {
             const int dy = q >> 1, dx = q & 1, t = 8 * (2 * ly + dy) + 2 * lx + dx;
@@ -347,22 +322,20 @@ __device__ void pos_decoder(const PwinParams& p, const u8v* __restrict__ img, co
             {
                 const int gnt = q * (C / 16) + nt;
                 half_t vv[8];
-                dvec8(p.w, L::EXPB + 32 * gnt, hf, vv);
-                f8v d;
+                dvec8(p.w, L::EXPB + 32 * gnt, vv);
+                acc8v d;
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     d[i] = (float)vv[i];
 #pragma unroll
                 for (int ks = 0; ks < L::KL; ++ks)
-                    d = mma16(img[(L::T_EXP + ks * L::EXPN + gnt) * 16 + m], xls[ks], d);
-                const u8v sv = gload_row16(sk + 16 * nt);
+                    d = mma16(op_img(img, (L::T_EXP + ks * L::EXPN + gnt) * 16 + m), xls[ks], d);
+                const op_t sv = op_gload(sk + 16 * nt);
                 half_t xo[8];
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
-                    xo[i] = (half_t)d[i] + op_get(sv, 2 * i + hf);
-                const u8v xv = operand_from_dt(xo);
-                if (!hf)
-                    store_row16(&act[t][16 * nt], xv);
+                    xo[i] = (half_t)d[i] + wm_op_acc_elem(sv, i);
+                op_store(&act[t][16 * nt], operand_from_dt(xo));
             }
         }
     }
@@ -380,22 +353,15 @@ __device__ void pos_decoder(const PwinParams& p, const u8v* __restrict__ img, co
         for (int nt = 0; nt < L::NOUTA / 16; ++nt)
         {
             half_t vv[8];
-            dvec8(q.w, L::HB + 32 * nt, hf, vv);
-            f8v d;
+            dvec8(q.w, L::HB + 32 * nt, vv);
+            acc8v d;
 #pragma unroll
             for (int i = 0; i < 8; ++i)
                 d[i] = (float)vv[i];
 #pragma unroll
             for (int kt = 0; kt < L::KT; ++kt)
-                d = mma16(img[(L::T_HEAD + kt * (L::NOUTA / 16) + nt) * 16 + m], lds_row16(&act[tok][16 * kt]), d);
-            const u8v ho = operand_from_f8(d);
-            const int valid = L::NOUT - 16 * nt;
-            if (!hf && inb)
-            {
-                gstore16(hout + 16 * nt, (u4v){ho[0], ho[1], ho[2], ho[3]});
-                if (valid >= 16)
-                    gstore16(hout + 16 * nt + 8, (u4v){ho[4], ho[5], ho[6], ho[7]});
-            }
+                d = mma16(op_img(img, (L::T_HEAD + kt * (L::NOUTA / 16) + nt) * 16 + m), op_lds(&act[tok][16 * kt]), d);
+            op_gstore(hout + 16 * nt, operand_from_f8(d), inb, L::NOUT - 16 * nt);
         }
     }
 }

kernels/k/pwin_wide.h

@@ -38,17 +38,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
     half_t* hrow = hb[tok];
 
     // ---- L2 norm: h1 = x0 * (rsqrt(sum x0^2) * gamma1) -> hb (all groups of a token tile compute the sum)
-    float ss = 0.0f;
-    for (int kt = 0; kt < KT; ++kt)
-    {
-        const u8v xv = lds_row16(arow + 16 * kt);
-#pragma unroll
-        for (int c = 0; c < 16; ++c)
-        {
-            const half_t v = op_get(xv, c);
-            ss += (float)(half_t)(v * v);
-        }
-    }
+    const half_t ss = l2_sum<C>(arow);
     const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss);
     for (int kt = 2 * g + (int)hf; kt < KT; kt += 2 * NG)
     {
@@ -61,7 +51,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
                           op_get(xv, 2 * j + 1) * (half_t)(r16 * op_get(gv, 2 * j + 1)));
         store_row16(hrow + 16 * kt, hv);
     }
-    // residual input of the owned output tiles (D^T order: channel 16 nt + 2 i + hf)
+    // residual input of the owned output tiles (D^T order: channel 16 nt + wm_acc_row(i))
     half_t x0[NTL][8];
 #pragma unroll
     for (int j = 0; j < NTL; ++j)
@@ -69,16 +59,16 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
         const int nt = g + NG * j;
         if (nt < KT)
         {
-            const u8v xv = lds_row16(arow + 16 * nt);
+            const op_t xv = op_lds(arow + 16 * nt);
 #pragma unroll
             for (int i = 0; i < 8; ++i)
-                x0[j][i] = op_get(xv, 2 * i + hf);
+                x0[j][i] = wm_op_acc_elem(xv, i);
         }
     }
     block_sync(); // hb complete, R free
 
     // ---- attention: NG heads per round; the output projection accumulates head by head
-    f8v acc[NTL];
+    acc8v acc[NTL];
 #pragma unroll
     for (int j = 0; j < NTL; ++j)
     {
@@ -86,7 +76,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
         if (nt < KT)
         {
             half_t vv[8];
-            dvec8(w, L::BO + 32 * nt, hf, vv);
+            dvec8(w, L::BO + 32 * nt, vv);
 #pragma unroll
             for (int i = 0; i < 8; ++i)
                 acc[j][i] = (float)vv[i];
@@ -99,40 +89,38 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
     for (int r = 0; r < D::ROUNDS; ++r)
     {
         const int h = g < NA ? r * NA + g : H;
-        u8v qop[2], oop[2];
+        op_t qop[2], oop[2];
         if (h < H)
         {
             // six independent chains (Q, K, V x 2 n-tiles), each over kt in order
-            f8v dq[6];
+            acc8v dq[6];
 #pragma unroll
             for (int u = 0; u < 6; ++u)
                 dq[u] = splat8(0.0f);
             for (int kt = 0; kt < KT; ++kt)
             {
-                const u8v row = lds_row16(hrow + 16 * kt);
+                const op_t row = op_lds(hrow + 16 * kt);
 #pragma unroll
                 for (int u = 0; u < 6; ++u)
-                    dq[u] = mma16(img[(L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)) * 16 + m], row, dq[u]);
+                    dq[u] = mma16(op_img(img, (L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)) * 16 + m), row, dq[u]);
             }
 #pragma unroll
             for (int j = 0; j < 3; ++j)
 #pragma unroll
                 for (int nt = 0; nt < 2; ++nt)
                 {
-                    const f8v d = dq[2 * j + nt];
+                    const acc8v d = dq[2 * j + nt];
                     if (j == 0)
                         qop[nt] = operand_from_f8(d);
                     else if (j == 1)
                     {
-                        const u8v kop = operand_from_f8(d);
-                        if (!hf)
-                            store_row16(&kl[g][tok][16 * nt], kop);
+                        op_store(&kl[g][tok][16 * nt], operand_from_f8(d));
                     }
                     else
                     {
 #pragma unroll
                         for (int i = 0; i < 8; ++i)
-                            vt[g][16 * nt + 2 * i + hf][tok] = (half_t)d[i];
+                            vt[g][16 * nt + wm_acc_row(i)][tok] = (half_t)d[i];
                     }
                 }
         }
@@ -145,7 +133,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
             for (int kt = 0; kt < 4; ++kt)
             {
                 const u4v bi = bias[((h * 4 + t) * 4 + kt) * 32 + l];
-                f8v d;
+                acc8v d;
 #pragma unroll
                 for (int i = 0; i < 4; ++i)
                 {
@@ -154,17 +142,18 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
                 }
 #pragma unroll
                 for (int ds = 0; ds < 2; ++ds)
-                    d = mma16(lds_row16(&kl[g][16 * kt + m][16 * ds]), qop[ds], d);
+                    d = mma16(op_lds(&kl[g][16 * kt + m][16 * ds]), qop[ds], d);
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                 {
                     e[kt][i] = softmax_e((half_t)d[i]);
-                    rs += (float)e[kt][i];
+                    if constexpr (!kShimRowSum)
+                        rs += (float)e[kt][i];
                 }
             }
-            rs += __builtin_bit_cast(float, other_half(__builtin_bit_cast(uint32_t, rs)));
+            rs = row_sum_total(rs, e);
             const half_t rc = (half_t)__builtin_amdgcn_rcpf((float)(half_t)rs);
-            u8v pop[4];
+            op_t pop[4];
 #pragma unroll
             for (int kt = 0; kt < 4; ++kt)
             {
@@ -177,18 +166,18 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
             {
-                f8v d = splat8(0.0f);
+                acc8v d = splat8(0.0f);
 #pragma unroll
                 for (int kt = 0; kt < 4; ++kt)
-                    d = mma16(lds_row16(&vt[g][16 * nt + m][16 * kt]), pop[kt], d);
+                    d = mma16(op_lds(&vt[g][16 * nt + m][16 * kt]), pop[kt], d);
                 oop[nt] = operand_from_f8(d);
             }
         }
         block_sync(); // K / V of the round consumed
-        if (h < H && !hf)
+        if (h < H)
         {
-            store_row16(&ob[g][tok][0], oop[0]);
-            store_row16(&ob[g][tok][16], oop[1]);
+            op_store(&ob[g][tok][0], oop[0]);
+            op_store(&ob[g][tok][16], oop[1]);
         }
         block_sync();
         for (int jh = 0; jh < NA; ++jh)
@@ -196,15 +185,15 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
             const int hh = r * NA + jh;
             if (hh >= H)
                 break;
-            const u8v o0 = lds_row16(&ob[jh][tok][0]), o1 = lds_row16(&ob[jh][tok][16]);
+            const op_t o0 = op_lds(&ob[jh][tok][0]), o1 = op_lds(&ob[jh][tok][16]);
 #pragma unroll
             for (int j = 0; j < NTL; ++j)
             {
                 const int nt = g + NG * j;
                 if (nt < KT)
                 {
-                    acc[j] = mma16(img[(L::T_WO + (2 * hh) * KT + nt) * 16 + m], o0, acc[j]);
-                    acc[j] = mma16(img[(L::T_WO + (2 * hh + 1) * KT + nt) * 16 + m], o1, acc[j]);
+                    acc[j] = mma16(op_img(img, (L::T_WO + (2 * hh) * KT + nt) * 16 + m), o0, acc[j]);
+                    acc[j] = mma16(op_img(img, (L::T_WO + (2 * hh + 1) * KT + nt) * 16 + m), o1, acc[j]);
                 }
             }
         }
@@ -220,8 +209,8 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
         if (nt < KT)
         {
             half_t mv[8], g2[8], b2[8];
-            dvec8(w, L::G2 + 32 * nt, hf, g2);
-            dvec8(w, L::B2 + 32 * nt, hf, b2);
+            dvec8(w, L::G2 + 32 * nt, g2);
+            dvec8(w, L::B2 + 32 * nt, b2);
 #pragma unroll
             for (int i = 0; i < 8; ++i)
             {
@@ -229,9 +218,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
                 mv[i] = x1 * g2[i];
                 acc[j][i] = (float)(half_t)(x1 + b2[i]);
             }
-            const u8v mo = operand_from_dt(mv);
-            if (!hf)
-                store_row16(hrow + 16 * nt, mo);
+            op_store(hrow + 16 * nt, operand_from_dt(mv));
         }
     }
     block_sync();
@@ -246,21 +233,19 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
         {
             if (c >= L::NMLP)
                 break;
-            f8v d;
+            acc8v d;
             {
                 half_t vv[8];
-                dvec8(w, L::B1 + 64 * c + 32 * hn, hf, vv);
+                dvec8(w, L::B1 + 64 * c + 32 * hn, vv);
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     d[i] = (float)vv[i];
             }
             for (int kt = 0; kt < KT; ++kt)
-                d = mma16(img[(L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m], lds_row16(hrow + 16 * kt), d);
+                d = mma16(op_img(img, (L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m), op_lds(hrow + 16 * kt), d);
             half_t gg[8];
             gelu8(d, gg);
-            const u8v gop = operand_from_dt(gg);
-            if (!hf)
-                store_row16(&hid[rr % D::NB][g][tok][16 * hn], gop);
+            op_store(&hid[rr % D::NB][g][tok][16 * hn], operand_from_dt(gg));
         }
         block_sync();
         for (int jc = 0; jc < NG; ++jc)
@@ -268,15 +253,15 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
             const int cc = rr * NG + jc;
             if (cc >= L::NMLP)
                 break;
-            const u8v h0 = lds_row16(&hid[rr % D::NB][jc][tok][0]), h1 = lds_row16(&hid[rr % D::NB][jc][tok][16]);
+            const op_t h0 = op_lds(&hid[rr % D::NB][jc][tok][0]), h1 = op_lds(&hid[rr % D::NB][jc][tok][16]);
 #pragma unroll
             for (int j = 0; j < NTL; ++j)
             {
                 const int nt = g + NG * j;
                 if (nt < KT)
                 {
-                    acc[j] = mma16(img[(L::T_W2 + (cc * 2) * KT + nt) * 16 + m], h0, acc[j]);
-                    acc[j] = mma16(img[(L::T_W2 + (cc * 2 + 1) * KT + nt) * 16 + m], h1, acc[j]);
+                    acc[j] = mma16(op_img(img, (L::T_W2 + (cc * 2) * KT + nt) * 16 + m), h0, acc[j]);
+                    acc[j] = mma16(op_img(img, (L::T_W2 + (cc * 2 + 1) * KT + nt) * 16 + m), h1, acc[j]);
                 }
             }
         }
@@ -290,9 +275,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
         const int nt = g + NG * j;
         if (nt < KT)
         {
-            const u8v yo = operand_from_f8(acc[j]);
-            if (!hf)
-                store_row16(hrow + 16 * nt, yo);
+            op_store(hrow + 16 * nt, operand_from_f8(acc[j]));
         }
     }
     block_sync();
@@ -346,10 +329,10 @@ __device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, c
     const uint8_t* w = p.w;
     for (int vt = wv; vt < L::PMT; vt += 4 * NG)
     {
-        f8v d;
+        acc8v d;
         {
             half_t vv[8];
-            dvec8(w, L::PMB + 32 * vt, hf, vv);
+            dvec8(w, L::PMB + 32 * vt, vv);
 #pragma unroll
             for (int i = 0; i < 8; ++i)
                 d[i] = (float)vv[i];
@@ -358,17 +341,10 @@ __device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, c
         {
             const int sub = ks / L::KT, dy = sub >> 1, dx = sub & 1;
             const int src = 8 * (2 * my + dy) + 2 * mx + dx;
-            d = mma16(img[(L::T_PM + ks * L::PMT + vt) * 16 + m], lds_row16(&hb[src][16 * (ks % L::KT)]), d);
+            d = mma16(op_img(img, (L::T_PM + ks * L::PMT + vt) * 16 + m), op_lds(&hb[src][16 * (ks % L::KT)]), d);
         }
-        const u8v mo = operand_from_f8(d);
         const int gw = vt / (L::NPA / 16), u = vt % (L::NPA / 16), valid = L::NPW - 16 * u;
-        half_t* dst = mout + L::NPW * gw + 16 * u;
-        if (!hf && minb)
-        {
-            gstore16(dst, (u4v){mo[0], mo[1], mo[2], mo[3]});
-            if (valid >= 16)
-                gstore16(dst + 8, (u4v){mo[4], mo[5], mo[6], mo[7]});
-        }
+        op_gstore(mout + L::NPW * gw + 16 * u, operand_from_f8(d), minb, valid);
     }
 }
 
@@ -406,24 +382,22 @@ __device__ void wide_decoder(const PwinParams& p, const u8v* __restrict__ img, c
         for (int nt = g; nt < C / 16; nt += NG)
         {
             const int gnt = q * (C / 16) + nt;
-            f8v d;
+            acc8v d;
             {
                 half_t vv[8];
-                dvec8(p.w, L::EXPB + 32 * gnt, hf, vv);
+                dvec8(p.w, L::EXPB + 32 * gnt, vv);
 #pragma unroll
                 for (int i = 0; i < 8; ++i)
                     d[i] = (float)vv[i];
             }
             for (int ks = 0; ks < L::KL; ++ks)
-                d = mma16(img[(L::T_EXP + ks * L::EXPN + gnt) * 16 + m], gload_row16(xl + 16 * ks), d);
-            const u8v sv = gload_row16(sk + 16 * nt);
+                d = mma16(op_img(img, (L::T_EXP + ks * L::EXPN + gnt) * 16 + m), op_gload(xl + 16 * ks), d);
+            const op_t sv = op_gload(sk + 16 * nt);
             half_t xo[8];
 #pragma unroll
             for (int i = 0; i < 8; ++i)
-                xo[i] = (half_t)d[i] + op_get(sv, 2 * i + hf);
-            const u8v xv = operand_from_dt(xo);
-            if (!hf)
-                store_row16(&act[tt][16 * nt], xv);
+                xo[i] = (half_t)d[i] + wm_op_acc_elem(sv, i);
+            op_store(&act[tt][16 * nt], operand_from_dt(xo));
         }
     }
     block_sync();

kernels/m/rrlite_enc1_4x4.hip

@@ -3,5 +3,6 @@
 #define SWIN_F32ACC
 #define SWIN_MAX_VGPR 192
 #define SWIN_MLP_R(NW) 1
+#define SWIN_SCALAR_BLOAD // scalar-base weight loads: faster here (no spills), slower in the 8-wave layers
 #include "swin_block.h"
 SWIN_MODULE_W(rrlite_enc1_4x4, 64, 2, 4, 96, false, CommonParams, 0)

kernels/m/rrlite_enc3_tube_4x4.hip

@@ -3,5 +3,6 @@
 #define SWIN_F32ACC
 #define SWIN_MAX_VGPR 192
 #define SWIN_MLP_R(NW) 2
+#define SWIN_PREP_SLOTS 8 // one weight image per tube block (six per frame)
 #include "swin_block.h"
 SWIN_MODULE_W(rrlite_enc3_tube_4x4, 128, 4, 8, 0, true, TubeParams, 0)

kernels/m/swin_block.h

@@ -5,7 +5,24 @@
 //   CIN  patch-expand input channels (0 = none): decoder input = f16(q8(expand(low-res CIN)) + skip)
 #pragma once
 #include "swin_common.h"
-#include <type_traits>
+#ifdef D4R_SWIN_DIAGNOSTICS
+extern "C" {
+__device__ int d4r_swin_debug_block[2] = {0, 4};
+__device__ half_t d4r_swin_debug_values[4 * 64 * 128];
+}
+template <int C, int AST> __device__ __forceinline__ void swin_snapshot(
+    const half_t* activation, int stage, int bx, int by, int tid, bool pair_order) {
+    if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1] && tid < 64)
+        for (int c = 0; c < C; ++c)
+            d4r_swin_debug_values[(stage * 64 + tid) * 128 + c] =
+                activation[tid * AST + (pair_order ? gperm(c) : apos(c))];
+    __syncthreads();
+}
+#endif
+
+// Immediate lane masks need only a compile-time integer, not the host C++
+// standard library (which is absent from device-only Windows HIP builds).
+template <int Mask> struct SwinLaneMask { static constexpr int value = Mask; };
 
 template <int C, int NW, int NPM, int CIN = 0> struct SwinLayout
 {
@@ -19,7 +36,7 @@ template <int C, int NW, int NPM, int CIN = 0> struct SwinLayout
     __host__ __device__ static constexpr int b1(int c) { return c == 0 ? M0 : M0 + 66 * C + 64 + (64 * C + 64) * (c - 1); }
     static constexpr int PM0 = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1);
     static constexpr int PMB = PM0 + 4 * C * NPM;
-    // f16 weight image (h16 units)
+    // weight image (slots of 16 K values, see expand_weights)
     __host__ __device__ static constexpr int qv(int h, int q) { return (2 * h + q) * 2 * C; }
     static constexpr int WO = 4 * C * NW;
     static constexpr int W1 = WO + 2 * NW * C;
@@ -33,6 +50,14 @@ template <int C, int NW, int NPM, int CIN = 0> struct SwinLayout
     static constexpr int NDESC = 2 * NW + 1 + 2 * NC + NPM / 32 + (CIN ? NW : 0);
 };
 
+// Start (in halves) of token T's row in the output staging buffer: stride C + 4 plus a shift by bits 4 and 5
+// of T. The patch merge's lanes read 8 bytes each from 16 tokens that differ in bits 1, 3, 4 and 5 (plus a
+// constant); this placement puts those 16 reads on 16 distinct 8-byte bank pairs for C = 64, 96 and 128.
+template <int C> __host__ __device__ constexpr int xo_row(int T)
+{
+    return (C + 4) * T + 4 * ((T >> 4) & 1) + 16 * (T >> 5);
+}
+
 template <int C, int NW, int NPM, int CIN> struct SwinDescs
 {
     GemmDesc d[SwinLayout<C, NW, NPM, CIN>::NDESC];
@@ -61,14 +86,19 @@ template <int C, int NW, int NPM, int CIN> constexpr SwinDescs<C, NW, NPM, CIN>
 
 // NH heads (NVIDIA's warps per block) run on NW waves (NW = NH, or 2 NH to halve LDS per wave)
 template <int C, int NH, int NW, int NPM, bool TUBE, int CIN>
-__device__ __forceinline__ void swin_block(const CommonParams& p, const TubeParams* tp, const h16* w16, const int bx,
+__device__ __forceinline__ void swin_block(const CommonParams& p, const TubeParams* tp, const wslot* w16, const int bx,
                                            const int by, const int gx)
 {
     using L = SwinLayout<C, NH, NPM, CIN>;
     static_assert(NW % NH == 0 && NW / NH <= 4, "waves per head");
     constexpr int NPL = L::NPL, NT = L::NT, INNER = L::INNER;
     constexpr int AST = (C > INNER ? C : INNER) + 8;
-    constexpr int GST = 40, QST = 40, PST = 24;
+    constexpr int GST = 40, PST = 24;
+#ifdef SWIN_EXACT_PV
+    constexpr int QST = 56;
+#else
+    constexpr int QST = 40;
+#endif
 #ifndef SWIN_MLP_R
 #define SWIN_MLP_R(NW) ((NW) / 2)
 #endif
@@ -77,8 +107,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     static_assert(NTW * NW == NT * MG, "column split");
     constexpr int PAST = CIN + 8;
     constexpr int SCR = NW * 16 * QST * 2 > (CIN ? 16 * PAST * 2 : 0) ? NW * 16 * QST * 2 : 16 * PAST * 2;
-    constexpr int U_ST4 = R * 64 * GST * 2, U_OB = NPM ? 64 * C : 0;
-    constexpr int USIZE = SCR > U_ST4 ? (SCR > U_OB ? SCR : U_OB) : (U_ST4 > U_OB ? U_ST4 : U_OB);
+    constexpr int U_ST4 = R * 64 * GST * 2;
+    constexpr int USIZE = SCR > U_ST4 ? SCR : U_ST4;
 
     __shared__ __attribute__((aligned(16))) half_t A[64 * AST];
     __shared__ __attribute__((aligned(16))) uint8_t U[USIZE];
@@ -118,11 +148,11 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 
     // block input words (encoders: fp8 input, decoders: fp8 skip), issued first so their latency
     // overlaps the patch expand; word k of thread tid is idx = tid + 32 NW k
-    constexpr int NSW = CIN ? 1 : 64 * C / 4 / (32 * NW);
-    static_assert(CIN || NSW * 32 * NW == 64 * C / 4, "input words per thread");
+    constexpr int NSW = 64 * C / 4 / (32 * NW);
+    static_assert(NSW * 32 * NW == 64 * C / 4, "input words per thread");
     uint32_t inw[NSW];
 #pragma unroll
-    for (int k = 0; k < (CIN ? 0 : NSW); ++k)
+    for (int k = 0; k < NSW; ++k)
     {
         const int idx = tid + 32 * NW * k;
         const int T = idx / (C / 4), c4 = 4 * (idx % (C / 4));
@@ -130,26 +160,24 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
         const uint8_t* src = CIN ? p.p32 : p.in;
         inw[k] = *(const uint32_t*)(src + ((size_t)((c4 >> 5) * p.th + Y) * p.tw + X) * 32 + (c4 & 31));
     }
+    // the input words as f16 into X0 (natural channel order); decoders add the patch expand to them later
+    auto stage_input_words = [&]() {
+#pragma unroll
+        for (int k = 0; k < NSW; ++k)
+        {
+            const int idx = tid + 32 * NW * k;
+            const int T = idx / (C / 4), c4 = 4 * (idx % (C / 4));
+            const uint32_t bytes = inw[k];
+#pragma unroll
+            for (int b = 0; b < 4; ++b)
+                X0[T * AST + c4 + b] = e4m3_to_half((bytes >> (8 * b)) & 0xffu);
+        }
+    };
 
     // ------------------------------------------------ stage 0: block input x0 (f16) into A
     // encoders: the fp8 input; decoders: q8(patch expand(low-res)) + skip
     if constexpr (CIN > 0)
     {
-        constexpr int NTEW_ = 4 * C / NW / 16;
-        uint32_t sk[NTEW_][8];
-#pragma unroll
-        for (int j = 0; j < NTEW_; ++j)
-        {
-            const int Gc = 16 * (NTEW_ * wv + j) + l16, q = Gc / C, ch = ginv(Gc % C);
-#pragma unroll
-            for (int i = 0; i < 8; ++i)
-            {
-                const int r = 2 * i + hi;
-                const int tx = 2 * (r & 3) + (q & 1), ty = 2 * (r >> 2) + (q >> 1);
-                const int X = mirror(8 * bx - p.sx + tx, p.tw), Y = mirror(8 * by - p.sy + ty, p.th);
-                sk[j][i] = p.p32[((size_t)((ch >> 5) * p.th + Y) * p.tw + X) * 32 + (ch & 31)];
-            }
-        }
         const int W2 = p.tw / 2, H2 = p.th / 2;
         for (int idx = tid; idx < 16 * CIN / 4; idx += 32 * NW)
         {
@@ -161,6 +189,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
             for (int b = 0; b < 4; ++b)
                 PA[r * PAST + apos(c4 + b)] = e4m3_to_half((bytes >> (8 * b)) & 0xffu);
         }
+        stage_input_words(); // the skip input
         __syncthreads();
         constexpr int NTEW = 4 * C / NW / 16; // expand tiles per wave (L::NTE per weight block)
         T16 e[NTEW];
@@ -170,7 +199,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
         for (int kc = 0; kc < CIN / 32; ++kc)
         {
-            const h16 a0 = lds16(PA + l16 * PAST + 32 * kc), a1 = lds16(PA + l16 * PAST + 32 * kc + 16);
+            const gop a0 = lda(PA + l16 * PAST + 32 * kc), a1 = lda(PA + l16 * PAST + 32 * kc + 16);
 #pragma unroll
             for (int j = 0; j < NTEW; ++j)
             {
@@ -185,28 +214,16 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
             for (int i = 0; i < 8; ++i)
             {
-                const int r = 2 * i + hi;
+                const int r = mrow(i);
                 const int tx = 2 * (r & 3) + (q & 1), ty = 2 * (r >> 2) + (q >> 1);
                 const int T = (tx & 3) + 4 * (ty & 3) + 16 * (tx >> 2) + 32 * (ty >> 2);
-                X0[T * AST + ch] = (half_t)(q8(e[j].get(i)) + e4m3_to_half(sk[j][i]));
+                // e4m3_to_half is exact in f16, so adding the staged skip value is the same sum
+                X0[T * AST + ch] = (half_t)(q8(e[j].get(i)) + X0[T * AST + ch]);
             }
         }
     }
     else
-    {
-#pragma unroll
-    for (int k = 0; k < NSW; ++k)
-    {
-        const int idx = tid + 32 * NW * k;
-        const int T = idx / (C / 4), c4 = 4 * (idx % (C / 4));
-        const uint32_t bytes = inw[k];
-#pragma unroll
-        for (int b = 0; b < 4; ++b)
-        {
-            X0[T * AST + c4 + b] = e4m3_to_half((bytes >> (8 * b)) & 0xffu);
-        }
-    }
-    }
+        stage_input_words();
     __syncthreads();
 
     // residual accumulators of the output projection: f16(x0 + b_o) in the C-fragment layout
@@ -223,8 +240,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
             const int ch = ginv(n);
 #pragma unroll
             for (int i = 0; i < 8; i += 2)
-                x[mm][j].set_pair(i / 2, (hv2){X0[(16 * (mg * MT + mm) + 2 * i + hi) * AST + ch],
-                                         X0[(16 * (mg * MT + mm) + 2 * i + 2 + hi) * AST + ch]} + (hv2){bo, bo});
+                x[mm][j].set_pair(i / 2, (hv2){X0[(16 * (mg * MT + mm) + mrow(i)) * AST + ch],
+                                         X0[(16 * (mg * MT + mm) + mrow(i + 1)) * AST + ch]} + (hv2){bo, bo});
         }
     __syncthreads(); // stage 1 overwrites x0 rows with h1
 
@@ -296,6 +313,9 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     __syncthreads();
 
     // ------------------------------------------------ stage 2: Q/V projection + attention (wave = head)
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<C, AST>(A, 0, bx, by, tid, false);
+#endif
     {
         constexpr int MW = 4 / (NW / NH); // windows per wave
         const int h = NW == NH ? wv : wv % NH, m0 = NW == NH ? 0 : MW * (wv / NH);
@@ -308,7 +328,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
         for (int kc = 0; kc < C / 32; ++kc)
         {
-            h16 b[2][4];
+            gop b[2][4];
 #pragma unroll
             for (int s = 0; s < 2; ++s)
 #pragma unroll
@@ -317,8 +337,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
             for (int m = 0; m < MW; ++m)
             {
-                const h16 a0 = lds16(A + (16 * (m0 + m) + l16) * AST + 32 * kc);
-                const h16 a1 = lds16(A + (16 * (m0 + m) + l16) * AST + 32 * kc + 16);
+                const gop a0 = lda(A + (16 * (m0 + m) + l16) * AST + 32 * kc);
+                const gop a1 = lda(A + (16 * (m0 + m) + l16) * AST + 32 * kc + 16);
 #pragma unroll
                 for (int q = 0; q < 4; ++q)
                     k32(acc[m][q], a0, b[0][q], a1, b[1][q]);
@@ -344,7 +364,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
         for (int i = 0; i < 8; ++i)
         {
-            const int ri = 2 * i + hi, cj = l16;
+            const int ri = mrow(i), cj = l16;
             const int bl = 4 * (ri & 7) + ((cj & 7) >> 1), word = (ri >> 3) + 2 * (cj >> 3);
             bias.set(i, hload(W, bbase + 2 * (8 * bl + 2 * word + (cj & 1))));
         }
@@ -359,14 +379,14 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
                 for (int i = 0; i < 8; i += 2)
                 {
-                    Qs[(2 * i + hi) * QST + 16 * q + l16] = qp[m][q][i / 2][0];
-                    Qs[(2 * i + 2 + hi) * QST + 16 * q + l16] = qp[m][q][i / 2][1];
+                    Qs[mrow(i) * QST + 16 * q + l16] = qp[m][q][i / 2][0];
+                    Qs[mrow(i + 1) * QST + 16 * q + l16] = qp[m][q][i / 2][1];
                 }
             wave_sync();
-            const h16 a0 = lds16(Qs + l16 * QST), a1 = lds16(Qs + l16 * QST + 16);
+            const gop a0 = lda(Qs + l16 * QST), a1 = lda(Qs + l16 * QST + 16);
             T16 S = bias;
             k32(S, a0, a0, a1, a1);
-            // rows 2k+hi and 2k+2+hi of this lane's column, packed: scale, clamp and the cubic
+            // rows mrow(2k), mrow(2k+1) of this lane's column, packed: scale, clamp and the cubic
             // polynomial as f16x2, the exponent trick per row on the (even, odd) column word
             hv2 wg[4];
             const bool oddc = l16 & 1;
@@ -392,21 +412,32 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
                 auto xr = [](hv2 v, auto mask) {
                     return __builtin_bit_cast(hv2, xor_lane<decltype(mask)::value>(__builtin_bit_cast(uint32_t, v)));
                 };
-                const hv2 v1 = wg[k] + xr(wg[k], std::integral_constant<int, 8>{});
-                const hv2 v2 = v1 + xr(v1, std::integral_constant<int, 2>{});
-                const hv2 v3 = v2 + xr(v2, std::integral_constant<int, 4>{});
-                const hv2 sum = v3 + xr(v3, std::integral_constant<int, 1>{});
+                const hv2 v1 = wg[k] + xr(wg[k], SwinLaneMask<8>{});
+                const hv2 v2 = v1 + xr(v1, SwinLaneMask<2>{});
+                const hv2 v3 = v2 + xr(v2, SwinLaneMask<4>{});
+                const hv2 sum = v3 + xr(v3, SwinLaneMask<1>{});
                 const hv2 rinv = {f16(1.0f / (float)sum[0]), f16(1.0f / (float)sum[1])};
                 const hv2 pw = wg[k] * rinv;
-                Qs[(4 * k + hi) * PST + l16] = pw[0];
-                Qs[(4 * k + 2 + hi) * PST + l16] = pw[1];
+                Qs[mrow(2 * k) * PST + l16] = pw[0];
+                Qs[mrow(2 * k + 1) * PST + l16] = pw[1];
             }
             wave_sync();
-            const h16 pa = lds16(Qs + l16 * PST);
+#ifdef SWIN_EXACT_PV
+            half_t* Vs = Qs + 16 * PST;
+            for (int nt = 0; nt < 2; ++nt)
+                for (int i = 0; i < 8; ++i)
+                    Vs[mrow(i) * 32 + 16 * nt + l16] = vp[m][nt][i / 2][i & 1];
+            wave_sync();
+#endif
+            const sop pa = lds16(Qs + l16 * PST);
 #pragma unroll
             for (int nt = 0; nt < 2; ++nt)
             {
-                // V as B operand: lane holds column l16 over the 16 tokens (rows); own rows 2i+hi
+                // V as B operand: lane holds column l16 over the 16 tokens (rows, the K of P V)
+#if D4R_WMMA_LAYOUT == 12
+                // own pair k holds rows (2k + 8hi, 2k + 1 + 8hi): this half's K values in order
+                const sop vb = __builtin_bit_cast(sop, vp[m][nt]);
+#else
                 // own pair k holds rows (4k+hi, 4k+2+hi); the other half-wave has (4k+1-hi, 4k+3-hi)
                 uint32_t vw[8];
                 const uint32_t s_lo = hi ? 0x01000504u : 0x05040100u, s_hi = hi ? 0x03020706u : 0x07060302u;
@@ -419,14 +450,27 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
                     vw[2 * k + 1] = __builtin_amdgcn_perm(other, own, s_hi); // rows 4k+2, 4k+3
                 }
                 const h16 vb = __builtin_bit_cast(h16, vw);
-                const f8v o = wmma(pa, vb, splat(0.0f));
+#endif
+                f8v o = wmma(pa, vb, splat(0.0f));
+#ifdef SWIN_EXACT_PV
+                // FP16 P/V products need more than 24 significant bits in a
+                // cancellation-heavy dot. An f32 WMMA can round to the wrong
+                // FP16 neighbour at a midpoint, then flip the FP8 output code.
+                // This baseline sums the 16 FP16 products before one FP16 round.
+                for (int i = 0; i < 8; ++i) {
+                    double dot = 0.0;
+                    for (int k = 0; k < 16; ++k)
+                        dot += (double)Qs[mrow(i) * PST + k] * (double)Vs[k * 32 + 16 * nt + l16];
+                    o[i] = (float)(half_t)dot;
+                }
+#endif
                 const int ocol = apos(32 * h + ginv(16 * nt + l16));
 #pragma unroll
                 for (int i = 0; i < 8; i += 2)
                 {
                     const hv2 oq = q8x2((hv2){(half_t)o[i], (half_t)o[i + 1]});
-                    A[(16 * (m0 + m) + 2 * i + hi) * AST + ocol] = oq[0];
-                    A[(16 * (m0 + m) + 2 * i + 2 + hi) * AST + ocol] = oq[1];
+                    A[(16 * (m0 + m) + mrow(i)) * AST + ocol] = oq[0];
+                    A[(16 * (m0 + m) + mrow(i + 1)) * AST + ocol] = oq[1];
                 }
             }
             wave_sync();
@@ -435,10 +479,13 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     __syncthreads();
 
     // ------------------------------------------------ stage 3: output projection + residual
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<INNER, AST>(A, 3, bx, by, tid, false);
+#endif
 #pragma unroll
     for (int kc = 0; kc < NH; ++kc)
     {
-        h16 b[2][NTW];
+        gop b[2][NTW];
 #pragma unroll
         for (int s = 0; s < 2; ++s)
 #pragma unroll
@@ -447,8 +494,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
         for (int mm = 0; mm < MT; ++mm)
         {
-            const h16 a0 = lds16(A + (16 * (mg * MT + mm) + l16) * AST + 32 * kc);
-            const h16 a1 = lds16(A + (16 * (mg * MT + mm) + l16) * AST + 32 * kc + 16);
+            const gop a0 = lda(A + (16 * (mg * MT + mm) + l16) * AST + 32 * kc);
+            const gop a1 = lda(A + (16 * (mg * MT + mm) + l16) * AST + 32 * kc + 16);
 #pragma unroll
             for (int j = 0; j < NTW; ++j)
                 k32(x[mm][j], a0, b[0][j], a1, b[1][j]);
@@ -463,11 +510,16 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
         for (int j = 0; j < NTW; ++j)
 #pragma unroll
             for (int i = 0; i < 8; ++i)
-                A[(16 * (mg * MT + mm) + 2 * i + hi) * AST + 16 * (ng * NTW + j) + l16] = x[mm][j].get(i);
+                A[(16 * (mg * MT + mm) + mrow(i)) * AST + 16 * (ng * NTW + j) + l16] = x[mm][j].get(i);
     __syncthreads();
     if (tid < 64)
     {
         // the row as 128-bit LDS loads: element 8 j + c of n8 tile j
+#ifdef D4R_SWIN_DIAGNOSTICS
+        if (bx == d4r_swin_debug_block[0] && by == d4r_swin_debug_block[1])
+            for (int c = 0; c < C; ++c)
+                d4r_swin_debug_values[(64 + tid) * 128 + c] = A[tid * AST + gperm(c)];
+#endif
         h8 row[L::NT8];
 #pragma unroll
         for (int j = 0; j < L::NT8; ++j)
@@ -514,18 +566,21 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
             for (int i = 0; i < 8; i += 2)
             {
-                const int row = 16 * (mg * MT + mm) + 2 * i + hi;
+                const int row = 16 * (mg * MT + mm) + mrow(i), row2 = 16 * (mg * MT + mm) + mrow(i + 1);
                 const hv2 v = x[mm][j].pair(i / 2);
-                const hv2 r2 = {R2[row], R2[row + 2]};
+                const hv2 r2 = {R2[row], R2[row2]};
                 const hv2 hq = q8x2(v * (hv2)(r2 * (hv2){g2, g2}));
                 A[row * AST + col] = hq[0];
-                A[(row + 2) * AST + col] = hq[1];
+                A[row2 * AST + col] = hq[1];
                 x[mm][j].set_pair(i / 2, v + (hv2){b2, b2});
             }
     }
     __syncthreads();
 
     // ------------------------------------------------ stage 4: MLP, R chunks per round
+#ifdef D4R_SWIN_DIAGNOSTICS
+    swin_snapshot<C, AST>(A, 2, bx, by, tid, false);
+#endif
     {
         constexpr int WPC = NW / R, MPW = 4 / WPC; // waves per chunk, fc1 m tiles per wave
         const int cw = wv / WPC, mb = MPW * (wv % WPC);
@@ -546,7 +601,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
             for (int kc = 0; kc < C / 32; ++kc)
             {
-                h16 b[2][2];
+                gop b[2][2];
 #pragma unroll
                 for (int s = 0; s < 2; ++s)
 #pragma unroll
@@ -555,8 +610,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
                 for (int mm = 0; mm < MPW; ++mm)
                 {
-                    const h16 a0 = lds16(A + (16 * (mb + mm) + l16) * AST + 32 * kc);
-                    const h16 a1 = lds16(A + (16 * (mb + mm) + l16) * AST + 32 * kc + 16);
+                    const gop a0 = lda(A + (16 * (mb + mm) + l16) * AST + 32 * kc);
+                    const gop a1 = lda(A + (16 * (mb + mm) + l16) * AST + 32 * kc + 16);
                     k32(z[mm][0], a0, b[0][0], a1, b[1][0]);
                     k32(z[mm][1], a0, b[0][1], a1, b[1][1]);
                 }
@@ -579,8 +634,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
                                           (hv2)((hv2){f16(0.08108133f), f16(0.08108133f)} * az);
                         const hv2 gl = zv * (hv2)((hv2){0.5f16, 0.5f16} + (hv2)(cz * inner));
                         const hv2 gq = q8x2(gl);
-                        Gc[(16 * (mb + mm) + 2 * i + hi) * GST + col] = gq[0];
-                        Gc[(16 * (mb + mm) + 2 * i + 2 + hi) * GST + col] = gq[1];
+                        Gc[(16 * (mb + mm) + mrow(i)) * GST + col] = gq[0];
+                        Gc[(16 * (mb + mm) + mrow(i + 1)) * GST + col] = gq[1];
                     }
                 }
             __syncthreads();
@@ -588,7 +643,7 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
             for (int cc = 0; cc < R; ++cc)
             {
                 const int c2 = R * r + cc;
-                h16 b[2][NTW];
+                gop b[2][NTW];
 #pragma unroll
                 for (int s = 0; s < 2; ++s)
 #pragma unroll
@@ -597,8 +652,8 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
                 for (int mm = 0; mm < MT; ++mm)
                 {
-                    const h16 a0 = lds16(G + (cc * 64 + 16 * (mg * MT + mm) + l16) * GST);
-                    const h16 a1 = lds16(G + (cc * 64 + 16 * (mg * MT + mm) + l16) * GST + 16);
+                    const gop a0 = lda(G + (cc * 64 + 16 * (mg * MT + mm) + l16) * GST);
+                    const gop a1 = lda(G + (cc * 64 + 16 * (mg * MT + mm) + l16) * GST + 16);
 #pragma unroll
                     for (int j = 0; j < NTW; ++j)
                         k32(x[mm][j], a0, b[0][j], a1, b[1][j]);
@@ -609,7 +664,10 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
     }
 
     // ------------------------------------------------ output: f16 staged in LDS, 16-value segments encoded by a loop
-    half_t* XO = A; // [64 tokens][C channels] natural order
+    // XO: [64 tokens][C channels] natural order; the row padding keeps the patch merge's 8-byte reads of 16
+    // tokens on distinct LDS banks (see xo_row)
+    half_t* XO = A;
+    static_assert(xo_row<C>(63) + C <= 64 * AST, "XO fits in A");
 #pragma unroll
     for (int j = 0; j < NTW; ++j)
     {
@@ -618,26 +676,28 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
         for (int mm = 0; mm < MT; ++mm)
 #pragma unroll
             for (int i = 0; i < 8; ++i)
-                XO[(16 * (mg * MT + mm) + 2 * i + hi) * C + ch] = x[mm][j].get(i);
+                XO[xo_row<C>(16 * (mg * MT + mm) + mrow(i)) + ch] = x[mm][j].get(i);
     }
     __syncthreads();
-    uint8_t* OB = U; // e4m3 codes [64][C] for the patch merge
 #pragma unroll 1
     for (int seg = tid; seg < 64 * C / 16; seg += 32 * NW)
     {
         const int T = seg / (C / 16), c16 = 16 * (seg % (C / 16));
+        // rows are 8-byte aligned (xo_row), so the segment moves as four 8-byte words of two f16 pairs
+        uint2* xs = (uint2*)(XO + xo_row<C>(T) + c16);
         uint32_t w[4];
 #pragma unroll
         for (int k = 0; k < 4; ++k)
         {
-            w[k] = 0;
-#pragma unroll
-            for (int b = 0; b < 4; ++b)
-                w[k] |= enc8(XO[T * C + c16 + 4 * k + b]) << (8 * b);
+            const uint2 pr = xs[k];
+            hv2 qa, qb;
+            const uint32_t ca = enc8x2(__builtin_bit_cast(hv2, pr.x), qa), cb = enc8x2(__builtin_bit_cast(hv2, pr.y), qb);
+            w[k] = pack_codes(ca, cb);
+            // the patch merge consumes decode(code) = q8_exact(x), written back over x (this thread's own values)
+            if constexpr (NPM > 0)
+                xs[k] = (uint2){__builtin_bit_cast(uint32_t, qa), __builtin_bit_cast(uint32_t, qb)};
         }
         const uint4 v = {w[0], w[1], w[2], w[3]};
-        if constexpr (NPM > 0)
-            *(uint4*)(OB + T * C + c16) = v;
         const int X = 8 * bx - p.sx + tok_x(T), Y = 8 * by - p.sy + tok_y(T);
         if (X >= 0 && X < p.tw && Y >= 0 && Y < p.th)
             *(uint4*)(p.out + ((size_t)((c16 >> 5) * p.th + Y) * p.tw + X) * 32 + (c16 & 31)) = v;
@@ -663,23 +723,28 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
         for (int kc = 0; kc < C / 8; ++kc)
         {
-            h16 as[2];
+            gop as[2];
 #pragma unroll
             for (int s = 0; s < 2; ++s)
             {
-                h16& a = as[s];
+                sop a;
+                // the operand's K slots in runs of 4 (gfx12: this half's slots 8 hi .. 8 hi + 7)
+                constexpr int RUNS = sizeof(sop) / sizeof(half_t) / 4;
 #pragma unroll
-                for (int run = 0; run < 4; ++run)
+                for (int rr = 0; rr < RUNS; ++rr)
                 {
+                    const int run = RUNS == 4 ? rr : 2 * hi + rr;
                     const int k = 32 * kc + kslot(s, 4 * run);
                     const int q = k / C, ch = k % C;
                     const int tx = 2 * mx + (q & 1), ty = 2 * my + (q >> 1);
                     const int T = (tx & 3) + 4 * (ty & 3) + 16 * (tx >> 2) + 32 * (ty >> 2);
-                    const uint32_t bytes = *(const uint32_t*)(OB + T * C + ch);
+                    typedef _Float16 h4 __attribute__((ext_vector_type(4)));
+                    const h4 v = *(const h4*)(XO + xo_row<C>(T) + ch);
 #pragma unroll
                     for (int b = 0; b < 4; ++b)
-                        a[4 * run + b] = e4m3_to_half((bytes >> (8 * b)) & 0xffu);
+                        a[4 * rr + b] = v[b];
                 }
+                as[s] = as_gop(a);
             }
 #pragma unroll
             for (int j = 0; j < NTP; ++j)
@@ -700,12 +765,21 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #pragma unroll
             for (int i = 0; i < 8; ++i)
             {
-                const int r = 2 * i + hi;
+                const int r = mrow(i);
                 const int MX = (8 * bx - p.sx) / 2 + (r & 3), MY = (8 * by - p.sy) / 2 + (r >> 2);
                 if (MX < 0 || MX >= W2 || MY < 0 || MY >= H2)
                     continue;
                 p.p48[((size_t)((ch >> 5) * H2 + MY) * W2 + MX) * 32 + (ch & 31)] =
-                    (uint8_t)enc8(y[j].get(i));
+                    (uint8_t)enc8(
+#ifdef SWIN_EXACT
+                        // The audited 310.9 patch-merge path bounds the f16
+                        // result to +/- 2*pi before e4m3 quantisation.
+                        __builtin_elementwise_max(__builtin_elementwise_min(y[j].get(i), (half_t)6.283185307179586f),
+                                                  (half_t)-6.283185307179586f)
+#else
+                        y[j].get(i)
+#endif
+                    );
             }
         }
     }
@@ -734,35 +808,79 @@ __device__ __forceinline__ void swin_block(const CommonParams& p, const TubePara
 #ifndef SWIN_PERSIST
 #define SWIN_PERSIST 0 // persistent workgroups in x (0: one workgroup per original block)
 #endif
-// Module boilerplate: f16 weight image, prep kernel and its grid for the ZLUDA override hook.
+#ifndef SWIN_PREP_SLOTS
+#define SWIN_PREP_SLOTS 0 // weight images kept per weights pointer (0: one image, re-prepared when the pointer changes)
+#endif
+
+// Weight-image slots for kernels launched with several weight sets (the six enc3 tube blocks). The ZLUDA
+// hook runs the prep only for a weights pointer it has not seen (d4r_prep_key_at / d4r_prep_key_slots);
+// the prep claims the first free slot for that pointer, and pointers beyond S share the overflow slot S,
+// which the hook re-prepares on every launch. Keys are written by one thread of the prep; any prep
+// thread that reads before that write finds the same (first free) slot.
+template <int S> __device__ __forceinline__ int prep_slot_find(const uint64_t* keys, uint64_t w)
+{
+    for (int i = 0; i < S; ++i)
+        if (__hip_atomic_load(keys + i, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT) == w)
+            return i;
+    return S;
+}
+template <int S> __device__ __forceinline__ int prep_slot_claim(uint64_t* keys, uint64_t w, bool writer)
+{
+    // one pass: the first slot holding w or nothing. Keys only ever go from 0 to a pointer, so every
+    // prep thread picks the same slot whether it reads that slot before or after the writer's store.
+    for (int i = 0; i < S; ++i)
+    {
+        const uint64_t k = __hip_atomic_load(keys + i, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+        if (k == w)
+            return i;
+        if (k == 0)
+        {
+            if (writer)
+                __hip_atomic_store(keys + i, w, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
+            return i;
+        }
+    }
+    return S;
+}
+
+// Module boilerplate: f16 weight image(s), prep kernel and its grid for the ZLUDA override hook.
 #define SWIN_MODULE(NAME, C, NW, NPM, TUBE, PARAMS, CIN) SWIN_MODULE_W(NAME, C, NW, NW, NPM, TUBE, PARAMS, CIN)
 // NH heads on NWAVES waves; d4r_block_z tells the ZLUDA hook the block z size to launch with
 #define SWIN_MODULE_W(NAME, C, NH, NWAVES, NPM, TUBE, PARAMS, CIN)                                                      \
     using NAME##_L = SwinLayout<C, NH, NPM, CIN>;                                                                       \
-    __device__ h16 g_w16[NAME##_L::TOTAL];                                                                              \
+    constexpr int NAME##_S = SWIN_PREP_SLOTS;                                                                           \
+    __device__ wslot g_w16[(NAME##_S + 1) * NAME##_L::TOTAL];                                                           \
+    __device__ uint64_t g_prep_keys[NAME##_S + 1];                                                                      \
     __constant__ SwinDescs<C, NH, NPM, CIN> g_descs = make_descs<C, NH, NPM, CIN>();                                    \
     extern "C" __device__ __attribute__((used)) uint32_t d4r_block_z = NWAVES;                                         \
     extern "C" __device__ __attribute__((used)) uint32_t d4r_prep_blocks = (NAME##_L::TOTAL + 127) / 128;              \
+    /* the prep reads only p.w (offset 0): skip it while the image for that pointer is resident */                     \
+    extern "C" __device__ __attribute__((used)) uint32_t d4r_prep_key_at = 0 + 1;                                      \
+    extern "C" __device__ __attribute__((used)) uint32_t d4r_prep_key_slots = NAME##_S;                                \
     extern "C" __global__ void __launch_bounds__(128) NAME##_prep(PARAMS p)                                             \
     {                                                                                                                   \
         const int idx = blockIdx.x * 128 + threadIdx.x;                                                                 \
+        const uint8_t* w = ((const CommonParams*)&p)->w;                                                                \
+        const int slot = NAME##_S ? prep_slot_claim<NAME##_S>(g_prep_keys, (uint64_t)w, idx == 0) : 0;                 \
         if (idx < NAME##_L::TOTAL)                                                                                      \
-            expand_weights(((const CommonParams*)&p)->w, g_w16, g_descs.d, NAME##_L::NDESC, idx);                      \
+            expand_weights(w, g_w16 + slot * NAME##_L::TOTAL, g_descs.d, NAME##_L::NDESC, idx);                        \
     }                                                                                                                   \
     extern "C" __device__ __attribute__((used)) uint32_t d4r_grid_x = SWIN_PERSIST;                                     \
     extern "C" __global__ void __launch_bounds__(32 * NWAVES) SWIN_VGPR_ATTR NAME(PARAMS p)                             \
     {                                                                                                                   \
         const CommonParams& cp = *(const CommonParams*)&p;                                                              \
+        const int slot = NAME##_S ? prep_slot_find<NAME##_S>(g_prep_keys, (uint64_t)cp.w) : 0;                         \
+        const wslot* w16 = g_w16 + slot * NAME##_L::TOTAL;                                                              \
         /* the original grid: 8x8-token blocks covering the shifted token grid */                                        \
         const int gx = (cp.tw + cp.sx + 7) / 8, gy = (cp.th + cp.sy + 7) / 8;                                           \
         if (SWIN_PERSIST == 0)                                                                                          \
         {                                                                                                               \
-            swin_block<C, NH, NWAVES, NPM, TUBE, CIN>(cp, (const TubeParams*)&p, g_w16, blockIdx.x, blockIdx.y, gx);    \
+            swin_block<C, NH, NWAVES, NPM, TUBE, CIN>(cp, (const TubeParams*)&p, w16, blockIdx.x, blockIdx.y, gx);      \
             return;                                                                                                     \
         }                                                                                                               \
         _Pragma("unroll 1") for (int b = blockIdx.x; b < gx * gy; b += gridDim.x)                                      \
         {                                                                                                               \
-            swin_block<C, NH, NWAVES, NPM, TUBE, CIN>(cp, (const TubeParams*)&p, g_w16, b % gx, b / gx, gx);            \
+            swin_block<C, NH, NWAVES, NPM, TUBE, CIN>(cp, (const TubeParams*)&p, w16, b % gx, b / gx, gx);              \
             __syncthreads();                                                                                            \
         }                                                                                                               \
     }

kernels/m/swin_common.h

@@ -1,8 +1,36 @@
-// Shared device code for native RDNA3 (gfx11 wave32 WMMA) rrlite Swin-block kernels (DLSS 310.7.0).
+// Shared device code for native rrlite Swin-block kernels (DLSS 310.7.0; gfx11 / gfx12 wave32 WMMA).
 // Semantics: ../ENC3_SPEC.md generalised (numpy reference ../swin_model.py).
+//
+// Activations are the WMMA A operand (lane = token row), weights the B operand (lane = channel column), and
+// the f32 result has lane = channel column, VGPR i = token row wm_acc_row(i) (../common/wmma_layout.h).
+// D4R_FP8_WMMA (gfx12 layout only): the GEMMs whose operands are both e4m3 values run as native FP8 WMMAs on
+// e4m3 bytes (weights prepared as bytes, activations encoded when loaded); P V stays f16.
 #pragma once
+#ifdef D4R_DEVICE_ONLY_MINIMAL
+#include "../common/hip_device_minimal.h"
+#else
 #include <hip/hip_runtime.h>
+#endif
 #include <stdint.h>
+#include "../common/wmma_layout.h"
+#ifdef D4R_ACCURACY
+#define SWIN_EXACT
+#endif
+#if defined(D4R_FP8_WMMA) && D4R_WMMA_LAYOUT != 12 && defined(__AMDGCN__)
+#error "D4R_FP8_WMMA needs the gfx12 WMMA layout"
+#endif
+// FP8 operands must hold e4m3 values, so the FP8 build re-quantises the intermediate activations as NVIDIA's
+// network does (the gfx11 "fast numerics" SWIN_NO_Q8 keeps them f16, which no gfx12 WMMA can multiply with
+// e4m3 weights)
+// (SWIN_FORCE_Q8 does the same on any target: the gfx11 reference of the FP8 build's numerics)
+#if (defined(D4R_FP8_WMMA) || defined(SWIN_FORCE_Q8) || defined(SWIN_EXACT)) && defined(SWIN_NO_Q8)
+#undef SWIN_NO_Q8
+#endif
+// SWIN_EXACT (tests): NVIDIA's numerics throughout, the semantics of tools/swin_model.py (e4m3 activations and
+// an f16 accumulator rounded after every k32 step)
+#if defined(SWIN_EXACT) && defined(SWIN_F32ACC)
+#undef SWIN_F32ACC
+#endif
 
 #pragma clang fp contract(off)
 
@@ -82,12 +110,9 @@ typedef _Float16 hv2 __attribute__((ext_vector_type(2)));
 typedef unsigned short u16x2 __attribute__((ext_vector_type(2)));
 typedef short i16x2 __attribute__((ext_vector_type(2)));
 
-// q8 on two halves with packed 16-bit integer / f16 ops (same per-half logic as q8; no carries cross halves)
-__device__ __forceinline__ hv2 q8x2(hv2 h)
+// q8_exact on two halves with packed 16-bit integer / f16 ops (same per-half logic; no carries cross halves)
+__device__ __forceinline__ hv2 q8x2_exact(hv2 h)
 {
-#ifdef SWIN_NO_Q8
-    return h;
-#endif
     const u16x2 u = __builtin_bit_cast(u16x2, h);
     const u16x2 m = u & (unsigned short)0x7fff;
     u16x2 r = (m + (unsigned short)0x3f + ((m >> 7) & (unsigned short)1)) & (unsigned short)0x7f80;
@@ -102,6 +127,48 @@ __device__ __forceinline__ hv2 q8x2(hv2 h)
     return __builtin_bit_cast(hv2, (u16x2)(r | (u & (unsigned short)0x8000)));
 }
 
+__device__ __forceinline__ hv2 q8x2(hv2 h)
+{
+#ifdef SWIN_NO_Q8
+    return h;
+#endif
+    return q8x2_exact(h);
+}
+
+// enc8 of two halves with packed ops: returns the two e4m3 codes in the low bytes of the 16-bit
+// halves, and q = q8_exact(h) (the values the codes decode to)
+__device__ __forceinline__ uint32_t codes8x2(hv2 q);
+__device__ __forceinline__ uint32_t enc8x2(hv2 h, hv2& q)
+{
+    q = q8x2_exact(h);
+    return codes8x2(q);
+}
+
+// the e4m3 codes of two halves that already hold e4m3 values (q8_exact results), in the low bytes of the
+// 16-bit halves
+__device__ __forceinline__ uint32_t codes8x2(hv2 q)
+{
+    const u16x2 qb = __builtin_bit_cast(u16x2, q);
+    const u16x2 m = qb & (unsigned short)0x7fff;
+    // normals: the exponent moves from bias 15 to bias 7
+    const u16x2 norm = (u16x2)(m >> 7) - (unsigned short)64;
+    // subnormals are k * 2^-9 (k = 0..7): f16(k + 1024) has k in its low bits
+    const hv2 mag = __builtin_bit_cast(hv2, m);
+    const hv2 k = (hv2)(mag * (hv2){(half_t)512.0f, (half_t)512.0f}) + (hv2){(half_t)1024.0f, (half_t)1024.0f};
+    const u16x2 sub = __builtin_bit_cast(u16x2, k) - (unsigned short)0x6400;
+    const u16x2 lt = __builtin_bit_cast(u16x2, (i16x2)(__builtin_bit_cast(i16x2, (u16x2)(m - (unsigned short)0x2400)) >> 15));
+    u16x2 code = (sub & lt) | (norm & ~lt);
+    const u16x2 gt = __builtin_bit_cast(u16x2, (i16x2)(__builtin_bit_cast(i16x2, (u16x2)((unsigned short)0x7c00 - m)) >> 15));
+    code = ((u16x2){0x7f, 0x7f} & gt) | (code & ~gt);
+    return __builtin_bit_cast(uint32_t, (u16x2)(((qb >> 8) & (unsigned short)0x80) | code));
+}
+
+// four e4m3 codes (a, b from enc8x2) as the bytes of one word in element order
+__device__ __forceinline__ uint32_t pack_codes(uint32_t a, uint32_t b)
+{
+    return __builtin_amdgcn_perm(b, a, 0x06040200u);
+}
+
 // e4m3 code of h (RNE, satfinite) derived from q8(h): normals shift the exponent, subnormals are k * 2^-9
 __device__ __forceinline__ uint32_t enc8(half_t h)
 {
@@ -153,9 +220,119 @@ __device__ __forceinline__ int woff(int base, int Ks, int Ns, int k, int n)
            4 * ((k & 31) >> 4) + (k & 3);
 }
 
-__device__ __forceinline__ f8v wmma(h16 a, h16 b, f8v c)
+// ---------------------------------------------------------------- WMMA operands
+// half-wave of this lane: blocks are 32 x 1 x waves, so the lane is threadIdx.x (the same value as wm_half(),
+// but from the source the kernels already use for their lane index)
+__device__ __forceinline__ int m_half()
 {
+    return (int)threadIdx.x >> 4;
+}
+// token row that accumulator VGPR i holds in this lane
+__device__ __forceinline__ int mrow(int i)
+{
+    return wm_acc_row_h(i, m_half());
+}
+// sop: an f16 operand (gfx11: the row's 16 K values; gfx12: this half's 8). gop: an operand of the e4m3 GEMMs
+// (k32 steps): sop, or with D4R_FP8_WMMA the 8 e4m3 bytes of this half's K values.
+#if D4R_WMMA_LAYOUT == 12
+typedef h8 sop;
+#else
+typedef h16 sop;
+#endif
+typedef uint32_t u2v __attribute__((ext_vector_type(2)));
+typedef uint32_t u4v_t __attribute__((ext_vector_type(4)));
+#ifdef D4R_FP8_WMMA
+typedef u2v gop;
+typedef u4v_t wslot; // weight image slot: 16 e4m3 bytes (K slots 0..15)
+#else
+typedef sop gop;
+typedef h16 wslot; // weight image slot: 16 f16 values
+#endif
+
+__device__ __forceinline__ f8v wmma(sop a, sop b, f8v c)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return wm_mma(__builtin_bit_cast(wm_op, a), __builtin_bit_cast(wm_op, b), c);
+#else
     return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#endif
+}
+
+// f16 operand from 16 contiguous halves in LDS (gfx12: this half's 8 of them)
+__device__ __forceinline__ sop lds16(const half_t* p)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return *(const h8*)(p + 8 * m_half());
+#else
+    h8 lo = *(const h8*)p, hi = *(const h8*)(p + 8);
+    return __builtin_shufflevector(lo, hi, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
+#endif
+}
+
+#ifdef D4R_FP8_WMMA
+// e4m3 bytes of an f16 operand whose values are e4m3 values (K order kept: byte j = element j)
+__device__ __forceinline__ gop to_fp8(sop v)
+{
+#if defined(D4R_FP8_HW_PACK) && defined(__GFX12__)
+    // Inputs already contain exact e4m3 values. gfx12's native packed convert
+    // avoids the software subnormal/sign/exponent reconstruction below. Keep
+    // opt-in until all finite encodings and layer/network outputs pass replay.
+    gop result;
+#pragma unroll
+    for (int word = 0; word < 2; ++word) {
+        int packed = __builtin_amdgcn_cvt_pk_fp8_f32((float)v[4 * word], (float)v[4 * word + 1], 0, false);
+        packed = __builtin_amdgcn_cvt_pk_fp8_f32((float)v[4 * word + 2], (float)v[4 * word + 3], packed, true);
+        result[word] = (uint32_t)packed;
+    }
+    return result;
+#else
+    uint32_t c[4];
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+        c[j] = codes8x2((hv2){v[2 * j], v[2 * j + 1]});
+    return (u2v){__builtin_amdgcn_perm(c[1], c[0], 0x06040200u), __builtin_amdgcn_perm(c[3], c[2], 0x06040200u)};
+#endif
+}
+// f32 += A B over 16 K for e4m3 operands
+__device__ __forceinline__ f8v wmma8(gop a, gop b, f8v c)
+{
+#if defined(__GFX12__)
+    return __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12(__builtin_bit_cast(wm_i2v, a), __builtin_bit_cast(wm_i2v, b), c);
+#else
+    // layout shim: the e4m3 values widened to f16 (exact), then the f16 WMMA
+    auto widen = [](gop q) {
+        sop r;
+#pragma unroll
+        for (int j = 0; j < 8; ++j)
+            r[j] = e4m3_to_half((q[j >> 2] >> (8 * (j & 3))) & 0xffu);
+        return r;
+    };
+    return wmma(widen(a), widen(b), c);
+#endif
+}
+__device__ __forceinline__ gop lda(const half_t* p)
+{
+    return to_fp8(lds16(p));
+}
+#else
+__device__ __forceinline__ f8v wmma8(gop a, gop b, f8v c)
+{
+    return wmma(a, b, c);
+}
+// GEMM A operand (activations) from LDS
+__device__ __forceinline__ gop lda(const half_t* p)
+{
+    return lds16(p);
+}
+#endif
+// an f16 operand built in registers -> GEMM operand
+__device__ __forceinline__ gop as_gop(sop v)
+{
+#ifdef D4R_FP8_WMMA
+    return to_fp8(v);
+#else
+    return v;
+#endif
 }
 
 __device__ __forceinline__ f8v round16(f8v v)
@@ -166,7 +343,7 @@ __device__ __forceinline__ f8v round16(f8v v)
     return v;
 }
 
-// A C-fragment tile: element i is row 2i + lane/16. Exact mode stores packed f16 (NVIDIA's f16
+// A C-fragment tile: element i is row wm_acc_row(i). Exact mode stores packed f16 (NVIDIA's f16
 // accumulator, rounded after every k32 step); SWIN_F32ACC keeps f32 through whole GEMM chains
 // (more precise than NVIDIA; values are rounded to f16 where they are consumed).
 struct T16
@@ -217,20 +394,20 @@ __device__ __forceinline__ void mixacc(T16& acc, f8v P)
 __device__ unsigned d4r_k32_bad[4 + 64 * 4];
 #endif
 // f16 accumulate of one k32 step (NVIDIA m16n8k32 semantics: products summed, one rounding with C)
-__device__ __forceinline__ void k32(T16& acc, h16 a0, h16 b0, h16 a1, h16 b1)
+__device__ __forceinline__ void k32(T16& acc, gop a0, gop b0, gop a1, gop b1)
 {
 #if defined(SWIN_F32ACC)
-    acc.f = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a1, b1, __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a0, b0, acc.f));
+    acc.f = wmma8(a1, b1, wmma8(a0, b0, acc.f));
 #elif defined(K32_MIX)
-    f8v P = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a0, b0, (f8v){0, 0, 0, 0, 0, 0, 0, 0});
-    P = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a1, b1, P);
+    f8v P = wmma8(a0, b0, (f8v){0, 0, 0, 0, 0, 0, 0, 0});
+    P = wmma8(a1, b1, P);
     mixacc(acc, P);
 #else // exact: C added inside the f32 WMMA, f16 rounding after the k32 step
     f8v c;
 #pragma unroll
     for (int i = 0; i < 8; ++i)
         c[i] = (float)acc.get(i);
-    c = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a1, b1, __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a0, b0, c));
+    c = wmma8(a1, b1, wmma8(a0, b0, c));
 #pragma unroll
     for (int i = 0; i < 8; ++i)
         acc.set(i, (half_t)c[i]);
@@ -273,12 +450,6 @@ __device__ __forceinline__ half_t hload(const uint8_t* w, int off)
     return __builtin_bit_cast(half_t, *(const uint16_t*)(w + off));
 }
 
-__device__ __forceinline__ h16 lds16(const half_t* p)
-{
-    h8 lo = *(const h8*)p, hi = *(const h8*)(p + 8);
-    return __builtin_shufflevector(lo, hi, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
-}
-
 __device__ __forceinline__ void wave_sync()
 {
     __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
@@ -296,21 +467,36 @@ __device__ __forceinline__ int tok_y(int T)
     return ((T >> 2) & 3) + 4 * (T >> 5);
 }
 
-// ---------------------------------------------------------------- f16 weight images
-// A GEMM's B operands expanded to f16 in WMMA lane order: h16 index
-// dst + ((kc * 2 + s) * NT + nt) * 16 + lane16, holding logical k = 32 kc + kslot(s, i), n = 16 nt + lane16.
+// ---------------------------------------------------------------- weight images
+// A GEMM's B operands in WMMA lane order: slot index dst + ((kc * 2 + s) * NT + nt) * 16 + lane16 holds the 16
+// K slots of logical k = 32 kc + kslot(s, i), n = 16 nt + lane16 (i = 0..15), as f16 values (wslot = h16) or,
+// with D4R_FP8_WMMA, as the e4m3 bytes. gfx12 lanes read the half of the slot with their 8 K slots.
 struct GemmDesc
 {
-    int dst; // in h16 units
+    int dst; // in slots
     int base, Ks, Ns, KC, NT;
 };
 
-__device__ __forceinline__ h16 bload(const h16* w16, int gemm_dst, int NT, int kc, int s, int nt, int l16)
+// The tile index is wave-uniform (it depends only on the wave, the loop counters and constants), so it
+// is formed in scalar registers and the load uses a scalar base plus the lane's constant offset.
+__device__ __forceinline__ gop bload(const wslot* w16, int gemm_dst, int NT, int kc, int s, int nt, int l16)
 {
-    return w16[gemm_dst + ((kc * 2 + s) * NT + nt) * 16 + l16];
+#ifdef SWIN_SCALAR_BLOAD
+    const int tile = __builtin_amdgcn_readfirstlane(gemm_dst + ((kc * 2 + s) * NT + nt) * 16);
+    const wslot* t = w16 + tile;
+    const int at = l16;
+#else
+    const wslot* t = w16;
+    const int at = gemm_dst + ((kc * 2 + s) * NT + nt) * 16 + l16;
+#endif
+#if D4R_WMMA_LAYOUT == 12
+    return ((const gop*)t)[2 * at + m_half()];
+#else
+    return t[at];
+#endif
 }
 
-__device__ __forceinline__ void expand_weights(const uint8_t* w, h16* w16, const GemmDesc* descs, int ndesc, int idx)
+__device__ __forceinline__ void expand_weights(const uint8_t* w, wslot* w16, const GemmDesc* descs, int ndesc, int idx)
 {
     int d = 0;
     while (d + 1 < ndesc && idx >= descs[d + 1].dst)
@@ -322,9 +508,22 @@ __device__ __forceinline__ void expand_weights(const uint8_t* w, h16* w16, const
     const int l16 = local & 15, rest = local >> 4;
     const int nt = rest % g.NT, s = (rest / g.NT) & 1, kc = rest / g.NT / 2;
     const int n = 16 * nt + l16;
+#ifdef D4R_FP8_WMMA
+    u4v_t v;
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+    {
+        uint32_t word = 0;
+#pragma unroll
+        for (int b = 0; b < 4; ++b)
+            word |= (uint32_t)w[woff(g.base, g.Ks, g.Ns, 32 * kc + kslot(s, 4 * j + b), n)] << (8 * b);
+        v[j] = word;
+    }
+#else
     h16 v;
 #pragma unroll
     for (int i = 0; i < 16; ++i)
         v[i] = e4m3_to_half(w[woff(g.base, g.Ks, g.Ns, 32 * kc + kslot(s, i), n)]);
+#endif
     w16[idx] = v;
 }

kernels/support/motion_dilate.hip

@@ -0,0 +1,64 @@
+#include "../../tools/d4r_motion_dilation.h"
+#pragma clang fp contract(off)
+
+__attribute__((device)) static int clamp_index(int x, uint32_t limit)
+{
+    return x < 0 ? 0 : x >= (int)limit ? (int)limit - 1 : x;
+}
+
+extern "C" __attribute__((global, amdgpu_flat_work_group_size(128, 128)))
+void d4r_motion_dilate(D4rMotionDilationParams p)
+{
+    const uint32_t id = __builtin_amdgcn_workgroup_id_x() * 128 + __builtin_amdgcn_workitem_id_x();
+    if ((uint64_t)id >= (uint64_t)p.motion_width * p.motion_height)
+        return;
+    const uint32_t x = id % p.motion_width, y = id / p.motion_width;
+    const int cx = clamp_index((int)__builtin_floorf((x + 0.5f) * p.render_width / p.motion_width + p.jitter_x), p.render_width);
+    const int cy = clamp_index((int)__builtin_floorf((y + 0.5f) * p.render_height / p.motion_height + p.jitter_y), p.render_height);
+    const uint8_t* depth = (const uint8_t*)p.depth;
+    const uint8_t* motion = (const uint8_t*)p.motion;
+    uint8_t* output = (uint8_t*)p.output;
+    const float current = *(const float*)(depth + (uint64_t)cy * p.depth_pitch + 4 * cx);
+    float best = current;
+    int bx = cx, by = cy;
+    for (int dy = -(int)p.radius; dy <= (int)p.radius; ++dy)
+        for (int dx = -(int)p.radius; dx <= (int)p.radius; ++dx)
+        {
+            const int nx = clamp_index(cx + dx, p.render_width), ny = clamp_index(cy + dy, p.render_height);
+            const float candidate = *(const float*)(depth + (uint64_t)ny * p.depth_pitch + 4 * nx);
+            if (p.reversed_depth ? candidate > best : candidate < best)
+                best = candidate, bx = nx, by = ny;
+        }
+    // Sky gaps need wider coverage than opaque edges. Select the closest
+    // foreground pixel within eight render pixels, rather than the closest
+    // depth (which can borrow a distant roof/chimney's velocity).
+    const bool sky = p.reversed_depth ? current <= 1.e-7f : current >= 1.0f - 1.e-7f;
+    bool sky_filled = false;
+    if (sky)
+    {
+        int best_distance = 65;
+        for (int dy = -8; dy <= 8; ++dy)
+            for (int dx = -8; dx <= 8; ++dx)
+            {
+                const int nx = cx + dx, ny = cy + dy;
+                const int distance = dx * dx + dy * dy;
+                if (nx < 0 || ny < 0 || nx >= (int)p.render_width || ny >= (int)p.render_height ||
+                    distance > 64 || distance > best_distance)
+                    continue;
+                const float candidate = *(const float*)(depth + (uint64_t)ny * p.depth_pitch + 4 * nx);
+                const bool foreground = p.reversed_depth ? candidate > 1.e-7f : candidate < 1.0f - 1.e-7f;
+                // Stable coordinate tie-break matches nearest-foreground fill.
+                if (foreground && (distance < best_distance || nx < bx || (nx == bx && ny < by)))
+                    best_distance = distance, best = candidate, bx = nx, by = ny, sky_filled = true;
+            }
+    }
+    uint32_t value = *(const uint32_t*)(motion + (uint64_t)y * p.motion_pitch + 4 * x);
+    const bool nearer = p.reversed_depth ? best > current * 1.02f + 1.e-7f : best < current * 0.98f - 1.e-7f;
+    if (sky_filled || nearer)
+    {
+        const int mx = clamp_index((int)__builtin_floorf((bx + 0.5f - p.jitter_x) * p.motion_width / p.render_width), p.motion_width);
+        const int my = clamp_index((int)__builtin_floorf((by + 0.5f - p.jitter_y) * p.motion_height / p.render_height), p.motion_height);
+        value = *(const uint32_t*)(motion + (uint64_t)my * p.motion_pitch + 4 * mx);
+    }
+    *(uint32_t*)(output + (uint64_t)y * p.motion_pitch + 4 * x) = value;
+}

kernels/tex/build_tex.sh

@@ -2,25 +2,36 @@
 # build_tex.sh KERNEL SRC OUT_DIR: one texture kernel. NVIDIA's PTX for KERNEL, edited by make_ptx.py so
 # part of it calls functions from SRC.hip, is compiled by ZLUDA with SRC's bitcode linked in
 # (D4R_ZLUDA_EXTRA_BC) and the resulting code object is saved as OUT_DIR/KERNEL.hsaco.
-# Called by kernels/build.sh, which sets D4R_ROCM_DIR, D4R_GPU_ARCH, D4R_DLSS_PTX_DIR, D4R_PTXLOAD and
-# needs D4R_ZLUDA_BUILD.
+# Called by kernels/build.sh, which sets D4R_ROCM_DIR, D4R_GPU_ARCH, D4R_DLSS_PTX_DIR, D4R_ZLUDA_EMIT,
+# D4R_ZLUDA_WAVE64 (1 = wave64 code object) and D4R_TEX_FP8 (1 = the gfx12 native-FP8 variant).
+# ZLUDA compiles offline for D4R_GPU_ARCH with its d4r_emit tool, so any target builds on any machine.
 set -euo pipefail
 K=$1 SRC=$2 OUT=$3
 D="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 R="${D4R_ROCM_DIR:-/opt/rocm}"
 ARCH="${D4R_GPU_ARCH:-gfx1101}"
+FP8="${D4R_TEX_FP8:-0}"
+ACCURACY="${D4R_PREFER_ACCURACY:-0}"
 LLVM="$R/lib/llvm/bin"
 B="$(mktemp -d)"
 trap 'rm -rf "$B"' EXIT
+# D4R_TEX_CFLAGS: extra flags for SRC.hip (tests, e.g. -DD4R_WMMA_LAYOUT=12 for the gfx12-layout shim on gfx11)
+read -r -a FLAGS <<< "${D4R_TEX_CFLAGS:-}"
+[[ "$FP8" == 1 ]] && FLAGS+=(-DD4R_TEX_FP8)
+[[ "$ACCURACY" == 1 ]] && FLAGS+=(-DD4R_ACCURACY)
 # SRC.hip -> bitcode without target attributes, so ZLUDA's own target settings apply after linking
 "$LLVM/clang++" -x hip -std=c++20 -nogpuinc -nogpulib -O3 -mno-wavefrontsize64 --offload-device-only \
-    --offload-arch="$ARCH" -fgpu-rdc -emit-llvm -c -Xclang -fdenormal-fp-math=dynamic -o "$B/raw.bc" "$D/$SRC.hip"
+    --offload-arch="$ARCH" -fgpu-rdc -emit-llvm -c -Xclang -fdenormal-fp-math=dynamic -DD4R_KERNEL_NAME="$K" \
+    "${FLAGS[@]}" -o "$B/raw.bc" "$D/$SRC.hip"
 "$LLVM/llvm-dis" "$B/raw.bc" -o "$B/raw.ll"
 sed -E -e '/@llvm.used/d' -e '/wchar_size/d' -e '/llvm.module.flags/d' -e '/__hip_cuid/d' -e 's/optnone//g' \
     -e "s/\"target-cpu\"=\"$ARCH\"//g" -e 's/"target-features"="[^"]+"//g' "$B/raw.ll" | "$LLVM/llvm-as" -o "$B/extra.bc" -
-python3 "$D/make_ptx.py" "$K" "$B/$K.ptx" >/dev/null
-env LD_LIBRARY_PATH="$D4R_ZLUDA_BUILD:$R/lib" XDG_CACHE_HOME="$B/jit" D4R_ZLUDA_DUMP_DIR="$B/dump" \
-    D4R_ZLUDA_EXTRA_BC="$B/extra.bc" D4R_ZLUDA_WMMA=1 D4R_ZLUDA_WMMA_FP8=1 D4R_ZLUDA_IGNORE_DENORMAL=1 \
-    D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 "$D4R_PTXLOAD" "$B/$K.ptx" >/dev/null
+D4R_PREFER_ACCURACY="$ACCURACY" python3 "$D/make_ptx.py" "$K" "$B/$K.ptx" >/dev/null
+# the code-generation switches of the runtime (d4r.ini [Kernels]), which ZLUDA's module cache keys on
+env LD_LIBRARY_PATH="$R/lib" D4R_ZLUDA_EXTRA_BC="$B/extra.bc" D4R_ZLUDA_WMMA=1 D4R_ZLUDA_WMMA_FP8=1 \
+    D4R_ZLUDA_WMMA_FP8_NATIVE="$FP8" D4R_ZLUDA_IGNORE_DENORMAL="$((1 - ACCURACY))" \
+    D4R_ZLUDA_WMMA_F32ACC=0 D4R_ZLUDA_FAST_MATH=0 D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 \
+    D4R_ZLUDA_WAVE64="$([[ "$ACCURACY" == 1 ]] && echo 0 || echo "${D4R_ZLUDA_WAVE64:-0}")" \
+    "$D4R_ZLUDA_EMIT" "$B/$K.ptx" "$B/dump" "$ARCH" >/dev/null
 cp "$B/dump/module.hsaco" "$OUT/$K.hsaco"
-printf '%-52s %s\n' "$K" "$(grep -E '^\s+\.(vgpr_count|private_segment_fixed_size):' "$B/dump/asm.s" | tr -s ' ' | paste -sd' ')"
+printf '%-52s %s\n' "$K" "$(grep -E '^\s+\.(vgpr_count|private_segment_fixed_size|wavefront_size):' "$B/dump/asm.s" | tr -s ' ' | paste -sd' ')"

kernels/tex/dec0_head.hip

@@ -6,8 +6,14 @@
 
 // weights as WMMA A images: index ((p * 2 + c) * 2 + h) * 16 + n16 for channel pair p (16p..16p+15),
 // k32 chunk c (input plane), k16 half h
+#if D4R_WMMA_LAYOUT == 12
+__attribute__((device)) kslot g_dec0_w16[8 * 2 * 2 * 16];
+#else
 __attribute__((device)) u8v g_dec0_w16[8 * 2 * 2 * 16];
+#endif
 DEV __attribute__((used)) uint32_t d4r_prep_blocks = 4;
+// the prep reads only prm.w (offset 0): the hook skips it while that pointer repeats
+DEV __attribute__((used)) uint32_t d4r_prep_key_at = 0 + 1;
 
 struct Dec0Params
 {
@@ -27,7 +33,11 @@ KERNEL __attribute__((amdgpu_flat_work_group_size(128, 128))) void rrlite_dec0_4
 #pragma unroll
     for (int j = 0; j < 4; ++j)
         d[j] = *(const uint32_t*)(w + 512 * (2 * p + c) + 16 * (4 * g + j) + 4 * (2 * second + h));
+#if D4R_WMMA_LAYOUT == 12
+    g_dec0_w16[idx] = kslot_from16(d);
+#else
     g_dec0_w16[idx] = operand16(d);
+#endif
 }
 
 __attribute__((device)) static inline uint32_t mirror_clamp(int32_t v, uint32_t full, uint32_t half)
@@ -47,6 +57,58 @@ DEV void d4r_dec0_head(uint32_t smem, uint64_t wp, uint64_t inp, uint32_t dimx,
     const uint32_t Y = mirror_clamp(oy / 2 + (int32_t)(m >> 2), dimy, H2);
     const uint32_t Xc = mirror_clamp(ox / 2 + (int32_t)(m & 3), dimx, W2);
     const uint32_t* in = (const uint32_t*)inp + rowstride * Y + 8 * Xc;
+#if D4R_WMMA_LAYOUT == 12
+    kop X[2][2];
+#pragma unroll
+    for (int c = 0; c < 2; ++c)
+#pragma unroll
+        for (int h = 0; h < 2; ++h)
+            X[c][h] = kop_from8(*(const uint2_t*)(in + plane * c + 4 * h + 2 * hf));
+
+    // E[p]: e4m3 codes of channels 16p + 8hf .. 16p + 8hf + 7
+    uint32_t E[8][2];
+#pragma unroll
+    for (int p = 0; p < 8; ++p)
+    {
+        f8v cf;
+#pragma unroll
+        for (int i = 0; i < 8; ++i)
+            cf[i] = (float)*(const half_t*)(w + 8192 + 2 * (16 * p + i + 8 * hf));
+        half_t acc[8];
+#pragma unroll
+        for (int c = 0; c < 2; ++c)
+        {
+            const int slot = ((p * 2 + c) * 2) * 16 + m;
+            cf = k32_e4m3(cf, kop_image(g_dec0_w16, slot), X[c][0], kop_image(g_dec0_w16, slot + 16), X[c][1]);
+#pragma unroll
+            for (int i = 0; i < 8; ++i)
+                acc[i] = (half_t)cf[i];
+#pragma unroll
+            for (int i = 0; i < 8; ++i)
+                cf[i] = (float)acc[i];
+        }
+        uint32_t codes[4];
+#pragma unroll
+        for (int k = 0; k < 4; ++k)
+            codes[k] = e4m3x2(pack2(acc[2 * k], acc[2 * k + 1]));
+        E[p][0] = codes[0] | (codes[1] << 16);
+        E[p][1] = codes[2] | (codes[3] << 16);
+    }
+
+    // staging (layout as below): this lane holds tiles 4G + hf and 4G + 2 + hf (channels 16p + 8hf ..), so it
+    // writes H = hf itself
+    const uint32_t base = 1024 * hf + 32 * (m & 3) + 256 * (m >> 2);
+    const uint32_t goff[4] = {0, 16, 128, 144};
+#pragma unroll
+    for (int G = 0; G < 4; ++G)
+    {
+        const uint32_t* p0 = E[2 * G];
+        const uint32_t* p1 = E[2 * G + 1];
+        u4v words = {__builtin_amdgcn_perm(p1[0], p0[0], 0x05040100u), __builtin_amdgcn_perm(p1[0], p0[0], 0x07060302u),
+                     __builtin_amdgcn_perm(p1[1], p0[1], 0x05040100u), __builtin_amdgcn_perm(p1[1], p0[1], 0x07060302u)};
+        *(__attribute__((address_space(3))) u4v*)(s + base + goff[G]) = words;
+    }
+#else
     u8v X[2][2];
 #pragma unroll
     for (int c = 0; c < 2; ++c)
@@ -109,4 +171,5 @@ DEV void d4r_dec0_head(uint32_t smem, uint64_t wp, uint64_t inp, uint32_t dimx,
                      __builtin_amdgcn_perm(b23, a23, 0x05040100u), __builtin_amdgcn_perm(b23, a23, 0x07060302u)};
         *(__attribute__((address_space(3))) u4v*)(s + base + goff[G]) = words;
     }
+#endif
 }

kernels/tex/enc0_tail.hip

@@ -1,4 +1,4 @@
-// Native tail of rrlite_enc0_4x4_mvhi_hdr_folded: the 2x2-pixel patch embedding GEMM (16 tokens x
+// Native tail of rrlite_enc0_4x4_*_folded (all flag combinations): the 2x2-pixel patch embedding GEMM (16 tokens x
 // K 128 x N 64, e4m3 inputs, f16 accumulation per k32 step, f16 bias), clamp to +-2pi and e4m3 store.
 // The feature code before it is ZLUDA's compile of the original PTX (enc0_tail.ptx calls this after
 // the kernel's own shared-memory staging); the arithmetic reproduces ZLUDA's m16n8k32 lowering
@@ -7,8 +7,14 @@
 
 // B operands (weights) as WMMA A images: index ((p * 4 + c) * 2 + h) * 16 + n16 for channel pair p
 // (channels 16p..16p+15), k32 chunk c, k16 half h.
+#if D4R_WMMA_LAYOUT == 12
+__attribute__((device)) kslot g_enc0_w16[4 * 4 * 2 * 16];
+#else
 __attribute__((device)) u8v g_enc0_w16[4 * 4 * 2 * 16];
+#endif
 DEV __attribute__((used)) uint32_t d4r_prep_blocks = 4;
+// the prep reads only prm.w (offset 0): the hook skips it while that pointer repeats
+DEV __attribute__((used)) uint32_t d4r_prep_key_at = 0 + 1;
 
 struct Enc0Params
 {
@@ -16,7 +22,13 @@ struct Enc0Params
     uint8_t rest[352];
 };
 
-KERNEL __attribute__((amdgpu_flat_work_group_size(128, 128))) void rrlite_enc0_4x4_mvhi_hdr_folded_prep(Enc0Params prm)
+// every flag combination (mvhi/mvlo, hdr/ldr) has this tail; build_tex.sh names the prep after the kernel
+#ifndef D4R_KERNEL_NAME
+#define D4R_KERNEL_NAME rrlite_enc0_4x4_mvhi_hdr_folded
+#endif
+#define D4R_CAT2(a, b) a##b
+#define D4R_CAT(a, b) D4R_CAT2(a, b)
+KERNEL __attribute__((amdgpu_flat_work_group_size(128, 128))) void D4R_CAT(D4R_KERNEL_NAME, _prep)(Enc0Params prm)
 {
     const uint32_t idx = __builtin_amdgcn_workgroup_id_x() * 128 + __builtin_amdgcn_workitem_id_x();
     if (idx >= 512)
@@ -29,7 +41,11 @@ KERNEL __attribute__((amdgpu_flat_work_group_size(128, 128))) void rrlite_enc0_4
 #pragma unroll
     for (int j = 0; j < 4; ++j)
         d[j] = *(const uint32_t*)(w + 4096 + 512 * (4 * p + c) + 16 * (4 * g + j) + 4 * (2 * second + h));
+#if D4R_WMMA_LAYOUT == 12
+    g_enc0_w16[idx] = kslot_from16(d);
+#else
     g_enc0_w16[idx] = operand16(d);
+#endif
 }
 
 DEV void d4r_enc0_tail(uint32_t smem, uint64_t wp, uint64_t outp, uint32_t dimx, uint32_t dimy, int32_t ox, int32_t oy)
@@ -38,6 +54,69 @@ DEV void d4r_enc0_tail(uint32_t smem, uint64_t wp, uint64_t outp, uint32_t dimx,
     const uint8_t* w = (const uint8_t*)wp;
     lds_u8* s = (lds_u8*)(uintptr_t)smem;
 
+#if D4R_WMMA_LAYOUT == 12
+    // activations: token m's 128 e4m3 features (addresses as below); this half's 8 bytes of each k16 part
+    const uint32_t rowoff = 32 * (m & 3) + 256 * (m >> 2);
+    const uint32_t choff[4] = {0, 16, 128, 144};
+    kop X[4][2];
+#pragma unroll
+    for (int c = 0; c < 4; ++c)
+#pragma unroll
+        for (int h = 0; h < 2; ++h)
+            X[c][h] = kop_from8(*(const __attribute__((address_space(3))) uint2_t*)(s + 1024 * h + rowoff + choff[c] + 8 * hf));
+
+    const hv2 lo = {(half_t)-6.2831855f, (half_t)-6.2831855f}, hi = {(half_t)6.2831855f, (half_t)6.2831855f};
+    // E[p]: e4m3 codes of channels 16p + 8hf .. 16p + 8hf + 7 (D^T element i = channel 16p + i + 8hf)
+    uint32_t E[4][2];
+#pragma unroll
+    for (int p = 0; p < 4; ++p)
+    {
+        f8v cf;
+#pragma unroll
+        for (int i = 0; i < 8; ++i)
+            cf[i] = (float)*(const half_t*)(w + 12288 + 2 * (16 * p + i + 8 * hf));
+        half_t acc[8];
+#pragma unroll
+        for (int c = 0; c < 4; ++c)
+        {
+            const int slot = ((p * 4 + c) * 2) * 16 + m;
+            cf = k32_e4m3(cf, kop_image(g_enc0_w16, slot), X[c][0], kop_image(g_enc0_w16, slot + 16), X[c][1]);
+#pragma unroll
+            for (int i = 0; i < 8; ++i)
+                acc[i] = (half_t)cf[i];
+#pragma unroll
+            for (int i = 0; i < 8; ++i)
+                cf[i] = (float)acc[i];
+        }
+        uint32_t codes[4];
+#pragma unroll
+        for (int k = 0; k < 4; ++k)
+        {
+            hv2 v = {acc[2 * k], acc[2 * k + 1]};
+            v = __builtin_elementwise_minimumnum(__builtin_elementwise_maximumnum(v, lo), hi);
+            codes[k] = e4m3x2(__builtin_bit_cast(uint32_t, v));
+        }
+        E[p][0] = codes[0] | (codes[1] << 16);
+        E[p][1] = codes[2] | (codes[3] << 16);
+    }
+
+    // plane pl, word 4 j + q = columns 2q, 2q + 1 of tile (2 pl, j) then of tile (2 pl + 1, j) (tile (p, j) =
+    // channels 16p + 8j ..): this lane holds tiles j = hf, i.e. words 4 hf .. 4 hf + 3 of both planes
+    const uint32_t H2 = dimy >> 1, W2 = dimx >> 1, rowstride = W2 * 8, plane = H2 * rowstride;
+    const int32_t Y = oy / 2 + (int32_t)(m >> 2), Xc = ox / 2 + (int32_t)(m & 3);
+    if ((uint32_t)Y < H2 && (uint32_t)Xc < W2)
+    {
+#pragma unroll
+        for (int pl = 0; pl < 2; ++pl)
+        {
+            const uint32_t* p0 = E[2 * pl];
+            const uint32_t* p1 = E[2 * pl + 1];
+            const u4v words = {__builtin_amdgcn_perm(p1[0], p0[0], 0x05040100u), __builtin_amdgcn_perm(p1[0], p0[0], 0x07060302u),
+                               __builtin_amdgcn_perm(p1[1], p0[1], 0x05040100u), __builtin_amdgcn_perm(p1[1], p0[1], 0x07060302u)};
+            *(u4v*)((uint32_t*)outp + rowstride * (uint32_t)Y + 8 * (uint32_t)Xc + plane * pl + 4 * hf) = words;
+        }
+    }
+#else
     // activations: token m's 128 e4m3 features; A[m][k] of chunk c at
     // 1024 * (k >> 4) + 32 * (m & 3) + 256 * (m >> 2) + (k & 15) + {0, 16, 128, 144}[c]
     const uint32_t rowoff = 32 * (m & 3) + 256 * (m >> 2);
@@ -121,4 +200,5 @@ DEV void d4r_enc0_tail(uint32_t smem, uint64_t wp, uint64_t outp, uint32_t dimx,
         uint32_t* out = (uint32_t*)outp + rowstride * (uint32_t)Y + 8 * (uint32_t)Xc + plane * hf;
         *(u8v*)out = words;
     }
+#endif
 }

kernels/tex/make_ptx.py

@@ -11,6 +11,28 @@ from pathlib import Path
 # PTX extracted from nvngx_dlss.dll by kernels/tools/extract_dlss_ptx.py (kernels/build.sh does this)
 PTX_DIR = Path(os.environ.get("D4R_DLSS_PTX_DIR", Path(__file__).resolve().parent.parent / "extracted" / "ptx"))
 
+ENC0_TAIL = """{{
+	.param .b32 q0;
+	st.param.b32 [q0+0], {smem};
+	.param .b64 q1;
+	st.param.b64 [q1+0], {w};
+	.param .b64 q2;
+	st.param.b64 [q2+0], {out};
+	.param .b32 q3;
+	st.param.b32 [q3+0], {dimx};
+	.param .b32 q4;
+	st.param.b32 [q4+0], {dimy};
+	.param .b32 q5;
+	st.param.b32 [q5+0], {ox};
+	.param .b32 q6;
+	st.param.b32 [q6+0], {oy};
+	call.uni d4r_enc0_tail, (q0, q1, q2, q3, q4, q5, q6);
+}}
+ret;
+
+}}
+"""
+
 KERNELS = {
     # enc0: cut after the bar.warp.sync that ends the smem staging of the GEMM A operand
     "rrlite_enc0_4x4_mvhi_hdr_folded": dict(
@@ -25,27 +47,7 @@ KERNELS = {
 )
 ;
 """,
-        tail="""{
-	.param .b32 q0;
-	st.param.b32 [q0+0], %r259;
-	.param .b64 q1;
-	st.param.b64 [q1+0], %rd106;
-	.param .b64 q2;
-	st.param.b64 [q2+0], %rd78;
-	.param .b32 q3;
-	st.param.b32 [q3+0], %r1066;
-	.param .b32 q4;
-	st.param.b32 [q4+0], %r1067;
-	.param .b32 q5;
-	st.param.b32 [q5+0], %r1230;
-	.param .b32 q6;
-	st.param.b32 [q6+0], %r1233;
-	call.uni d4r_enc0_tail, (q0, q1, q2, q3, q4, q5, q6);
-}
-ret;
-
-}
-""",
+        tail=ENC0_TAIL.format(smem="%r259", w="%rd106", out="%rd78", dimx="%r1066", dimy="%r1067", ox="%r1230", oy="%r1233"),
     ),
     # dec0: the GEMM (from the B fragment loads to the e4m3 staging stores) becomes a native call; the
     # A loads before it are left dead. %r560 (shared base) is the only register of the range used later.
@@ -206,12 +208,86 @@ def replace_rz_round(lines):
     return out, n
 
 
+ENC0_REF = "rrlite_enc0_4x4_mvhi_hdr_folded"
+# Surface-store and round-idiom recipes for the other flag combinations.
+SUST_ONLY_RE = re.compile(r"^(hiluma_engine_output_depth(inv|reg)_mv(hi|lo)_(hdr|ldr)(_max)?_v[12]_rel|"
+                          r"hiluma_engine_input_depth(inv|reg)_mv(hi|lo)_(hdr|ldr)_v2_rel|"
+                          r"rrlite_post_3_[12]_mv(hi|lo)_(hdr|ldr)(_folded)?|"
+                          r"rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)|rrlite_dec0_4x4|"
+                          r"rrlite_downsample_kernel_(static|dynamic)_(hdr|ldr))$")
+ENC0_RE = re.compile(r"^rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)_folded$")
+
+
+def read_lines(path):
+    return path.read_text().replace("\r\n", "\n").split("\n")
+
+
+def enc0_registers(lines):
+    """Cut line (1-based) and the tail call's registers of an enc0 variant. The code after the third
+    bar.warp.sync is the same GEMM in every flag combination, only register numbers differ: the shared
+    base and weights pointer are read off by aligning that block with the reference variant; the output
+    pointer and grid size are the kernel's loads of parameters +40 and +8; the block offsets are the two
+    values halved (shr 31 / add) by the epilogue."""
+    syncs = [i for i, l in enumerate(lines) if l.strip() == "bar.warp.sync -1;"]
+    if len(syncs) != 3:
+        sys.exit(f"enc0: expected 3 bar.warp.sync, found {len(syncs)}")
+    cut = syncs[2] + 1
+    before, after = lines[:cut], lines[cut:]
+    out = [m.group(1) for m in (re.match(r"^ld\.param\.u64 (%rd\d+), \[%rd\d+\+40\];$", l.strip()) for l in before) if m]
+    dims = [m.groups() for m in (re.match(r"^ld\.param\.v2\.u32 \{(%r\d+), (%r\d+)\}, \[%rd\d+\+8\];$", l.strip()) for l in before) if m]
+    halves = []
+    for a, b in zip(after, after[1:]):
+        m = re.match(r"^shr\.u32 (%r\d+), (%r\d+), 31;$", a.strip())
+        if m and re.match(rf"^add\.s32 %r\d+, {re.escape(m.group(2))}, {re.escape(m.group(1))};$", b.strip()):
+            halves.append(m.group(2))
+    if len(out) != 1 or len(dims) != 1 or len(halves) < 2:
+        sys.exit(f"enc0: parameter loads or epilogue not found ({out}, {dims}, {halves[:2]})")
+    return cut, dict(out=out[0], dimx=dims[0][0], dimy=dims[0][1], oy=halves[0], ox=halves[1])
+
+
+REG_RE = re.compile(r"%(?:rd|rs|r|fd|f|p)\d+")
+
+
+def enc0_variant(name):
+    ref = read_lines(module_file(ENC0_REF, KERNELS[ENC0_REF]))
+    path = module_file(name, {"file": ""})
+    lines = read_lines(path)
+    rcut, rregs = enc0_registers(ref)
+    k = KERNELS[ENC0_REF]
+    known = dict(smem="%r259", w="%rd106", out="%rd78", dimx="%r1066", dimy="%r1067", ox="%r1230", oy="%r1233")
+    if rcut != k["cut_after"] or any(rregs[key] != known[key] for key in rregs):
+        sys.exit(f"enc0: the register search does not reproduce {ENC0_REF} ({rcut}, {rregs})")
+    cut, regs = enc0_registers(lines)
+    # align the GEMM block (up to the reference's first closing brace) and map registers
+    n = next(i for i, l in enumerate(ref[rcut:]) if l.startswith("}"))
+    mapping = {}
+    for a, b in zip(ref[rcut:rcut + n], lines[cut:cut + n]):
+        if REG_RE.sub("R", a) != REG_RE.sub("R", b):
+            sys.exit(f"enc0: {name} differs from {ENC0_REF} after the cut: {a!r} / {b!r}")
+        for x, y in zip(REG_RE.findall(a), REG_RE.findall(b)):
+            if mapping.setdefault(x, y) != y:
+                sys.exit(f"enc0: inconsistent register mapping {x} -> {mapping[x]} / {y}")
+    regs.update(smem=mapping[known["smem"]], w=mapping[known["w"]])
+    return dict(file=path.name, cut_after=cut, cut_check="bar.warp.sync -1;", sust=True, extern=k["extern"],
+                tail=ENC0_TAIL.format(**regs))
+
+
+def kernel_spec(name):
+    if name in KERNELS:
+        return KERNELS[name]
+    if ENC0_RE.match(name):
+        return enc0_variant(name)
+    if SUST_ONLY_RE.match(name):
+        return dict(file="", sust=True, extern="", rz_round=name.startswith("hiluma_engine_"))
+    sys.exit(f"make_ptx: no recipe for {name}")
+
+
 def module_file(name, k):
     """The extracted PTX module that defines NAME: the 310.7 file number, else any module that does
     (other DLSS versions number their modules differently)."""
     entry = re.compile(rf"\.entry\s+{re.escape(name)}\s*\(")
     known = PTX_DIR / k["file"]
-    if known.exists() and entry.search(known.read_text()):
+    if k["file"] and known.is_file() and entry.search(known.read_text()):
         return known
     for path in sorted(PTX_DIR.glob("*.ptx")):
         if entry.search(path.read_text()):
@@ -221,8 +297,8 @@ def module_file(name, k):
 
 def main():
     name, out = sys.argv[1], Path(sys.argv[2])
-    k = KERNELS[name]
-    lines = module_file(name, k).read_text().replace("\r\n", "\n").split("\n")
+    k = kernel_spec(name)
+    lines = read_lines(module_file(name, k))
     if "cut_after" in k:
         cut = k["cut_after"]
         if lines[cut - 1].strip() != k["cut_check"]:

kernels/tex/tex_common.h

@@ -3,6 +3,10 @@
 // only plain clang builtins are used (compiled with -nogpuinc -nogpulib).
 #pragma once
 #include <stdint.h>
+#include "../common/wmma_layout.h"
+#if defined(D4R_TEX_FP8) && D4R_WMMA_LAYOUT != 12 && defined(__HIP_DEVICE_COMPILE__)
+#error "D4R_TEX_FP8 needs the gfx12 WMMA layout"
+#endif
 
 #pragma clang fp contract(off)
 
@@ -10,7 +14,7 @@
 // Kernels that ZLUDA compiles in strict-FP mode (constrained intrinsics) cannot take inlined FP code from
 // here (the AMDGPU backend fails on some constrained operations): build them with -DD4R_SUST_OUTLINE,
 // which keeps the FP bodies of the surface stores in noinline leaf functions (no calls, no stack).
-#ifdef D4R_SUST_OUTLINE
+#if defined(D4R_SUST_OUTLINE) || defined(D4R_ACCURACY)
 #define FP_BODY __attribute__((device, noinline)) static
 #else
 #define FP_BODY __attribute__((device, always_inline)) static inline
@@ -65,6 +69,75 @@ __attribute__((device)) static inline f8v wmma(u8v a, u8v b, f8v c)
     return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(h16, a), __builtin_bit_cast(h16, b), c);
 }
 
+// ---------------------------------------------------------------- gfx12 m16n8k32 e4m3 steps
+// On gfx12 the GEMM tails do a k32 step as ZLUDA's gfx12 lowering of NVIDIA's m16n8k32 e4m3 MMA does, so they
+// stay bit-identical to ZLUDA's compile of the kernel they replace: a lane's k16 operand holds this half's
+// e4m3 k 8h .. 8h + 7 (ZLUDA's gfx12 slot order), either widened as above with C * 2^-16 inside the f32
+// WMMAs (d4r_wmma12_*), or with D4R_TEX_FP8 (ZLUDA's D4R_ZLUDA_WMMA_FP8_NATIVE) as the bytes of a native FP8
+// WMMA with no scaling (d4r_wmma12f8_*). (The gfx11 tails use operand16 and wmma above.)
+#if D4R_WMMA_LAYOUT == 12
+// 8 e4m3 bytes (k in order) widened to value * 2^-8 f16 pairs, k in order
+__attribute__((device)) static inline u4v widen8(uint2_t d)
+{
+    uint32_t a0, a1, b0, b1;
+    e4m3x4_scaled(d.x, a0, a1);
+    e4m3x4_scaled(d.y, b0, b1);
+    return (u4v){a0, a1, b0, b1};
+}
+#ifdef D4R_TEX_FP8
+typedef uint2_t kop;
+typedef u4v kslot; // weight image slot: the 16 e4m3 bytes of one k16 operand row
+__attribute__((device)) static inline kop kop_from8(uint2_t d)
+{
+    return d;
+}
+__attribute__((device)) static inline kslot kslot_from16(u4v d)
+{
+    return d;
+}
+#else
+typedef u4v kop;
+typedef u8v kslot; // weight image slot: the 16 widened values of one k16 operand row, k in order
+__attribute__((device)) static inline kop kop_from8(uint2_t d)
+{
+    return widen8(d);
+}
+__attribute__((device)) static inline kslot kslot_from16(u4v d)
+{
+    const u4v lo = widen8((uint2_t){d[0], d[1]}), hi = widen8((uint2_t){d[2], d[3]});
+    return (u8v){lo[0], lo[1], lo[2], lo[3], hi[0], hi[1], hi[2], hi[3]};
+}
+#endif
+// this half's k16 operand from 16 consecutive e4m3 bytes of NVIDIA's k order
+__attribute__((device)) static inline kop kop_from16(u4v d)
+{
+    return kop_from8(lane_id() >= 16 ? (uint2_t){d[2], d[3]} : (uint2_t){d[0], d[1]});
+}
+// this half's operand from slot idx of a weight image
+__attribute__((device)) static inline kop kop_image(const kslot* img, int idx)
+{
+    return ((const kop*)img)[2 * idx + (lane_id() >> 4)];
+}
+// f32 C (true scale) + one k32 step
+__attribute__((device)) static inline f8v k32_e4m3(f8v c, const kop& a0, const kop& b0, const kop& a1, const kop& b1)
+{
+#if defined(D4R_TEX_FP8) && defined(__GFX12__)
+    typedef int i2v __attribute__((ext_vector_type(2)));
+    c = __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12(__builtin_bit_cast(i2v, a0), __builtin_bit_cast(i2v, b0), c);
+    return __builtin_amdgcn_wmma_f32_16x16x16_fp8_fp8_w32_gfx12(__builtin_bit_cast(i2v, a1), __builtin_bit_cast(i2v, b1), c);
+#elif defined(D4R_TEX_FP8)
+    // layout shim, like ZLUDA's: each FP8 WMMA as the scaled widening (exact powers of two)
+    c = wm_mma_zluda(widen8(a0), widen8(b0), c * 0x1p-16f) * 0x1p16f;
+    return wm_mma_zluda(widen8(a1), widen8(b1), c * 0x1p-16f) * 0x1p16f;
+#else
+    c = c * 0x1p-16f;
+    c = wm_mma_zluda(a0, b0, c);
+    c = wm_mma_zluda(a1, b1, c);
+    return c * 0x1p16f;
+#endif
+}
+#endif
+
 // ZLUDA's f16x2_to_e4m3x2_satfinite_bits: RNE satfinite e4m3 of both halves, low half -> low byte.
 __attribute__((device)) static inline uint32_t e4m3x2(uint32_t bits)
 {

kernels/tools/dump_runner.cpp

@@ -44,7 +44,18 @@ int main(int argc, char** argv)
     hipFunction_t prep = nullptr;
     if (hipModuleGetFunction(&prep, mod, (std::string(argv[2]) + "_prep").c_str()) != hipSuccess)
         prep = nullptr;
-    const unsigned prepGrid = getenv("PREP_GRID") ? atoi(getenv("PREP_GRID")) : 88;
+    // the prep grid: PREP_GRID, else the module's d4r_prep_blocks (what the ZLUDA hook launches), else 88
+    unsigned prepGrid = 88;
+    {
+        hipDeviceptr_t pb;
+        size_t pbs = 0;
+        uint32_t v = 0;
+        if (hipModuleGetGlobal(&pb, &pbs, mod, "d4r_prep_blocks") == hipSuccess && pbs == 4 &&
+            hipMemcpyDtoH(&v, pb, 4) == hipSuccess && v != 0)
+            prepGrid = v;
+    }
+    if (getenv("PREP_GRID"))
+        prepGrid = atoi(getenv("PREP_GRID"));
 
     std::vector<char> args = read_file(dump + "/args.bin");
     std::ifstream man(dump + "/manifest.txt");
@@ -58,6 +69,13 @@ int main(int argc, char** argv)
         std::istringstream s(line);
         std::string kind;
         s >> kind;
+        if (kind == "texture" || kind == "surface")
+        {
+            // texture/surface object handles in the arguments would point at nothing here: the kernel faults
+            fprintf(stderr, "%s: the dump uses texture or surface objects, which dump_runner cannot recreate\n",
+                    dump.c_str());
+            return 3;
+        }
         if (kind == "launch")
             s >> grid[0] >> grid[1] >> grid[2] >> block[0] >> block[1] >> block[2] >> shared;
         else if (kind == "alloc")

kernels/tools/model_enc3.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Numeric primitives and replay-dump access for swin_model.py (the rrlite Swin blocks of DLSS 4.5, preset M).
+
+Everything here is NVIDIA's arithmetic as the native kernels reproduce it (kernels/m/swin_common.h,
+swin_block.h): f16 rounding after every operation, e4m3 (OCP, satfinite, RNE) re-quantisation, one f16 rounding
+per k32 MMA step, and the softmax's bit-level exponent trick.
+"""
+import struct
+
+import numpy as np
+
+F16, F32, F64 = np.float16, np.float32, np.float64
+
+
+def f16(x):
+    """round to f16; the result stays float64 (on the f16 grid), so products and sums of such values are exact
+    until the next f16() (numpy would otherwise do f16 arithmetic, e.g. in matmul)"""
+    return np.asarray(x, F64).astype(F16).astype(F64)
+
+
+def _e4m3_table():
+    t = np.zeros(256, F64)
+    for c in range(256):
+        s, e, m = c >> 7, (c >> 3) & 15, c & 7
+        if e == 15 and m == 7:
+            v = np.nan
+        elif e == 0:
+            v = m * 2.0 ** -9
+        else:
+            v = (1 + m / 8) * 2.0 ** (e - 7)
+        t[c] = -v if s else v
+    return t
+
+
+E4M3 = _e4m3_table()
+_POS = E4M3[:127]  # codes 0..126: the finite non-negative values, ascending
+
+
+def q8(x):
+    """e4m3 RNE satfinite round trip of f16 values (NaN stays NaN), as f16"""
+    h = f16(x).astype(F64)
+    a = np.abs(h)
+    i = np.clip(np.searchsorted(_POS, a), 1, 126)
+    lo, hi = _POS[i - 1], _POS[i]
+    # nearest; ties to the even code
+    pick_hi = (hi - a < a - lo) | ((hi - a == a - lo) & ((i & 1) == 0))
+    v = np.where(pick_hi, hi, lo)
+    v = np.where(a >= _POS[126], _POS[126], v)
+    v = np.where(a <= 0, 0.0, v)
+    v = np.where(np.isnan(h), np.nan, np.copysign(v, h))
+    return f16(v)
+
+
+# softmax constants (swin_block.h stage 2)
+C_SCALE = f16(0.0972222164273262 / 5.656854152679443)
+CLAMP = F16(0.55615234375)
+P1 = f16(0.92730712890625)
+P0 = F16(1.375)
+
+
+def exp_trick(poly):
+    """per row, columns (2j, 2j+1): the 32-bit word even | odd << 16 of the f16 bits, << 5, + 0x7FF88000; the even
+    column takes its low half, the odd column its high half (as f16 bits)"""
+    b = poly.astype(F16).view(np.uint16).astype(np.uint64)
+    out = np.empty_like(b)
+    even, odd = b[:, 0::2], b[:, 1::2]
+    word = ((even | (odd << np.uint64(16))) << np.uint64(5)) + np.uint64(0x7FF88000)
+    word &= np.uint64(0xFFFFFFFF)
+    out[:, 0::2] = word & np.uint64(0xFFFF)
+    out[:, 1::2] = word >> np.uint64(16)
+    return out.astype(np.uint16).view(F16)
+
+
+def row_sum16(w):
+    """f16 butterfly over the 16 columns: partner c ^ 8, then ^ 2, ^ 4, ^ 1 (every column ends with the same sum)"""
+    s = w.astype(F16)
+    c = np.arange(16)
+    for m in (8, 2, 4, 1):
+        s = f16(s.astype(F64) + s[:, c ^ m].astype(F64))
+    return s
+
+
+def gelu_poly(z):
+    z = f16(z)
+    cz = np.clip(z, F16(-2.0), F16(2.0))
+    inner = f16(f16(0.41216981).astype(F64) - f16(f16(0.08108133).astype(F64) * np.abs(cz).astype(F64)).astype(F64))
+    return f16(z.astype(F64) * f16(0.5 + f16(cz.astype(F64) * inner.astype(F64)).astype(F64)).astype(F64))
+
+
+def mma_chain(A, W, C0=None):
+    """chained k32 MMA steps with f16 accumulation: D = f16(A[:, 32k:32k+32] W[32k:32k+32] + D), exact sums"""
+    M, K = A.shape
+    D = np.zeros((M, W.shape[1]), F64) if C0 is None else f16(C0)
+    A = np.asarray(A, F64)
+    W = np.asarray(W, F64)
+    for k in range(0, K, 32):
+        D = f16(A[:, k:k + 32] @ W[k:k + 32] + D.astype(F64))
+    return D
+
+
+class Params:
+    """one rrlite Swin-block launch from a replay dump (CommonParams: w 0, sx sy 8, tw th 16, in 24, p32 32,
+    out 40, p48 48)"""
+
+    def __init__(self, d):
+        man = open(d + '/manifest.txt').read().split('\n')
+        self.kernel = man[0].split()[1]
+        la = man[1].split()
+        self.grid = tuple(map(int, la[1:4]))
+        self.args = open(d + '/args.bin', 'rb').read()
+        self.allocs = {}
+        for line in man:
+            p = line.split()
+            if p and p[0] == 'alloc':
+                self.allocs[int(p[1])] = (int(p[2], 16), np.fromfile(f'{d}/alloc-{p[1]}.bin', np.uint8))
+        self.w_ptr, = struct.unpack_from('<Q', self.args, 0)
+        self.sx, self.sy, self.W, self.H = struct.unpack_from('<iiii', self.args, 8)
+        self.inp, self.p32, self.out, self.p48 = struct.unpack_from('<QQQQ', self.args, 24)
+        wbase, wbuf = self.alloc_of(self.w_ptr)
+        self.wbuf = wbuf[self.w_ptr - wbase:]
+        self.abase, self.arena = self.alloc_of(self.inp)
+
+    def alloc_of(self, a):
+        for base, buf in self.allocs.values():
+            if base <= a < base + buf.size:
+                return base, buf
+        raise ValueError(hex(a))
+
+    def f16vec(self, off, n):
+        return self.wbuf[off:off + 2 * n].view(F16).copy()
+
+    def token_xy(self, bx, by):
+        T = np.arange(64)
+        tx = (T & 3) + 4 * ((T >> 4) & 1)
+        ty = ((T >> 2) & 3) + 4 * (T >> 5)
+        return 8 * bx - self.sx + tx, 8 * by - self.sy + ty
+
+    @staticmethod
+    def mirror(v, n):
+        v = np.abs(v)
+        return np.minimum(v, 2 * n - 2 - v)

kernels/tools/ptxsim.py

@@ -96,7 +96,8 @@ def split_args(s):
 
 
 def parse_kernel(path, name):
-    src = open(path).read()
+    with open(path) as file:
+        src = file.read()
     i = src.index('.entry ' + name + '(')
     j = src.find('\n.visible', i + 10)
     j = len(src) if j < 0 else j
@@ -340,12 +341,11 @@ class Block:
         steps = 0
         while pc < len(stmts):
             st = stmts[pc]
-            for l in st.labels:
-                if l in pending:
-                    active |= pending.pop(l)
+            if pc in pending:
+                active |= pending.pop(pc)
             if not active.any():
-                # jump to the nearest label someone waits at
-                targets = sorted(self.labels[l] for l in pending)
+                # Reconverge at either a forward target or a loop fallthrough.
+                targets = sorted(pending)
                 if not targets:
                     break
                 pc = targets[0]
@@ -365,10 +365,16 @@ class Block:
                     if not taking.any():
                         pc += 1
                         continue
-                    raise NotImplementedError('divergent backward branch at ' + st.text)
+                    # Lanes finishing a loop wait at its fallthrough while the
+                    # other lanes execute their remaining iterations. Their
+                    # registers and memory writes stay masked until reconvergence.
+                    pending[pc + 1] = pending.get(pc + 1, np.zeros(self.T, bool)) | (active & ~taking)
+                    active = taking.copy()
+                    pc = target
+                    continue
                 active = active & ~taking
                 if taking.any():
-                    pending[st.args[0]] = pending.get(st.args[0], np.zeros(self.T, bool)) | taking
+                    pending[target] = pending.get(target, np.zeros(self.T, bool)) | taking
                 pc += 1
                 continue
             if mask.any() or st.op.startswith(('mma', 'shfl', 'movmatrix', 'bar')):

kernels/tools/pwin_check.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Compares native preset-K layer outputs with the numpy model (pwin_model.py) on sampled windows of a replay dump.
+
+usage: pwin_check.py DUMP_DIR OUT_DIR [OUT_DIR...] [--windows N]
+  DUMP_DIR  replay dump of one dltss_pwin_* launch (inputs)
+  OUT_DIR   dump_runner output of a native kernel on that dump (alloc-N.bin after the launch)
+For every sampled window (the four corners, border windows and random interior ones) the model output and the
+kernel's stored values are compared per f16 element: exact matches, and the error in f16 ulps at the model
+value's magnitude (values below 2^-10 count with the ulp of 2^-10, so near-zero sign flips do not dominate).
+The kernels round as NVIDIA's f16 wmma does except in the PWIN_F32ACC layers, and gfx12 WMMA groups its f32
+sums differently from gfx11, so neither build is expected to match the model exactly.
+"""
+import os
+import random
+import sys
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import pwin_model as pm  # noqa: E402
+
+F16 = np.float16
+
+# layer -> (kind, H, C, COUT/CL, extra)
+LAYERS = {
+    'enc0': ('enc0', 2, 32, 64, None), 'enc1': ('enc', 2, 64, 64, True), 'enc2': ('enc', 2, 64, 96, False),
+    'enc3': ('enc', 4, 96, 128, False), 'enc4': ('enc', 4, 128, 160, False), 'dec5': ('plain', 8, 160, 16, None),
+    'dec4': ('dec', 4, 128, 160, None), 'dec3': ('dec', 4, 96, 128, None), 'dec2': ('dec', 2, 64, 96, None),
+    'dec1': ('dec', 2, 64, 64, None), 'dec0': ('dec0', 2, 32, 64, None),
+}
+
+
+class Outputs:
+    """the dump's pointer map over the allocations a native run saved"""
+    def __init__(self, dump, out_dir):
+        self.d = dump
+        self.allocs = {i: (base, np.fromfile(f'{out_dir}/alloc-{i}.bin', np.uint8)) for i, (base, _) in dump.allocs.items()}
+
+    def rows(self, off_param, index, width):
+        a = self.d.u64(off_param)
+        for base, buf in self.allocs.values():
+            if base <= a < base + buf.size:
+                o = a - base + 2 * index * width
+                return buf[o:o + 2 * width].view(F16)
+        raise ValueError(hex(a))
+
+
+def ulps(model, got):
+    m = model.astype(np.float64)
+    g = got.astype(np.float64)
+    scale = np.maximum(np.abs(m), 2.0 ** -10)
+    ulp = 2.0 ** (np.floor(np.log2(scale)) - 10)
+    return np.abs(m - g) / ulp
+
+
+def windows(grid, n, seed=1):
+    gx, gy = grid[0], grid[1]
+    w = {(0, 0), (gx - 1, 0), (0, gy - 1), (gx - 1, gy - 1), (gx // 2, 0), (0, gy // 2), (gx - 1, gy // 2), (gx // 2, gy - 1)}
+    r = random.Random(seed)
+    while len(w) < min(n, gx * gy):
+        w.add((r.randrange(gx), r.randrange(gy)))
+    return sorted(w)
+
+
+def model_window(P, layer, bx, by):
+    kind, H, C, X, extra = LAYERS[layer]
+    if kind == 'enc0':
+        return pm.pwin_enc0(P, bx, by)
+    if kind == 'enc':
+        return pm.pwin_encoder(P, bx, by, H, C, X, posattn=extra)
+    if kind == 'plain':
+        return pm.swin_core(P, pm.Layout(H, C, C), pm.load_window(P, bx, by, C)), None
+    if kind == 'dec':
+        return pm.pwin_decoder(P, bx, by, H, C, X), None
+    return pm.pwin_dec0(P, bx, by)
+
+
+def stored(P, O, layer, bx, by, y, extra_out):
+    """(model values, kernel values) of everything the kernel stores for window (bx, by)"""
+    kind, H, C, X, _ = LAYERS[layer]
+    W, Hh = P.i32x2(0)
+    sx, sy = P.i32x2(56)
+    ms, gs = [], []
+    full_off = 32 if kind in ('enc0', 'enc') else 24
+    for t in range(64):
+        Y0, X0 = 8 * by - sy + (t >> 3), 8 * bx - sx + (t & 7)
+        if not (0 <= Y0 < Hh and 0 <= X0 < W):
+            continue
+        if kind == 'dec0':
+            ms.append(extra_out[t]); gs.append(O.rows(24, Y0 * W + X0, 40))
+        else:
+            ms.append(y[t]); gs.append(O.rows(full_off, Y0 * W + X0, C))
+    if kind in ('enc0', 'enc'):
+        cout = extra_out.shape[1]
+        for r in range(16):
+            Ym, Xm = int((8 * by - sy) / 2) + (r >> 2), int((8 * bx - sx) / 2) + (r & 3)
+            if 0 <= Ym < Hh // 2 and 0 <= Xm < W // 2:
+                ms.append(extra_out[r]); gs.append(O.rows(24, Ym * (W // 2) + Xm, cout))
+    return np.concatenate(ms), np.concatenate(gs)
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith('--')]
+    n = 12
+    if '--windows' in sys.argv:
+        n = int(sys.argv[sys.argv.index('--windows') + 1])
+        args.remove(str(n))
+    dump_dir, outs = args[0], args[1:]
+    P = pm.Dump(dump_dir)
+    layer = P.kernel.replace('dltss_pwin_', '').replace('_layer', '')
+    wins = windows(P.grid, n)
+    models = {w: model_window(P, layer, *w) for w in wins}
+    for out in outs:
+        O = Outputs(P, out)
+        m, g = zip(*(stored(P, O, layer, bx, by, *models[(bx, by)]) for bx, by in wins))
+        m, g = np.concatenate(m), np.concatenate(g)
+        u = ulps(m, g)
+        exact = (m.view(np.uint16) == g.view(np.uint16)).mean()
+        md, gd = m.astype(np.float64), g.astype(np.float64)
+        psnr = 10 * np.log10(np.abs(md).max() ** 2 / max(((md - gd) ** 2).mean(), 1e-30))
+        print(f'{layer:5s} {os.path.basename(os.path.normpath(out)):10s} {len(wins)} windows {m.size} values: exact {100 * exact:5.1f}%  '
+              f'<=1ulp {100 * (u <= 1).mean():5.1f}%  <=4ulp {100 * (u <= 4).mean():6.2f}%  max {u.max():7.1f} ulp  '
+              f'mean {u.mean():.3f} ulp  psnr {psnr:.1f} dB  finite {np.isfinite(g.astype(np.float32)).mean() * 100:.0f}%')
+
+
+if __name__ == '__main__':
+    main()

kernels/tools/pwin_model.py

@@ -6,6 +6,38 @@ import numpy as np
 
 F16, F32, F64 = np.float16, np.float32, np.float64
 
+# Legacy replay reports retain their original approximate norm by default.
+# Windows strict K validation selects the PTX fragment's rounded reduction.
+NORM_FRAGMENT_ORDER = False
+
+
+def norm_rows(x):
+    if not NORM_FRAGMENT_ORDER:
+        return f16(np.sum(x.astype(F32).astype(F64) ** 2, axis=1))
+    tokens, channels = x.shape
+    assert channels in (32, 64, 96, 128, 160)
+    square = f16(x.astype(F64) ** 2)
+    partial = []
+    for lane in range(4):
+        halves = []
+        for parity in range(2):
+            pairs = [add16(square[:, 2 * lane + 8 * parity + 32 * g:2 * lane + 8 * parity + 32 * g + 2],
+                           square[:, 2 * lane + 8 * parity + 32 * g + 16:2 * lane + 8 * parity + 32 * g + 18])
+                     for g in range(channels // 32)]
+            value = pairs[0]
+            if channels >= 64:
+                value = add16(value, pairs[1])
+            if channels == 96:
+                value = add16(value, pairs[2])
+            if channels >= 128:
+                value = add16(value, add16(pairs[2], pairs[3]))
+            if channels == 160:
+                value = add16(value, pairs[4])
+            halves.append(value)
+        partial.append(add16(halves[0], halves[1]))
+    total = add16(add16(partial[0], partial[1]), add16(partial[2], partial[3]))
+    return add16(total[:, 0], total[:, 1])
+
 
 class Dump:
     def __init__(self, d):
@@ -181,7 +213,7 @@ def pwin_enc2(P, bx, by, pm_order=((0, 0), (0, 1), (1, 0), (1, 1)), b2_in_c0=Tru
     """logical enc2 layer (H 2, C 64 -> merged 96) for one window; returns (y [64 x 64], merged [16 x 96])"""
     C, H, Dh = 64, 2, 32
     x0 = load_window(P, bx, by, C)
-    s = f16(np.sum(x0.astype(F32).astype(F64) ** 2, axis=1))
+    s = norm_rows(x0)
     r = rsqrt16(s)
     h1 = mul16(x0, mul16(r[:, None], vec(P, 0, C)[None, :]))
     Os = []
@@ -248,7 +280,7 @@ def swin_core(P, L, x0, dbg=None, posattn=False):
     """norm -> attention -> Wo + residual -> MLP (+ residual): the block output y [64 x C]"""
     C, H = L.C, L.H
     NCH = C // 32
-    s = f16(np.sum(x0.astype(F32).astype(F64) ** 2, axis=1))
+    s = norm_rows(x0)
     r = rsqrt16(s)
     h1 = mul16(x0, mul16(r[:, None], vec(P, L.G1, C)[None, :]))
     Os = []

kernels/tools/swin_check.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Compares native preset-M Swin-block outputs with the numpy model (swin_model.py) on sampled blocks of a dump.
+
+usage: swin_check.py DUMP_DIR OUT_DIR [OUT_DIR...] [--blocks N]
+  DUMP_DIR  replay dump of one rrlite_{enc1,enc2,enc3_tube,dec1,dec2}_4x4 launch
+  OUT_DIR   dump_runner output of a native kernel on that dump
+The stored e4m3 outputs (block output, and the patch merge of encoders that have one) are compared code by code:
+the share of identical codes, of codes one e4m3 step apart, and the PSNR of the decoded values. The model is
+NVIDIA's arithmetic (e4m3 activations, f16 accumulation per k32 step); the production kernels keep activations
+f16 and accumulate in f32 (SWIN_NO_Q8, SWIN_F32ACC), so they are not expected to match it exactly.
+"""
+import os
+import random
+import sys
+
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import model_enc3 as me  # noqa: E402
+import swin_model as sm  # noqa: E402
+
+# kernel -> (C, heads, NPM, CIN)
+LAYERS = {'enc1': (64, 2, 96, 0), 'enc2': (96, 4, 128, 0), 'enc3_tube': (128, 4, 0, 0), 'dec2': (96, 4, 0, 128),
+          'dec1': (64, 2, 0, 96)}
+
+# Preserve the existing Linux oracle by default; Windows strict replay enables
+# the audited 310.9 f16 patch-merge bound before final e4m3 quantisation.
+CLAMP_PATCH_MERGE = False
+
+
+def blocks(grid, n, seed=1):
+    gx, gy = grid[0], grid[1]
+    b = {(0, 0), (gx - 1, 0), (0, gy - 1), (gx - 1, gy - 1), (gx // 2, 0), (0, gy // 2), (gx - 1, gy // 2), (gx // 2, gy - 1)}
+    r = random.Random(seed)
+    while len(b) < min(n, gx * gy):
+        b.add((r.randrange(gx), r.randrange(gy)))
+    return sorted(b)
+
+
+def model_block(P, layer, bx, by):
+    C, heads, NPM, CIN = LAYERS[layer]
+    pre = 4 * C * CIN + 8 * C if CIN else 0
+    x0 = sm.decoder_input(P, bx, by, C, CIN, heads, P.p32) if CIN else sm.load_tokens(P, bx, by, C // 32)
+    x2, out, L = sm.swin_block(P, x0, C, heads, pre)
+    merged = None
+    if NPM:
+        A = sm.patch_merge_input(out, C)
+        PM0 = L['end']
+        cols = []
+        for g in range(NPM // 32):
+            Wg = me.E4M3[P.wbuf[sm.woff_table(PM0 + g * 128 * C, 512, 64 * C, 4 * C, 32)]]
+            bias = P.f16vec(PM0 + 4 * C * NPM + 64 * g, 32)
+            cols.append(me.mma_chain(A, Wg, C0=np.broadcast_to(bias, (16, 32)).copy()))
+        merged = sm.pair_to_nat(np.concatenate(cols, axis=1))
+        if CLAMP_PATCH_MERGE:
+            limit = me.f16(2 * np.pi)
+            merged = np.clip(merged, -limit, limit)
+        merged = me.q8(merged)
+    return out, merged
+
+
+def codes_at(buf, base, ptr, W, H, X, Y, C):
+    """e4m3 codes of tokens (X, Y) (in bounds) from planes of 32 channels at ptr"""
+    c = np.arange(C)
+    off = ptr - base + (((c[None, :] >> 5) * H + Y[:, None]) * W + X[:, None]) * 32 + (c[None, :] & 31)
+    return buf[off]
+
+
+def main():
+    args = [a for a in sys.argv[1:] if not a.startswith('--')]
+    n = 8
+    if '--blocks' in sys.argv:
+        n = int(sys.argv[sys.argv.index('--blocks') + 1])
+        args.remove(str(n))
+    dump, outs = args[0], args[1:]
+    P = me.Params(dump)
+    layer = P.kernel.replace('rrlite_', '').replace('_4x4', '')
+    C, heads, NPM, CIN = LAYERS[layer]
+    bl = blocks(P.grid, n)
+    models = {b: model_block(P, layer, *b) for b in bl}
+    for out_dir in outs:
+        got, want = [], []
+        for (bx, by) in bl:
+            base, _ = P.alloc_of(P.out)
+            aid = next(i for i, (b0, buf) in P.allocs.items() if b0 == base)
+            buf = np.fromfile(f'{out_dir}/alloc-{aid}.bin', np.uint8)
+            X, Y = P.token_xy(bx, by)
+            ok = (X >= 0) & (X < P.W) & (Y >= 0) & (Y < P.H)
+            out, merged = models[(bx, by)]
+            got.append(codes_at(buf, base, P.out, P.W, P.H, X[ok], Y[ok], C).ravel())
+            want.append(out[ok].ravel())
+            if NPM:
+                W2, H2 = P.W // 2, P.H // 2
+                r = np.arange(16)
+                MX, MY = (8 * bx - P.sx) // 2 + (r & 3), (8 * by - P.sy) // 2 + (r >> 2)
+                okm = (MX >= 0) & (MX < W2) & (MY >= 0) & (MY < H2)
+                mbase, _ = P.alloc_of(P.p48)
+                mid = next(i for i, (b0, _) in P.allocs.items() if b0 == mbase)
+                mbuf = buf if mid == aid else np.fromfile(f'{out_dir}/alloc-{mid}.bin', np.uint8)
+                got.append(codes_at(mbuf, mbase, P.p48, W2, H2, MX[okm], MY[okm], NPM).ravel())
+                want.append(merged[okm].ravel())
+        g = me.E4M3[np.concatenate(got)]
+        w = np.concatenate(want).astype(np.float64)
+        rank = lambda v: np.searchsorted(me._POS, np.abs(v)) * np.sign(v)  # noqa: E731
+        steps = np.abs(rank(g) - rank(w))
+        psnr = 10 * np.log10(np.nanmax(np.abs(w)) ** 2 / max(np.nanmean((g - w) ** 2), 1e-30))
+        print(f'{layer:9s} {os.path.basename(os.path.normpath(out_dir)):10s} {len(bl)} blocks {g.size} codes: '
+              f'identical {100 * (g == w).mean():5.1f}%  <=1 step {100 * (steps <= 1).mean():6.2f}%  '
+              f'max {int(steps.max())} steps  psnr {psnr:.1f} dB')
+
+
+if __name__ == '__main__':
+    main()

kernels/tools/swin_model.py

@@ -98,7 +98,7 @@ def layout(C, heads, pre=0):
     L['end'] = M0 + 66 * C + 64 + (64 * C + 64) * (NC - 1)
     return L
 
-def swin_block(P, x0, C, heads, pre=0):
+def swin_block(P, x0, C, heads, pre=0, stages=None):
     """x0: 64 x C natural (f16 values). Returns (x2 pair order, out natural fp8 values)."""
     fp8 = lambda T: E4M3[P.wbuf[T]]
     L = layout(C, heads, pre)
@@ -116,6 +116,8 @@ def swin_block(P, x0, C, heads, pre=0):
         Wo[32 * s:32 * s + 32] = woff_table(L['head'][s] + 64 * C + 512, 0, 512, 32, C)
     x1 = mma_chain(pair_to_nat(q8(O)), fp8(Wo), C0=f16(nat_to_pair(x0) + P.f16vec(L['bo'], C)[None, :]))
     h2 = pair_to_nat(rms_from_sum(x1, tree_norm2(x1), P.f16vec(L['g2'], C)))
+    if stages is not None:
+        stages.update(h1=h1, x1=pair_to_nat(x1), h2=h2, O=pair_to_nat(q8(O)))
     x2 = f16(x1 + P.f16vec(L['b2'], C)[None, :])
     idx = gp(np.arange(32))
     for c, B1 in enumerate(L['b1']):

kernels/tools/tail_check.hip

@@ -0,0 +1,141 @@
+// Standalone check of the texture-kernel GEMM tails (kernels/tex/enc0_tail.hip, dec0_head.hip) outside the
+// ZLUDA-compiled kernels they are linked into: random e4m3 weights and activations, several tile positions
+// (including partial tiles at the image border), all outputs written to one file. Builds of different WMMA
+// layouts (gfx11, the gfx12-layout shim, gfx12, gfx12 native FP8) are then compared byte for byte (same
+// arithmetic) or by value (gfx12 rounding). The tails touch no images, so this also runs in rocjitsu.
+//
+// build: clang++ -x hip --offload-arch=ARCH -O3 [-DD4R_WMMA_LAYOUT=12] [-DD4R_TEX_FP8] -I kernels/tex
+//        kernels/tools/tail_check.hip -o tail_check           usage: tail_check OUT.bin
+#include <hip/hip_runtime.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+// the surface-store helpers of tex_common.h reference ZLUDA's helper library, which the tails do not use
+extern "C" __device__ void __zluda_ptx_impl_surfobj_b_2d_v2_b16_zero(uint64_t, int __attribute__((ext_vector_type(2))),
+                                                                     short __attribute__((ext_vector_type(2))))
+{
+}
+
+// both tails define the same hook globals
+#define D4R_KERNEL_NAME tail_enc0
+#define d4r_prep_blocks d4r_prep_blocks_enc0
+#define d4r_prep_key_at d4r_prep_key_at_enc0
+#include "enc0_tail.hip"
+#undef d4r_prep_blocks
+#undef d4r_prep_key_at
+#define d4r_prep_blocks d4r_prep_blocks_dec0
+#define d4r_prep_key_at d4r_prep_key_at_dec0
+#include "dec0_head.hip"
+
+extern "C" __global__ void __launch_bounds__(32) run_enc0(const uint32_t* lds_in, const uint8_t* w, uint32_t* out,
+                                                          uint32_t dimx, uint32_t dimy, int ox, int oy)
+{
+    __shared__ uint32_t s[512 + 4];
+    for (int i = threadIdx.x; i < 512; i += 32)
+        s[i] = lds_in[i];
+    __syncthreads();
+    d4r_enc0_tail((uint32_t)(uintptr_t)(__attribute__((address_space(3))) void*)s, (uint64_t)w, (uint64_t)out, dimx,
+                  dimy, ox, oy);
+}
+
+extern "C" __global__ void __launch_bounds__(32) run_dec0(const uint8_t* w, const uint32_t* in, uint32_t* staged,
+                                                          uint32_t dimx, uint32_t dimy, int ox, int oy)
+{
+    __shared__ uint32_t s[512 + 4];
+    for (int i = threadIdx.x; i < 512; i += 32)
+        s[i] = 0xdeadbeefu;
+    __syncthreads();
+    d4r_dec0_head((uint32_t)(uintptr_t)(__attribute__((address_space(3))) void*)s, (uint64_t)w, (uint64_t)in, dimx,
+                  dimy, ox, oy);
+    __syncthreads();
+    for (int i = threadIdx.x; i < 512; i += 32)
+        staged[i] = s[i];
+}
+
+#define CHECK(x)                                                                                                   \
+    do                                                                                                             \
+    {                                                                                                              \
+        hipError_t e_ = (x);                                                                                       \
+        if (e_ != hipSuccess)                                                                                      \
+        {                                                                                                          \
+            fprintf(stderr, "%s: %s\n", #x, hipGetErrorString(e_));                                                \
+            return 1;                                                                                              \
+        }                                                                                                          \
+    } while (0)
+
+static uint8_t e4m3(uint32_t r)
+{
+    // Keep the matrix values in a useful range so saturation cannot hide a wrong WMMA result.
+    const uint8_t magnitude = (r & 31) == 0 ? 0 : (uint8_t)(0x18 + (r & 31));
+    return (uint8_t)(magnitude | ((r >> 8) & 0x80));
+}
+static uint16_t f16_bits(float v)
+{
+    return __builtin_bit_cast(uint16_t, (_Float16)v);
+}
+
+int main(int argc, char** argv)
+{
+    if (argc < 2)
+    {
+        fprintf(stderr, "usage: %s OUT.bin\n", argv[0]);
+        return 2;
+    }
+    srand(12345);
+    auto rnd = []() { return (uint32_t)rand() * 2654435761u + (uint32_t)rand(); };
+    // weights: e4m3 fragments, f16 biases (enc0 at 12288, dec0 at 8192)
+    std::vector<uint8_t> w(16384 + 512);
+    for (auto& b : w)
+        b = e4m3(rnd());
+    for (int i = 0; i < 64; ++i)
+        *(uint16_t*)&w[12288 + 2 * i] = f16_bits((float)(int)(rnd() % 2001 - 1000) / 256.0f);
+    for (int i = 0; i < 128; ++i)
+        *(uint16_t*)&w[8192 + 2 * i] = f16_bits((float)(int)(rnd() % 2001 - 1000) / 256.0f);
+    const uint32_t dimx = 20, dimy = 14, W2 = dimx / 2, H2 = dimy / 2, rowstride = W2 * 8, plane = H2 * rowstride;
+    std::vector<uint32_t> lds(512), in(2 * plane);
+    for (auto& v : lds)
+        v = (uint32_t)e4m3(rnd()) | (uint32_t)e4m3(rnd()) << 8 | (uint32_t)e4m3(rnd()) << 16 | (uint32_t)e4m3(rnd()) << 24;
+    for (auto& v : in)
+        v = (uint32_t)e4m3(rnd()) | (uint32_t)e4m3(rnd()) << 8 | (uint32_t)e4m3(rnd()) << 16 | (uint32_t)e4m3(rnd()) << 24;
+
+    uint8_t *dw, *dummy;
+    uint32_t *dlds, *din, *dout, *dstaged;
+    CHECK(hipMalloc(&dw, w.size()));
+    CHECK(hipMalloc(&dlds, lds.size() * 4));
+    CHECK(hipMalloc(&din, in.size() * 4));
+    CHECK(hipMalloc(&dout, 2 * plane * 4));
+    CHECK(hipMalloc(&dstaged, 512 * 4));
+    CHECK(hipMalloc(&dummy, 512));
+    CHECK(hipMemcpy(dw, w.data(), w.size(), hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(dlds, lds.data(), lds.size() * 4, hipMemcpyHostToDevice));
+    CHECK(hipMemcpy(din, in.data(), in.size() * 4, hipMemcpyHostToDevice));
+    // the prep kernels read the weight pointer at parameter offset 0
+    Enc0Params pe{};
+    pe.w = (uint64_t)dw;
+    Dec0Params pd{};
+    pd.w = (uint64_t)dw;
+    hipLaunchKernelGGL(tail_enc0_prep, dim3(4), dim3(128), 0, 0, pe);
+    hipLaunchKernelGGL(rrlite_dec0_4x4_folded_prep, dim3(4), dim3(128), 0, 0, pd);
+    CHECK(hipDeviceSynchronize());
+
+    FILE* f = fopen(argv[1], "wb");
+    const int offsets[][2] = {{0, 0}, {8, 4}, {12, 6}, {16, 8}, {-4, -2}, {18, 12}};
+    std::vector<uint32_t> host(2 * plane);
+    for (const auto& o : offsets)
+    {
+        CHECK(hipMemset(dout, 0x5a, 2 * plane * 4));
+        hipLaunchKernelGGL(run_enc0, dim3(1), dim3(32), 0, 0, dlds, dw, dout, dimx, dimy, o[0], o[1]);
+        CHECK(hipDeviceSynchronize());
+        CHECK(hipMemcpy(host.data(), dout, 2 * plane * 4, hipMemcpyDeviceToHost));
+        fwrite(host.data(), 4, 2 * plane, f);
+        hipLaunchKernelGGL(run_dec0, dim3(1), dim3(32), 0, 0, dw, din, dstaged, dimx, dimy, o[0], o[1]);
+        CHECK(hipDeviceSynchronize());
+        CHECK(hipMemcpy(host.data(), dstaged, 512 * 4, hipMemcpyDeviceToHost));
+        fwrite(host.data(), 4, 512, f);
+    }
+    fclose(f);
+    printf("wrote %s\n", argv[1]);
+    return 0;
+}

packaging/D4R_README.txt

@@ -1,19 +1,23 @@
-d4r @VERSION@: NVIDIA DLSS on AMD Radeon RDNA3 under Linux and Proton
+d4r @VERSION@: NVIDIA DLSS on AMD Radeon RDNA3 and RDNA4 under Linux and Proton
 =====================================================================
 
-d4r runs NVIDIA's own DLSS Super Resolution (DLSS 4 by default) on AMD Radeon RX 7000 GPUs in
+d4r runs NVIDIA's own DLSS Super Resolution (DLSS 4 by default) on AMD Radeon RDNA3 and RDNA4 GPUs in
 DirectX 12 games under Proton. OptiScaler (included) catches the game's DLSS calls and hands them to
 d4r. d4r then runs NVIDIA's DLSS on the Radeon through ZLUDA, with the heaviest parts replaced by
-kernels written for RDNA3.
+kernels written for RDNA3 and RDNA4.
 
-This is an early release. It has been tested on one GPU (Radeon RX 7700 XT) in
-SILENT HILL Townfall and Ghost of Tsushima DIRECTOR'S CUT. Expect problems in other games.
+This is an early release, tested on one GPU (Radeon RX 7700 XT) in about a dozen DirectX 12 games.
+SUPPORTED_GAMES.md in the source repository lists them, with per-game setup notes. Expect problems
+in other games.
 
 
 What you need
 -------------
-- An AMD Radeon RX 7000 series GPU (RDNA3). The fast kernels are built for gfx1101
-  (RX 7700 XT, RX 7800 XT). Other RDNA3 cards run DLSS without them, which is much slower.
+- An AMD RDNA3 or RDNA4 GPU. This build includes native DLSS 4 and 4.5 network kernels for
+  gfx1100–gfx1103 and gfx1200–gfx1201 unless the builder selected fewer targets. Only the RX 7700 XT
+  (gfx1101) has been tested on a real GPU. gfx1201 has emulator testing; gfx1200 has compile testing
+  only. Neither establishes RDNA4 runtime support, performance or driver stability. Texture-kernel
+  optimisations are included only for targets supplied by the builder.
 - Linux with the amdgpu kernel driver (/dev/kfd). ROCm itself is not needed: the zip includes the
   ROCm 7.2.4 runtime in d4r/rocm. RocmDir in d4r/d4r.ini selects another ROCm installation instead.
 - GE-Proton 11 (tested: GE-Proton11-3), selected for the game in Steam.
@@ -33,6 +37,7 @@ Install
 1. Extract everything in this zip into the folder that holds the game's main .exe. For Unreal
    Engine games that is <game>/<Project>/Binaries/Win64/, next to <Project>-Win64-Shipping.exe.
    You get dxgi.dll, OptiScaler.ini, d3d12.dll, d3d12core.dll, this file and an d4r folder.
+   When updating, replace both d3d12 DLLs with the shim: it requires their resource-lifetime extension.
    If the folder already has a dxgi.dll or OptiScaler.ini (another OptiScaler install), move those
    out of the way first. Also remove PROTON_USE_OPTISCALER from the game's launch options if you
    used GE-Proton's built-in OptiScaler: d4r brings its own.
@@ -54,8 +59,23 @@ compiled. They are cached in ~/.cache/d4r, so later starts are quick.
 
 Settings
 --------
+- In d4r/d4r.ini, [Kernels] PreferAccuracy = true prioritizes fidelity over performance for every
+  native kernel and the translated fallback. It selects separately built accuracy kernels, restores
+  intermediate FP8 quantization and per-MMA f16 rounding, preserves denormal handling, and keeps
+  NGX's synchronization calls. It defaults to false. Restart the game after changing it; the first
+  start may compile a separate cache. Missing accuracy variants use translated kernels, never the
+  fast native set. The mode aims to match NVIDIA's output, but 1:1 RTX image quality is not proven.
 - d4r/d4r.ini: d4r's settings for this game. The main one is the DLSS model:
-  K (DLSS 4, default), E (DLSS 3 CNN, a little faster) or M (DLSS 4.5, slowest).
+  K (DLSS 4, default), E (DLSS 3 CNN, a little faster), M (DLSS 4.5), or L (DLSS 4.5
+  for Ultra Performance, especially 4K; experimental). Set Model = L and select Ultra Performance
+  in game or OptiScaler. Choosing the model does not change the render resolution.
+- On RDNA4, [Kernels] NativeFp8 is on by default. It selects native FP8 WMMA and the matching
+  gfx12-fp8 kernel folder; set it to false to use f16 widening. The FP8 path has not been tested
+  on a real RDNA4 GPU.
+- If L/M shows extra trails in motion, try [Kernels] NativeSwinEncoders = false with
+  PreferAccuracy = true. This uses original translated enc1/enc2 layers while retaining other
+  native acceleration. It reduced trails in a captured Townfall sequence on RX 7700 XT and costs
+  some GPU time. Restart after changing it; other scenes and RDNA4 hardware still need testing.
 - OptiScaler.ini: OptiScaler's settings, such as its fps overlay and the render resolution of each
   quality mode. Press Insert in game for OptiScaler's menu.
 
@@ -108,7 +128,7 @@ cache in ~/.cache/d4r can be deleted too.
 
 Licenses
 --------
-d4r is Apache License 2.0. The release also contains ZLUDA (Apache 2.0 or MIT), vkd3d-proton
+d4r is GNU General Public License, version 3 only (GPL-3.0-only). The release also contains ZLUDA (Apache 2.0 or MIT), vkd3d-proton
 (LGPL 2.1, patched) and OptiScaler 0.9.4 (GPL 3.0, unmodified). d4r/source/SOURCES.txt says where
 each file comes from, and d4r/licenses has the license texts. d4r is not affiliated with NVIDIA,
 AMD or the OptiScaler project.

packaging/RELEASE_NOTES.md

@@ -1,65 +1,63 @@
-# d4r 0.1.0
+# d4r 0.1.3
 
-NVIDIA's own DLSS Super Resolution on AMD Radeon RX 7000 (RDNA3) GPUs, in DirectX 12 games under Proton. This is the first release packaged as a drop-in zip, to install extract it next to the game's executable and set one line of launch options.
+This release adds an optional accuracy mode and fixes the GLIBC compatibility failure, DLSS artifacts, and stale output-buffer redirects reported after 0.1.2. Accuracy mode is **off by default**.
 
-It is an early release, tested on one GPU (RX 7700 XT) in SILENT HILL Townfall and Ghost of Tsushima DIRECTOR'S CUT.
+## Added
 
-## Install
+- **Prefer accuracy over performance.** Set `[Kernels] PreferAccuracy = true` in `d4r/d4r.ini` and restart the game. The release includes separate accuracy variants of every native network and texture kernel for all supported RDNA3 and RDNA4 targets, including RDNA4's native FP8 variants.
+  - K restores FP16 accumulator rounding after each matrix step. M also restores intermediate FP8 quantization.
+  - The accuracy path retains denormal handling, disables relaxed accumulation/fast-math switches, and keeps NGX's synchronization calls. It uses a separate compilation cache.
+  - Missing accuracy variants use translated NVIDIA kernels and are logged; the fast native set is never silently substituted.
+  - This aims to preserve NVIDIA's arithmetic. It is slower, its FPS cost has not been benchmarked, and **1:1 image quality against RTX DLSS is not proven**.
+- **Reproducible GLIBC 2.41 build environment.** A pinned Debian build image and release-build script rebuild ZLUDA/LLVM, the Wine bridge, the Windows DLLs, and both kernel sets. The ZIP includes toolchain details in `d4r/source/BUILD_INFO.txt`.
 
-1. Extract `d4r-0.1.0.zip` into the folder that holds the game's main `.exe`. For Unreal Engine games that is `<Project>/Binaries/Win64`.
-2. In Steam, force GE-Proton 11 for the game, and set these launch options:
+## Fixed
+
+- **GLIBC loader compatibility ([#6](https://github.com/countervolts/d4r/issues/6)).** The previous release required `GLIBC_2.44` from `libm.so.6`, preventing ZLUDA from loading on systems such as the reporter's Bazzite installation. This release is built in a GLIBC 2.41 environment; the package check rejects any bundled Linux library requiring a newer ABI.
+- **Cyberpunk 2077 preset-K black artifacts ([#5](https://github.com/countervolts/d4r/issues/5)).** Preserve FP16/FP64 denormal requirements while limiting the performance optimization to FP32. The fix covers both translated kernels and the rebuilt native texture kernels. Captured-scene replays showed substantially fewer sky artifacts; this is not a guarantee that every scene is artifact-free.
+- **Stale direct-output redirects after resource destruction ([#8](https://github.com/countervolts/d4r/issues/8)).** Reused CUDA array handles no longer inherit an earlier feature's output buffer. This addresses the quality-switch corruption reported with preset M in Spider-Man 2.
+
+## Improved
+
+- **Synchronization diagnostics and an optional blocking output wait.** `D4R_SHIM_BLOCKING_SYNC=1` yields the CPU while waiting for DLSS output, with a fallback to context synchronization. It is an opt-in mitigation for [#7](https://github.com/countervolts/d4r/issues/7); reduced CPU usage on the reporter's RDNA4 setup still needs confirmation.
+- **More useful failure logs.** Rejected input/output formats and dimensions are now recorded even after startup, with repeated errors rate-limited. Output-wait CPU and wall time are reported separately, and GPU timing queries are deferred until asynchronous work finishes.
+- **More conservative accuracy texture builds.** Strict surface conversions use separate helper functions, avoiding a compiler failure when the original denormal handling is retained.
+- **Accuracy build safeguards.** The builder refuses to label a directory containing older fast binaries as an accuracy set. Configuration tests cover every release target, missing-set fallback, invalid markers, environment precedence, and the default-off behavior.
+
+## Also included from 0.1.2
+
+These capabilities remain in the release; they are not newly validated hardware results:
+
+- Experimental RDNA4 builds for gfx1200/gfx1201, with `NativeFp8` enabled by default, alongside gfx1100–gfx1103 RDNA3 builds.
+- Kernel selection uses the GPU with the most SIMDs when integrated and discrete GPUs coexist; `D4R_GPU_ARCH` overrides it.
+- DLSS 3 matrix-kernel acceleration; M's cached weight preparation, packed FP8 encoding, improved patch-merge/weight loads, and direct output; K's CU-mode layers; wave64 performance texture variants.
+- Native texture/output variants for the game's motion-vector resolution, HDR/LDR input and depth direction, compiled offline per GPU target.
+- The bundled ROCm 7.2.4 runtime: no separate ROCm installation is required.
+
+The 0.1.2 performance measurements remain historical results. They are not benchmarks of this rebuild or the new accuracy mode.
+
+## Install or upgrade
+
+1. Back up your `d4r/d4r.ini` and `OptiScaler.ini` if you customized them.
+2. Extract `d4r-0.1.3.zip` into the folder holding the game's main `.exe` (`<Project>/Binaries/Win64` for Unreal Engine games), keeping the ZIP's layout.
+3. Restore your settings if needed. Older INIs omit `PreferAccuracy`, which defaults to off; add it under `[Kernels]` to enable it.
+4. Select GE-Proton 11 in Steam (tested integration: GE-Proton11-3), with:
    ```
    PROTON_FORCE_NVAPI=1 DXVK_NVAPI_GPU_ARCH=AD100 %command%
    ```
-3. Select DLSS in the game. The first start compiles DLSS's GPU kernels, which takes about a minute and a half; later starts use the cache in `~/.cache/d4r`.
+5. Select DLSS in the game and restart after changing the accuracy option.
 
-`D4R_README.txt` in the zip has the details, and `sh d4r/d4r-check.sh` checks an install.
+The rebuilt runtime may compile a new kernel cache on first start. `sh d4r/d4r-check.sh` checks the install. [SUPPORTED_GAMES.md](https://github.com/countervolts/d4r/blob/main/SUPPORTED_GAMES.md) lists game-specific setup and known issues.
 
-## Requirements
+## Validation and limits
 
-- AMD Radeon RX 7000 series. The fast kernels are built for gfx1101 (RX 7700 XT, RX 7800 XT). Other RDNA3 cards run DLSS without them, which is much slower.
-- The amdgpu kernel driver (`/dev/kfd`). The ROCm 7.2.4 runtime is included in `d4r/rocm`; nothing needs to be installed.
-- GE-Proton 11 (tested: GE-Proton11-3).
-
-## What is in the zip
-
-- **d4r:** the NGX shim, the Wine CUDA bridge, and hand-written RDNA3 kernels for the DLSS 4 (K) and DLSS 4.5 (M) network layers.
-  - Each kernel is used only when the DLSS library's code for it matches the code it was written for. Swapping in another DLSS version therefore still works, only slower.
-  - Linear 8-bit colour inputs and outputs can stay in VRAM when Vulkan supports the required blits, fixing the slow staging path seen in Ghost of Tsushima.
-  - `d4r/d4r.ini` holds the settings for that game. The default model is DLSS 4 (K); E (DLSS 3 CNN) and M (DLSS 4.5) are one line away.
-- **NVIDIA:** DLSS 310.7.0 (`nvngx_dlss.dll`), the NGX runtime from driver 596.36 (`_nvngx.dll`), and five DLSS kernels built from NVIDIA's code with parts replaced by d4r's.
-- **ZLUDA** with d4r's patches: CUDA on ROCm.
-- **ROCm 7.2.4 runtime** (HIP, HSA, comgr), unmodified, from AMD's Ubuntu 22.04 packages.
-- **vkd3d-proton** with d4r's patch, so every frame shows its own DLSS result (no added latency).
-- **OptiScaler 0.9.4**, unmodified, set up to hand the game's DLSS calls to d4r.
-
-Nothing in the Proton prefix or the system is changed. Uninstalling means deleting the extracted files.
-
-## Performance
-
-Radeon RX 7700 XT, SILENT HILL Townfall at 2560×1440, average fps over the same 62-second walk:
-
-| Mode | DLSS 3 CNN (E) | DLSS 4 (K) | DLSS 4.5 (M) | FSR 4 |
-|---|---|---|---|---|
-| Quality | 72 | 69 | 51 | 76 |
-| Balanced | 80 | 76 | 59 | 84 |
-| Performance | 88 | 82 | 69 | 94 |
-| Ultra Performance | 89 | 93 | 90 | 107 |
-
-Native 2560×1440 without upscaling (the game's TSR at 100%) runs at 49 fps, so DLSS 4 at Quality is 40% faster than native.
-
-The Townfall figures were measured before the 8-bit VRAM conversion update, using the same native kernels. Ghost of Tsushima was checked in a live run with the updated shim; its VRAM path stayed on and gameplay performance improved, but no matched benchmark was recorded.
-
-## Known limitations
-
-- Only DLSS Super Resolution in DirectX 12 games. No Frame Generation or Ray Reconstruction.
-- Tested with two games. Do not use it in games with anti-cheat.
-- DLSS 4.5 (M) is much slower than FSR 4 on RDNA3.
+- All bundled Linux ELF libraries are checked against the GLIBC 2.41 ABI limit; release loading is checked inside that environment.
+- The release contains performance and accuracy kernels for all eight target variants. Configuration tests and sampled CPU-emulated kernel/reference checks cover the new accuracy path; these do not establish full-game or RTX parity.
+- RDNA4 remains experimental: emulator/compiler coverage does not prove hardware speed, image quality, or driver stability. The external RX 7900 XTX report is RDNA3, not RDNA4 validation.
+- Super Resolution only: Frame Generation, Ray Reconstruction, and DLSS 5 Neural Rendering are unsupported. Keep Ray Reconstruction off.
 
 ## Licenses
 
-d4r is Apache 2.0. The zip also contains ZLUDA (Apache 2.0 or MIT), vkd3d-proton (LGPL 2.1, patched; the patch is in `d4r/source`) and OptiScaler 0.9.4 (GPL 3.0, unmodified; source at https://github.com/optiscaler/OptiScaler/tree/v0.9.4). `d4r/source/SOURCES.txt` lists where every file comes from.
+d4r is GNU General Public License, version 3 only (GPL-3.0-only). ZLUDA, LLVM, vkd3d-proton, OptiScaler, and the bundled ROCm libraries retain their respective licenses in `d4r/licenses`; sources and patches are listed in `d4r/source`. NVIDIA's libraries and texture kernels compiled from its code are not covered by those licenses. d4r is not affiliated with NVIDIA or AMD.
 
-NVIDIA's files in the zip, and the kernels built from NVIDIA's code, belong to NVIDIA and are not covered by any of these licenses. d4r is not affiliated with NVIDIA, AMD or the OptiScaler project.
-
-**SHA-256** `d4r-0.1.0.zip`: `487f8c98c0c5617449dddd5f573bfd4b7333c8c2f5a87198d485256673cabdba`
+The attached `.sha256` file contains the ZIP's checksum.

packaging/SOURCES.txt

@@ -1,21 +1,21 @@
 d4r @VERSION@: where every file in this release comes from
 ============================================================
 
-Built from source by d4r (Apache License 2.0, d4r/licenses/d4r-LICENSE.txt):
+Built from source by d4r (GPL-3.0-only, d4r/licenses/d4r-LICENSE.txt):
   d4r/nvngx.dll          NGX shim            tools/d4r_nvngx_shim.cpp in the d4r repository
   d4r/nvcuda.dll         Wine CUDA bridge    tools/wine_nvcuda_bridge.c
-  d4r/kernels/           native RDNA3 kernels for the DLSS 4 and 4.5 network layers, kernels/k and kernels/m
+  d4r/kernels/           native RDNA3/RDNA4 kernels for the DLSS 4 and 4.5 network layers, kernels/k and kernels/m
                          (d4r-kernels.txt lists hashes of the DLSS code each kernel replaces; it holds no NVIDIA code)
   d4r/d4r.ini, d4r/d4r-check.sh, D4R_README.txt
 
 Third-party software, unmodified or patched by d4r:
   d4r/zluda/libcuda.so   ZLUDA (Apache License 2.0 or MIT), https://github.com/vosen/ZLUDA
-                         commit @ZLUDA_COMMIT@ with d4r/source/patches/zluda/0002 to 0005 applied.
+                         commit @ZLUDA_COMMIT@ with d4r/source/patches/zluda/0002 to 0007 applied.
                          It contains LLVM (Apache License 2.0 with LLVM exceptions).
                          Licenses: d4r/licenses/ZLUDA-LICENSE-*.txt, d4r/licenses/LLVM-LICENSE.txt
   d3d12.dll, d3d12core.dll
                          vkd3d-proton (GNU LGPL 2.1), https://github.com/HansKristian-Work/vkd3d-proton
-                         commit @VKD3D_COMMIT@ with d4r/source/patches/vkd3d-proton/0001 applied,
+                         commit @VKD3D_COMMIT@ with d4r/source/patches/vkd3d-proton/0001 and 0002 applied,
                          built with its build-win64.txt cross file (meson --buildtype release).
                          Licenses: d4r/licenses/vkd3d-proton-*.txt
   d4r/rocm/lib/          ROCm 7.2.4 runtime, unmodified, from AMD's Ubuntu 22.04 packages

packaging/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.3

packaging/build/Dockerfile.glibc241

@@ -0,0 +1,19 @@
+FROM debian:trixie-slim@sha256:a99cfc517144bc59b1978475ec53b46ecabec7e43635402ee5b77cc54cd1b20a
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    build-essential clang cmake ninja-build pkg-config libssl-dev python3 python3-numpy \
+    git git-lfs wine64-tools libwine-dev gcc-mingw-w64-x86-64-posix \
+    g++-mingw-w64-x86-64-posix meson libvulkan-dev zip unzip ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# ROCm loads the system's AMD DRM library even for the CPU-only loader check.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    libdrm-amdgpu1 libdrm2 mingw-w64-tools glslang-tools spirv-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/wine/winegcc /usr/local/bin/winegcc \
+    && ln -s /usr/lib/wine/winebuild /usr/local/bin/winebuild \
+    && ln -s /usr/bin/clang-19 /usr/local/bin/clang-cl \
+    && useradd --uid 1000 --create-home builder
+USER builder
+WORKDIR /work

packaging/d4r.ini

@@ -3,12 +3,19 @@
 ; that config/d4r.ini.default in the source repository lists for it, e.g. in the game's launch options.
 
 [DLSS]
-; Which DLSS network runs:
-;   K     DLSS 4 transformer (default)
-;   E     DLSS 3 CNN: a little faster, softer image
-;   M     DLSS 4.5 transformer: best image, much slower on RDNA3
-;   game  whatever the game (or OptiScaler's preset override) asks for
-; Frame rates measured on an RX 7700 XT at 2560x1440, Quality mode: see D4R_README.txt.
+; DLSS model recommendations:
+;
+; Model | DLSS  | Recommended modes               | Notes
+; ------+-------+---------------------------------+----------------------------------
+; K     | 4     | DLAA / Quality / Balanced       | Default; faster alternative to M/L
+; M     | 4.5   | Performance                     | More demanding on RDNA3
+; L     | 4.5   | Ultra Performance               | Especially 4K; experimental
+; E     | 3 CNN | Quality / Balanced / Performance | Legacy faster fallback
+; F     | 3 CNN | DLAA / Ultra Performance        | Deprecated; prefer K or L
+;
+; Select the upscaling mode separately in game/OptiScaler. These are recommendations;
+; other modes are allowed. L requires the L texture build.
+; game = use the game/OptiScaler choice. Benchmarks: D4R_README.txt.
 Model = K
 ; NVIDIA's library marks the image "DLSS SDK - DO NOT DISTRIBUTE" in this setup; d4r turns that off.
 ; true shows it.
@@ -21,16 +28,30 @@ ShowWatermark = false
 FrameAge = 0
 
 [Kernels]
-; Hand-written RDNA3 kernels for the DLSS 4 and 4.5 networks (d4r\kernels). Each one is used only
-; when your DLSS library's code for that kernel matches the code it was written for; otherwise that
-; kernel is compiled from NVIDIA's code instead, which is slower but still correct.
+; Prioritize fidelity over speed. Uses accuracy variants of all native kernels, preserves NVIDIA's
+; intermediate FP8/f16 rounding and denormal handling, and keeps NGX's synchronization calls.
+; Slower but aims to match RTX DLSS.
+PreferAccuracy = false
+; false reduces L/M motion trails by using original enc1/enc2 layers; other native kernels stay on.
+; Best paired with PreferAccuracy = true; costs some GPU time. Restart after changing this.
+NativeSwinEncoders = true
+; Hand-written RDNA3 and RDNA4 kernels for the DLSS 4 and 4.5 networks (d4r\kernels). Each one is used
+; only when your DLSS library's code for that kernel matches the code it was written for; otherwise
+; that kernel is compiled from NVIDIA's code instead, which is slower but still correct.
 NativeKernels = on
+; RDNA4 (RX 9000 series) only: run DLSS's FP8 math on the GPU's native FP8 matrix units, with NVIDIA's
+; own FP8 rounding in the DLSS 4.5 (M) layers. off runs it as 16-bit math, as on RDNA3. Ignored on
+; other GPUs.
+NativeFp8 = on
 
 [Interop]
-; These keep DLSS's inputs and output in video memory and move the waits between the game and DLSS
-; onto the GPU. Leave them on unless you are chasing a problem.
+; Experimental thin-feature tracking: 0 = off; 1/2 = opaque-surface dilation radius.
+; Both enabled values fill sky gaps from the nearest foreground within 8 render pixels.
+; Requires render- or display-resolution, unjittered motion vectors and same-frame linear VRAM inputs. Restart to change.
+; Unsupported features keep original vectors; larger radii may affect disocclusion edges.
+MotionVectorDilation = 0
+; Keep inputs and output in video memory. Input and output readiness waits use CPU sleeps.
 VramInterop = true
-GpuWait = true
 LinearInputs = true
 ElideNgxSync = true
 ; Lets DLSS's last kernel write straight into the game's output buffer (DLSS 4 / K only). It takes
@@ -51,3 +72,5 @@ Profile = false
 
 [Env]
 ; Extra environment variables for d4r, NAME = value.
+; Sleeping event queries are the default; 0 restores context waits for an A/B comparison.
+; D4R_SHIM_BLOCKING_SYNC = 0

packaging/windows/OptiScaler.ini

@@ -0,0 +1,27 @@
+; OptiScaler settings for the d4r Windows drag-in release. Values not listed keep OptiScaler's
+; defaults. The DLSS model is chosen in d4r\d4r.ini, not here.
+[Upscalers]
+Dx12Upscaler=dlss
+[FrameGen]
+Enabled=false
+FGInput=nofg
+FGOutput=nofg
+[DLSS]
+Enabled=true
+AllowExternalBackend=true
+RenderPresetOverride=false
+UseGenericAppIdWithDlss=false
+[Libraries]
+; Relative to this folder (needs d4r's OptiScaler with patch 0006).
+NvngxPath=d4r
+NvngxDlssPath=d4r
+NvngxFeaturePath=d4r
+[Hotfix]
+RestoreComputeSignature=false
+RestoreGraphicSignature=false
+ExtendedStateRestore=false
+[Log]
+LogToFile=true
+LogLevel=2
+[Menu]
+DisableSplash=true

packaging/windows/WINDOWS_README.txt

@@ -0,0 +1,63 @@
+d4r for Windows: NVIDIA DLSS on AMD Radeon RDNA3 and RDNA4
+===========================================================
+
+d4r runs NVIDIA's own DLSS (DLSS 4 by default) on a Radeon GPU in DirectX 12 games. OptiScaler
+(included) catches the game's upscaler calls and hands them to d4r.
+
+This is an early release. Only the Radeon RX 9070 XT has been tested on real hardware; every other
+GPU in this release is compile-tested. SUPPORTED_GAMES.md in the source repository lists the
+tested games. Expect problems.
+
+What you need
+-------------
+- Windows 11, a Radeon RX 7000 or RX 9000 series GPU, a current AMD Adrenalin driver.
+- A DirectX 12 game without anti-cheat (OptiScaler injects a DLL; it can get an account banned).
+- Two files from NVIDIA, which this ZIP does not include:
+    nvngx_dlss.dll  version 310.9.1 (tested) or 310.7.0. Many games ship one in their folder;
+                    right-click > Properties > Details shows the version.
+    _nvngx.dll      version 32.0.16.1714 (tested) or 32.0.15.9636. It is part of an NVIDIA display
+                    driver package: extract the installer with 7-Zip and search for _nvngx.dll.
+
+Install
+-------
+1. Extract this ZIP into the folder that holds the game's main .exe. For Unreal Engine games that
+   is <game>\<Project>\Binaries\Win64, next to <Project>-Win64-Shipping.exe.
+   If the folder already has a dxgi.dll or OptiScaler.ini (another OptiScaler install), or a
+   d3d12.dll from a Linux install, move those out first.
+2. Copy nvngx_dlss.dll into the d4r folder and _nvngx.dll into d4r\ngx.
+3. Optional: check the install. In the game folder, open PowerShell and run
+       powershell -ExecutionPolicy Bypass -File d4r\d4r-check.ps1
+4. Start the game from Steam, Epic or a shortcut as usual. In its graphics settings choose DLSS
+   (or FSR 3.1/XeSS if DLSS is hidden; OptiScaler then feeds DLSS). Keep frame generation off.
+   Press Insert in game for OptiScaler's menu.
+
+The first time DLSS starts, the game can freeze for several minutes while DLSS's GPU kernels are
+compiled. They are cached in %LOCALAPPDATA%\d4r\cache, so later starts are quick.
+
+Settings
+--------
+Edit d4r\d4r.ini (it is commented) and restart the game. Each game has its own copy.
+- Model: K (DLSS 4, default) or M (DLSS 4.5, heavier). Select the quality mode in the game.
+- AsyncInterop, ValidateOutput, Log: for testing and bug reports; the defaults are the tested setting.
+OptiScaler's own settings (overlay, render ratios) are in OptiScaler.ini.
+
+If something goes wrong
+-----------------------
+- Run d4r\d4r-check.ps1: it prints ok / MISSING / note lines with the fix for each problem.
+- d4r\d4r_nvngx.log is rewritten at every launch. A line "d4r: DLSS disabled: ..." says why.
+- DLSS missing in the menu and no d4r_nvngx.log: the files are not next to the game's main .exe.
+- For a bug report run  d4r\d4r-check.ps1 -Report  and send the ZIP it puts on your Desktop,
+  with your GPU, the game, the model and what the image looked like. Logs can contain folder names.
+- Not supported: frame generation, ray reconstruction, DirectX 11, Vulkan, models E and L.
+
+Uninstall
+---------
+Delete dxgi.dll, OptiScaler.ini, OptiScaler.log (if present), WINDOWS_README.txt and the d4r folder
+from the game folder. The cache in %LOCALAPPDATA%\d4r can be deleted too. Nothing else was changed.
+
+Licenses
+--------
+d4r is GNU General Public License, version 3 only (GPL-3.0-only). This ZIP also contains ZLUDA (Apache 2.0 or MIT), OptiScaler (GPL 3.0,
+patched; the patches are in the source repository), and AMD's HIP runtime. d4r\licenses has the
+texts. d4r is not affiliated with NVIDIA, AMD or the OptiScaler project. NVIDIA's files are yours
+to supply, under NVIDIA's terms.

packaging/windows/d4r.ini

@@ -0,0 +1,41 @@
+; d4r settings for this game (Windows). This file lives in the game's d4r folder and applies to
+; this game only. Edit it with Notepad, save, and restart the game. Mistakes are reported in
+; d4r\d4r_nvngx.log and by d4r-check.ps1; an invalid value falls back to the default shown here.
+;
+; Only settings that exist and are checked on Windows are listed. The Linux settings for other
+; models, kernels and interop are not available here.
+
+[DLSS]
+; DLSS model:
+;
+; Model | DLSS | Recommended modes          | Notes
+; ------+------+----------------------------+--------------------------------------
+; K     | 4    | DLAA / Quality / Balanced  | Default; faster
+; M     | 4.5  | Performance                | More demanding
+;
+; Select the quality mode (Quality, Performance, ...) in the game or the OptiScaler menu (Insert).
+; Models E and L are not available on Windows.
+Model = K
+; NVIDIA's library stamps "DLSS SDK - DO NOT DISTRIBUTE" over the image; d4r turns that off.
+; true shows it.
+ShowWatermark = false
+
+[Interop]
+; Experimental: lets the game continue while DLSS runs on the GPU instead of waiting for it.
+; Requires a game that uses one D3D12 command queue. Off is the tested setting. Whether it is
+; faster depends on the game and has not been measured broadly; turn it off if the game crashes,
+; stutters or shows a wrong image.
+AsyncInterop = false
+
+[Debug]
+; Check every DLSS output for invalid values (NaN/Inf) on the GPU. Costs some speed; use it only
+; when reporting a problem.
+ValidateOutput = false
+; normal: errors and a summary in d4r\d4r_nvngx.log. verbose: every API call and kernel launch
+; (large log; use it only when asked for a bug report).
+Log = normal
+
+[Paths]
+; Folder for the compiled DLSS kernels. auto = %LOCALAPPDATA%\d4r\cache. The first launch fills it
+; and can take several minutes; later launches are quick.
+CacheDir = auto

patches/optiscaler/0001-windows-external-d4r-backend.patch

@@ -0,0 +1,328 @@
+From 33bbac250ac7e2ca4eff67e8817ada9dad578208 Mon Sep 17 00:00:00 2001
+From: Codex <codex@localhost>
+Date: Thu, 1 Oct 2026 16:10:08 +0300
+Subject: [PATCH] Support explicitly selected Windows d4r DLSS backend on
+ physical AMD adapters
+
+---
+ OptiScaler/Config.cpp                         |  3 ++
+ OptiScaler/Config.h                           |  1 +
+ OptiScaler/OptiScaler.vcxproj                 |  4 +--
+ OptiScaler/State.h                            |  1 +
+ OptiScaler/Util.cpp                           | 12 +++++--
+ OptiScaler/dllmain.cpp                        | 33 +++++++++++++++++--
+ OptiScaler/hooks/LibraryLoad_Hooks.cpp        | 17 ++++++++++
+ OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp         | 17 ++++++++++
+ OptiScaler/misc/IdentifyGpu.cpp               | 20 +++++++++++
+ OptiScaler/proxies/NVNGX_Proxy.h              |  5 ++-
+ OptiScaler/upscalers/FeatureProvider_Dx12.cpp | 10 ++++++
+ 11 files changed, 115 insertions(+), 8 deletions(-)
+
+diff --git a/OptiScaler/Config.cpp b/OptiScaler/Config.cpp
+index d8c62d3..f444387 100644
+--- a/OptiScaler/Config.cpp
++++ b/OptiScaler/Config.cpp
+@@ -323,6 +323,7 @@ bool Config::Reload(std::filesystem::path iniPath)
+         {
+             // Don't enable again if set false because of no nvngx found
+             DLSSEnabled.set_from_config(readBool("DLSS", "Enabled"));
++            DLSSAllowExternalBackend.set_from_config(readBool("DLSS", "AllowExternalBackend"));
+             UseGenericAppIdWithDlss.set_from_config(readBool("DLSS", "UseGenericAppIdWithDlss"));
+ 
+             RenderPresetOverride.set_from_config(readBool("DLSS", "RenderPresetOverride"));
+@@ -1168,6 +1169,8 @@ bool Config::SaveIni()
+     // DLSS
+     {
+         ini.SetValue("DLSS", "Enabled", GetBoolValue(Instance()->DLSSEnabled.value_for_config()).c_str());
++        ini.SetValue("DLSS", "AllowExternalBackend",
++                     GetBoolValue(Instance()->DLSSAllowExternalBackend.value_for_config()).c_str());
+         ini.SetValue("DLSS", "RenderPresetOverride",
+                      GetBoolValue(Instance()->RenderPresetOverride.value_for_config()).c_str());
+         ini.SetValue("DLSS", "RenderPresetForAll",
+diff --git a/OptiScaler/Config.h b/OptiScaler/Config.h
+index 4ab62a4..0caf8c8 100644
+--- a/OptiScaler/Config.h
++++ b/OptiScaler/Config.h
+@@ -275,6 +275,7 @@ class Config
+ 
+     // DLSS
+     CustomOptional<bool> DLSSEnabled { true };
++    CustomOptional<bool> DLSSAllowExternalBackend { false };
+     CustomOptional<bool> RenderPresetOverride { false };
+     CustomOptional<uint32_t> RenderPresetForAll { 0 };
+     CustomOptional<uint32_t> RenderPresetDLAA { 0 };
+diff --git a/OptiScaler/OptiScaler.vcxproj b/OptiScaler/OptiScaler.vcxproj
+index 24ca80b..dfbfc97 100644
+--- a/OptiScaler/OptiScaler.vcxproj
++++ b/OptiScaler/OptiScaler.vcxproj
+@@ -236,7 +236,7 @@
+       <SubSystem>Windows</SubSystem>
+       <EnableCOMDATFolding>true</EnableCOMDATFolding>
+       <OptimizeReferences>true</OptimizeReferences>
+-      <GenerateDebugInformation>false</GenerateDebugInformation>
++      <GenerateDebugInformation>true</GenerateDebugInformation>
+       <EnableUAC>false</EnableUAC>
+       <AdditionalDependencies>winhttp.lib;WindowsApp.lib;Dbghelp.lib;dxgi.lib;d3d11.lib;d3d12.lib;vulkan-1.lib;dxguid.lib;freetype.lib;ffx_fsr2_api_x64.lib;ffx_fsr2_api_dx11_x64.lib;ffx_fsr2_api_dx12_x64.lib;ffx_fsr2_api_vk_x64.lib;ffx_fsr2_212_api_dx12_x64.lib;ffx_fsr2_212_api_vk_x64.lib;ffx_fsr2_212_api_x64.lib;d3dcompiler.lib;ffx_backend_dx11_x64.lib;ffx_fsr3_x64.lib;ffx_frameinterpolation_x64.lib;ffx_fsr3upscaler_x64.lib;ffx_opticalflow_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
+       <ModuleDefinitionFile>Source.def</ModuleDefinitionFile>
+@@ -873,4 +873,4 @@ copy NUL "$(SolutionDir)x64\Release\a\!! EXTRACT ALL FILES TO GAME FOLDER !!" /Y
+   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+   <ImportGroup Label="ExtensionTargets">
+   </ImportGroup>
+-</Project>
+\ No newline at end of file
++</Project>
+diff --git a/OptiScaler/State.h b/OptiScaler/State.h
+index 42164de..84f8263 100644
+--- a/OptiScaler/State.h
++++ b/OptiScaler/State.h
+@@ -265,6 +265,7 @@ class State
+     int dlssgDetectedInterpolationCount = 0;
+ 
+     // DLSS
++    bool dlssExternalBackend = false;
+     bool dlssPresetsOverriddenExternally = false;
+     bool dlssPresetsOverridenByOpti = false;
+     uint32_t dlssRenderPresetExternal = 0;
+diff --git a/OptiScaler/Util.cpp b/OptiScaler/Util.cpp
+index f7ac6fa..7367a91 100644
+--- a/OptiScaler/Util.cpp
++++ b/OptiScaler/Util.cpp
+@@ -486,8 +486,16 @@ bool Util::GetFileVersion(std::wstring dllPath, version_t* fileVersionOut, versi
+ 
+ bool Util::IsSubpath(const std::filesystem::path& path, const std::filesystem::path& base)
+ {
+-    auto rel = std::filesystem::relative(path, base);
+-    auto first = *rel.begin();
++    // Called by LdrLoadDll hooks: relative()/weakly_canonical() perform file
++    // queries, can re-enter the hooks, and throw when an optional directory or
++    // a bare DLL search name cannot be resolved. This is a lexical routing
++    // check, not a filesystem access control decision.
++    auto normalizedPath = std::filesystem::path(ToLower(path.lexically_normal().wstring()));
++    auto normalizedBase = std::filesystem::path(ToLower(base.lexically_normal().wstring()));
++    auto rel = normalizedPath.lexically_relative(normalizedBase);
++    if (rel.empty())
++        return false;
++    const auto first = *rel.begin();
+     return first != "." && first != "..";
+ }
+ 
+diff --git a/OptiScaler/dllmain.cpp b/OptiScaler/dllmain.cpp
+index cba14b1..b4b786e 100644
+--- a/OptiScaler/dllmain.cpp
++++ b/OptiScaler/dllmain.cpp
+@@ -948,7 +948,7 @@ static void CheckWorkingMode()
+ 
+     // NVAPI
+     // Doesn't seem to like GetModuleHandle for some reason, so call our load to make sure
+-    if (GetDllNameWModule(&nvapiNamesW) != nullptr)
++    if (!State::Instance().dlssExternalBackend && GetDllNameWModule(&nvapiNamesW) != nullptr)
+     {
+         // This hooks nvapi as well when possible
+         auto nvapi64 = LibraryLoadHooks::LoadNvApi();
+@@ -1958,7 +1958,33 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv
+             return nvidiaDetected;
+         };
+ 
+-        bool possibleNvidia = isNvidiaViaNvapi();
++        // Opt in only to a configured, versioned Windows d4r backend. The
++        // marker performs no HIP/NGX initialization under DLL_PROCESS_ATTACH.
++        // GPU support is still checked by its public NGX requirements API.
++        if (Config::Instance()->DLSSEnabled.value_or_default() &&
++            Config::Instance()->DLSSAllowExternalBackend.value_or_default() &&
++            Config::Instance()->NvngxPath.has_value())
++        {
++            auto backendPath = std::filesystem::path(Config::Instance()->NvngxPath.value()) / L"_nvngx.dll";
++            if (backendPath.is_absolute())
++            {
++                auto module = NtdllProxy::LoadLibraryExW_Ldr(backendPath.c_str(), nullptr,
++                    LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
++                if (module)
++                {
++                    using BackendVersion = unsigned (*)();
++                    auto version = reinterpret_cast<BackendVersion>(
++                        KernelBaseProxy::GetProcAddress_()(module, "d4r_WindowsBackendVersion"));
++                    State::Instance().dlssExternalBackend = version && version() == 1;
++                    NtdllProxy::FreeLibrary_Ldr(module);
++                }
++            }
++            if (State::Instance().dlssExternalBackend)
++                spdlog::info("Validated external Windows d4r DLSS backend; retaining physical GPU identity");
++            else
++                spdlog::error("AllowExternalBackend requires an absolute NvngxPath containing Windows d4r ABI 1");
++        }
++        bool possibleNvidia = !State::Instance().dlssExternalBackend && isNvidiaViaNvapi();
+ 
+         spdlog::info("");
+         spdlog::info("Check for DLSS files");
+@@ -1992,7 +2018,8 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv
+             State::Instance().NVNGX_DLSSG_Path = Util::FindFilePath(exePath, "nvngx_dlssg.dll");
+ 
+         // Not 100% accurate for Nvidia cards without DLSS
+-        if (Config::Instance()->DLSSEnabled.value_or_default() && possibleNvidia)
++        if (Config::Instance()->DLSSEnabled.value_or_default() &&
++            (possibleNvidia || State::Instance().dlssExternalBackend))
+         {
+             if (State::Instance().NVNGX_DLSS_Path.has_value())
+             {
+diff --git a/OptiScaler/hooks/LibraryLoad_Hooks.cpp b/OptiScaler/hooks/LibraryLoad_Hooks.cpp
+index 90bdb89..108ea5b 100644
+--- a/OptiScaler/hooks/LibraryLoad_Hooks.cpp
++++ b/OptiScaler/hooks/LibraryLoad_Hooks.cpp
+@@ -46,6 +46,18 @@ HMODULE LibraryLoadHooks::LoadLibraryCheckW(std::wstring libName, LPCWSTR lpLibF
+ {
+     auto libNameA = wstring_to_string(libName);
+ 
++    // The external D3D12 shim itself uses a locally supplied CUDA NGX core.
++    // Keep that exact absolute path distinct from game-facing NGX interception.
++    if (State::Instance().dlssExternalBackend)
++    {
++        wchar_t corePath[32768] {};
++        const auto length = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, 32768);
++        if (length && length < 32768 &&
++            Util::ToLower(std::filesystem::path(libName).lexically_normal().wstring()) ==
++                Util::ToLower(std::filesystem::path(corePath).lexically_normal().wstring()))
++            return nullptr;
++    }
++
+ #ifdef LOG_LIB_OPERATIONS
+     LOG_TRACE("{}", libNameA);
+ #endif
+@@ -132,6 +144,11 @@ HMODULE LibraryLoadHooks::LoadLibraryCheckW(std::wstring libName, LPCWSTR lpLibF
+     // NvApi64.dll
+     if (CheckDllNameW(&libName, &nvapiNamesW))
+     {
++        // An explicit external CUDA backend owns its NVAPI topology provider.
++        // Returning OptiScaler's fake-NVAPI would replace that provider and can
++        // recurse into DXGI while the NGX loader holds the loader lock.
++        if (State::Instance().dlssExternalBackend)
++            return nullptr; // Continue through the original loader.
+         LOG_INFO("{} call!", libNameA);
+ 
+         return LibraryLoadHooks::LoadNvApi();
+diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+index 68e1b42..89c9ba8 100644
+--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
++++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+@@ -687,6 +687,7 @@ static NVSDK_NGX_Result TryCreateOptiFeature(ID3D12GraphicsCommandList* InCmdLis
+     }
+ 
+     // Assign handle
++    const bool allocatedHandle = *OutHandle == nullptr;
+     if (*OutHandle == nullptr)
+         *OutHandle = new NVSDK_NGX_Handle { handleId };
+     else
+@@ -705,6 +706,16 @@ static NVSDK_NGX_Result TryCreateOptiFeature(ID3D12GraphicsCommandList* InCmdLis
+     }
+     else
+     {
++        if (state.dlssExternalBackend && upscalerBackend == Upscaler::DLSS)
++        {
++            LOG_ERROR("External DLSS initialization failed; refusing FSR fallback");
++            if (shouldRestoreSigs)
++                D3D12Hooks::RestoreRoot(InCmdList);
++            D3D12Hooks::SetRootSignatureTracking(true);
++            Dx12Contexts.erase(handleId);
++            if (allocatedHandle) { delete *OutHandle; *OutHandle = nullptr; }
++            return NVSDK_NGX_Result_Fail;
++        }
+         LOG_ERROR("Feature '{}' initialization failed falling back to FSR 2.1.2", UpscalerDisplayName(upscalerBackend));
+         state.newBackend = Upscaler::FSR21;
+         state.changeBackend[handleId] = true;
+@@ -1027,6 +1038,12 @@ static NVSDK_NGX_Result TryEvaluateOptiFeature(ID3D12GraphicsCommandList* InCmdL
+     // Fallback to FSR 2.1.2 if feature failed to initialize and user didn't explicitly request it
+     if (!feature->IsInited() && cfg.Dx12Upscaler.value_or_default() != Upscaler::FSR21)
+     {
++        if (state.dlssExternalBackend && cfg.Dx12Upscaler.value_or_default() == Upscaler::DLSS)
++        {
++            LOG_ERROR("External DLSS feature is not initialized; refusing FSR fallback");
++            D3D12Hooks::SetRootSignatureTracking(true);
++            return NVSDK_NGX_Result_Fail;
++        }
+         LOG_WARN("Feature '{}' failed to initialize. Falling back to FSR 2.1.2", feature->Name());
+         ImGui::InsertNotification({ ImGuiToastType::Warning, 10000, "Falling back to FSR 2.1.2" });
+ 
+diff --git a/OptiScaler/misc/IdentifyGpu.cpp b/OptiScaler/misc/IdentifyGpu.cpp
+index 3877f9f..6c05904 100644
+--- a/OptiScaler/misc/IdentifyGpu.cpp
++++ b/OptiScaler/misc/IdentifyGpu.cpp
+@@ -8,6 +8,7 @@
+ #include <proxies/D3d12_Proxy.h>
+ #include "nvapi/NvApiTypes.h"
+ #include <hooks/Amdxc64_Hooks.h>
++#include <proxies/NVNGX_Proxy.h>
+ 
+ using Microsoft::WRL::ComPtr;
+ 
+@@ -187,6 +188,25 @@ std::vector<GpuInformation> IdentifyGpu::checkGpuInfo()
+ 
+     for (auto& gpuInfo : localCachedInfo)
+     {
++        if (State::Instance().dlssExternalBackend && !gpuInfo.softwareAdapter)
++        {
++            // Query the configured D3D12 backend outside DLL_PROCESS_ATTACH.
++            // Preserve vendor/device/architecture data obtained from DXGI.
++            auto requirements = NVNGXProxy::D3D12_GetFeatureRequirements();
++            ComPtr<IDXGIAdapter1> physicalAdapter;
++            if (requirements && SUCCEEDED(factory->EnumAdapterByLuid(gpuInfo.luid, IID_PPV_ARGS(&physicalAdapter))))
++            {
++                NVSDK_NGX_FeatureDiscoveryInfo discovery {};
++                discovery.SDKVersion = NVSDK_NGX_Version_API;
++                discovery.FeatureID = NVSDK_NGX_Feature_SuperSampling;
++                NVSDK_NGX_FeatureRequirement supported {};
++                const auto result = requirements(physicalAdapter.Get(), &discovery, &supported);
++                gpuInfo.dlssCapable = result == NVSDK_NGX_Result_Success &&
++                    supported.FeatureSupported == NVSDK_NGX_FeatureSupportResult_Supported;
++                LOG_INFO("External D3D12 DLSS backend: {} support={} result={:X}",
++                         gpuInfo.name, gpuInfo.dlssCapable, (unsigned) result);
++            }
++        }
+         if (gpuInfo.vendorId == VendorId::Nvidia)
+         {
+             queryNvapi(gpuInfo);
+diff --git a/OptiScaler/proxies/NVNGX_Proxy.h b/OptiScaler/proxies/NVNGX_Proxy.h
+index f6c08e5..7892009 100644
+--- a/OptiScaler/proxies/NVNGX_Proxy.h
++++ b/OptiScaler/proxies/NVNGX_Proxy.h
+@@ -500,7 +500,10 @@ class NVNGXProxy
+         {
+             LOG_INFO("Getting nvngx method addresses");
+ 
+-            HookNgxApi(_module.dll);
++            // The external bridge reports physical adapter support itself.
++            // Do not overwrite its unsupported result with NVIDIA spoofing.
++            if (!State::Instance().dlssExternalBackend)
++                HookNgxApi(_module.dll);
+ 
+             _module.D3D11_Init =
+                 (PFN_D3D11_Init) KernelBaseProxy::GetProcAddress_()(_module.dll, "NVSDK_NGX_D3D11_Init");
+diff --git a/OptiScaler/upscalers/FeatureProvider_Dx12.cpp b/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
+index d738498..1191b29 100644
+--- a/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
++++ b/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
+@@ -42,6 +42,11 @@ bool FeatureProvider_Dx12::GetFeature(Upscaler upscaler, UINT handleId, NVSDK_NG
+         break;
+ 
+     case Upscaler::DLSS:
++        if (state.dlssExternalBackend && (!primaryGpu.dlssCapable || !state.NVNGX_DLSS_Path.has_value()))
++        {
++            LOG_ERROR("Configured external D3D12 DLSS backend is unavailable; refusing FSR fallback");
++            return false;
++        }
+         if (primaryGpu.dlssCapable && state.NVNGX_DLSS_Path.has_value())
+         {
+             *feature = std::make_unique<DLSSFeatureDx12>(handleId, parameters);
+@@ -77,6 +82,11 @@ bool FeatureProvider_Dx12::GetFeature(Upscaler upscaler, UINT handleId, NVSDK_NG
+ 
+     if (!loaded)
+     {
++        if (state.dlssExternalBackend && upscaler == Upscaler::DLSS)
++        {
++            LOG_ERROR("External D3D12 DLSS module failed to load; refusing FSR fallback");
++            return false;
++        }
+         // Fail after the constructor
+         ImGui::InsertNotification({ ImGuiToastType::Warning, 10000, "Falling back to FSR 2.1.2" });
+         *feature = std::make_unique<FSR2FeatureDx12_212>(handleId, parameters);
+-- 
+2.55.0.windows.5
+

patches/optiscaler/0002-early-d3d12-command-backend.patch

@@ -0,0 +1,45 @@
+From 7ed1f1d66b430ba35a197fbd9880cd6a76af8f06 Mon Sep 17 00:00:00 2001
+From: Codex <codex@localhost>
+Date: Thu, 1 Oct 2026 16:45:01 +0300
+Subject: [PATCH] Install the external d4r command backend before NGX
+ initialization
+
+---
+ OptiScaler/hooks/D3D12_Hooks.cpp | 14 ++++++++++++++
+ 1 file changed, 14 insertions(+)
+
+diff --git a/OptiScaler/hooks/D3D12_Hooks.cpp b/OptiScaler/hooks/D3D12_Hooks.cpp
+index 29c0bec..3b757be 100644
+--- a/OptiScaler/hooks/D3D12_Hooks.cpp
++++ b/OptiScaler/hooks/D3D12_Hooks.cpp
+@@ -14,6 +14,7 @@
+ #include <proxies/IGDExt_Proxy.h>
+ #include <proxies/Streamline_Proxy.h>
+ #include <proxies/KernelBase_Proxy.h>
++#include <proxies/NVNGX_Proxy.h>
+ 
+ #include <detours/detours.h>
+ 
+@@ -2230,6 +2231,19 @@ static void HookToDevice(ID3D12Device* InDevice)
+     if (Util::CheckForRealObject(__FUNCTION__, InDevice, (IUnknown**) &realDevice))
+         pVTable = *(PVOID**) realDevice;
+ 
++    if (State::Instance().dlssExternalBackend)
++    {
++        // Capture command signatures/resources created before the game's NGX
++        // Init. Install before adding OptiScaler's own method trampolines.
++        using InstallCommandBackend = unsigned (*)(ID3D12Device*);
++        auto install = reinterpret_cast<InstallCommandBackend>(KernelBaseProxy::GetProcAddress_()(
++            NVNGXProxy::NVNGXModule(), "d4r_D3D12_InstallCommandBackend"));
++        auto result = install ? install(realDevice ? realDevice : InDevice) : 0;
++        LOG_INFO("External d4r command backend installed at device discovery: result={:X}", result);
++        if (result != 1)
++            LOG_ERROR("External d4r early command tracking failed; DLSS initialization must be checked");
++    }
++
+     // hudless
+     o_D3D12DeviceRelease = (PFN_Release) pVTable[2];
+     o_CreateSampler = (PFN_CreateSampler) pVTable[22];
+-- 
+2.55.0.windows.5
+

patches/optiscaler/0003-legacy-project-initialization.patch

@@ -0,0 +1,37 @@
+From 09488c47855aadbf85f68ab8c72c0f3ac5f915e9 Mon Sep 17 00:00:00 2001
+From: Codex <codex@localhost>
+Date: Thu, 1 Oct 2026 17:01:32 +0300
+Subject: [PATCH] Preserve legacy ProjectID initialization in the external
+ backend
+
+---
+ OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp | 6 ++++++
+ 1 file changed, 6 insertions(+)
+
+diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+index 89c9ba8..1f723f1 100644
+--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
++++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+@@ -313,6 +313,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_ProjectID(const char* InProj
+ 
+             if (result == NVSDK_NGX_Result_Success)
+                 NVNGXProxy::SetDx12Inited(true);
++            else if (State::Instance().dlssExternalBackend)
++                return result;
+         }
+     }
+ 
+@@ -343,6 +345,10 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_with_ProjectID(
+ {
+     LOG_FUNC();
+ 
++    if (State::Instance().dlssExternalBackend)
++        return NVSDK_NGX_D3D12_Init_ProjectID(InProjectId, InEngineType, InEngineVersion,
++                                               InApplicationDataPath, InDevice, InSDKVersion, InFeatureInfo);
++
+     LOG_INFO("InProjectId: {0}", InProjectId);
+     LOG_INFO("InEngineType: {0}", (int) InEngineType);
+     LOG_INFO("InEngineVersion: {0}", InEngineVersion);
+-- 
+2.55.0.windows.5
+

patches/optiscaler/0004-serialize-device-hook-installation.patch

@@ -0,0 +1,34 @@
+diff --git a/OptiScaler/hooks/D3D12_Hooks.cpp b/OptiScaler/hooks/D3D12_Hooks.cpp
+index 3b757be..05fc194 100644
+--- a/OptiScaler/hooks/D3D12_Hooks.cpp
++++ b/OptiScaler/hooks/D3D12_Hooks.cpp
+@@ -2217,11 +2217,20 @@ static HRESULT hkD3D12GetInterface(REFCLSID rclsid, REFIID riid, void** ppvDebug
+     return result;
+ }
+ 
++// Device discovery and NGX initialization can overlap. Installing d4r's
++// command hooks can also create a device and re-enter this function. Never
++// overwrite live Detours trampolines from a second installation attempt.
++static std::recursive_mutex deviceHookMutex;
++static bool deviceHookInstalling = false;
+ static void HookToDevice(ID3D12Device* InDevice)
+ {
+-    if (o_CreateSampler != nullptr || InDevice == nullptr)
++    std::lock_guard<std::recursive_mutex> lock(deviceHookMutex);
++    if (o_CreateSampler != nullptr || InDevice == nullptr || deviceHookInstalling)
+         return;
+ 
++    deviceHookInstalling = true;
++    struct InstallationGuard { ~InstallationGuard() { deviceHookInstalling = false; } } installationGuard;
++
+     LOG_DEBUG("Dx12");
+ 
+     // Get the vtable pointer
+@@ -2334,6 +2343,7 @@ static void HookToDevice(ID3D12Device* InDevice)
+ 
+ static void UnhookDevice()
+ {
++    std::lock_guard<std::recursive_mutex> lock(deviceHookMutex);
+     LOG_FUNC();
+ 
+     DetourTransactionBegin();

patches/optiscaler/0005-forward-numeric-init-errors.patch

@@ -0,0 +1,22 @@
+diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+index 1f723f1..ef5ec77 100644
+--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
++++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+@@ -184,6 +184,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_Ext(unsigned long long InApp
+ 
+             if (result == NVSDK_NGX_Result_Success)
+                 NVNGXProxy::SetDx12Inited(true);
++            else if (State::Instance().dlssExternalBackend)
++                return result;
+         }
+         else
+         {
+@@ -254,6 +256,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init(unsigned long long InApplica
+ 
+             if (result == NVSDK_NGX_Result_Success)
+                 NVNGXProxy::SetDx12Inited(true);
++            else if (State::Instance().dlssExternalBackend)
++                return result;
+         }
+     }
+ 

patches/optiscaler/0006-relative-d4r-library-paths.patch

@@ -0,0 +1,30 @@
+Subject: [PATCH] Resolve relative d4r library paths against the OptiScaler folder
+
+Lets a drag-in d4r install ship OptiScaler.ini with NvngxPath=d4r instead of
+an absolute path that only an installer could know. Absolute paths behave
+exactly as before.
+---
+diff --git a/OptiScaler/Config.cpp b/OptiScaler/Config.cpp
+--- a/OptiScaler/Config.cpp
++++ b/OptiScaler/Config.cpp
+@@ -787,9 +787,17 @@
+         {
+             MainDllPath.set_from_config(readWString("Libraries", "OptiDllPath"));
+ 
+-            NvngxPath.set_from_config(readWString("Libraries", "NvngxPath"));
+-            NVNGX_DLSS_Library.set_from_config(readWString("Libraries", "NvngxDlssPath"));
+-            DLSSFeaturePath.set_from_config(readWString("Libraries", "NvngxFeaturePath"));
++            // A relative library folder (d4r drag-in installs use "d4r") is resolved
++            // against this DLL's folder, as LogFileName is. Absolute paths are unchanged.
++            auto libraryPath = [](std::optional<std::wstring> value) -> std::optional<std::wstring>
++            {
++                if (!value.has_value() || value->empty() || std::filesystem::path(*value).is_absolute())
++                    return value;
++                return (Util::DllPath().parent_path() / *value).lexically_normal().wstring();
++            };
++            NvngxPath.set_from_config(libraryPath(readWString("Libraries", "NvngxPath")));
++            NVNGX_DLSS_Library.set_from_config(libraryPath(readWString("Libraries", "NvngxDlssPath")));
++            DLSSFeaturePath.set_from_config(libraryPath(readWString("Libraries", "NvngxFeaturePath")));
+             NvapiDllPath.set_from_config(readWString("Libraries", "NvapiPath"));
+ 
+             FfxDx12Path.set_from_config(readWString("Libraries", "FfxDx12Path"));

patches/optiscaler/README.md

@@ -0,0 +1,62 @@
+# Explicit Windows d4r backend
+
+Base: OptiScaler `45a2001303ddff632e279f77aef85ceede5832cb` (2026-09-30).
+
+`0004` serializes D3D12 device hook installation/removal and guards recursive
+device discovery while the d4r command backend installs. The previous frontend
+can start two installations, fail a Detours transaction with `0x10dd`, clear
+live original-function pointers and call null. This was reproduced in the
+numeric-ID standalone harness; no game was required.
+
+`0005` propagates numeric Init/Init_Ext failures for the explicitly selected
+external backend, matching the existing ProjectID behavior. FeatureNotFound
+must fail initialization instead of being hidden until repeated CreateFeature
+attempts. `Libraries.NvngxDlssPath` takes a directory, not the DLL filename;
+the Windows installer and standalone runner now use that directory.
+The modifications to OptiScaler are GPL-3.0, matching its upstream license.
+No NVIDIA DLL is supplied or patched.
+
+`0001` introduces `[DLSS] AllowExternalBackend=true`, default false. Set
+`[Libraries] NvngxPath` to the absolute directory containing the Windows d4r
+shim as `_nvngx.dll`. Discovery requires its GPU-free ABI-1 marker. Physical
+adapter support is queried through the bridge's D3D12 requirements API after
+DLL startup; vendor, LUID and HIP architecture are retained.
+
+The configured `D4R_NGX_CORE` absolute path bypasses game-facing NGX DLL
+redirection. The external backend loads its own NVAPI provider. An explicitly
+selected external DLSS backend reports initialization/load errors instead of
+silently switching to FSR. The patch fixes the upstream loader's filesystem
+exception by making IsSubpath a lexical check, and retains a PDB for diagnosis.
+
+`0002` installs the command backend at D3D12 device discovery, before NGX Init
+and before OptiScaler's own method hooks. Command signatures created during
+game startup therefore retain their public argument descriptions for replay.
+The standalone runner's `-EarlyIndirectProbe` exercises this startup order.
+
+`0003` preserves the Project ID and engine identity through OptiScaler's
+older `Init_with_ProjectID` entry point used by its XeSS/FSR inputs. It also
+propagates an explicit external backend initialization failure. Reproduce
+with the D3D12 runner's `-NgxAbi project-legacy`.
+
+Build with Microsoft Build Tools 2022, v143 and Windows SDK 10.0.26100:
+
+```powershell
+.\scripts\windows\build-optiscaler-windows.ps1
+```
+
+The script clones the pinned source if absent, checks/applies the patch,
+initializes pinned submodules, invokes MSBuild and writes the DLL, PDB, license
+and build metadata into `dist/optiscaler-windows-d4r`. Existing source is not
+reset. The upstream packaging post-build event is disabled for this build.
+The official bootstrapper is https://aka.ms/vs/17/release/vs_buildtools.exe;
+install the `Microsoft.VisualStudio.Workload.VCTools`,
+`Microsoft.VisualStudio.Component.VC.Tools.x86.x64` and
+`Microsoft.VisualStudio.Component.Windows11SDK.26100` components. The audited
+local build uses MSVC 14.44.35207 / Build Tools 17.14.
+
+For standalone verification add
+`-OptiScalerDll <absolute-dist-path>/OptiScaler.dll -CaptureExceptions`
+to the documented K/M runner. Its private INI selects DLSS, disables frame
+generation and keeps RestoreComputeSignature, RestoreGraphicSignature and
+ExtendedStateRestore false. Use `-RequireNativeNetwork`; finite output alone
+cannot distinguish DLSS from an unwanted FSR replacement.

patches/rocm/0001-external-memory-view-ownership.patch

@@ -0,0 +1,25 @@
+From: d4r Windows port
+Subject: [PATCH] Backport AMD external-memory view ownership fix to ROCm 7.2
+
+Upstream fix: ROCm/clr 529f6b1641de436dafbb095d5438b0fbf773765d
+(ROCm/rocm-systems #3528). Already present in current upstream.
+The constructor owns reference 1. MemObjMap does not retain the view;
+hipFree removes the entry and releases reference 1. The extra retain
+leaks the view and its parent imported allocation.
+
+Hardware regression: d4r_d3d12_hip_interop_probe --interop-mode map
+--iterations 64. Windows SDK 7.2.60201 leaks one HANDLE / 256 KiB each
+cycle; tested TheRock 10.2.0a20260929 (HIP 7.17.26386) stays bounded.
+This patch is for rebuilding old runtime sources, never binary patching.
+
+diff --git a/hipamd/src/hip_memory.cpp b/hipamd/src/hip_memory.cpp
+--- a/hipamd/src/hip_memory.cpp
++++ b/hipamd/src/hip_memory.cpp
+@@ -179,7 +179,6 @@ hipError_t hipExternalMemoryGetMappedBuffer(void** devPtr, hipExternalMemory_t extMem,
+   // Map the device memory to the user pointer
+   *devPtr = reinterpret_cast<void*>(devMem->virtualAddress());
+   amd::MemObjMap::AddMemObj(*devPtr, view);
+-  view->retain();
+ 
+   HIP_RETURN(hipSuccess);
+ }

patches/vkd3d-proton/0002-retain-external-resources.patch

@@ -0,0 +1,148 @@
+diff --git a/include/vkd3d_device_vkd3d_ext.idl b/include/vkd3d_device_vkd3d_ext.idl
+--- a/include/vkd3d_device_vkd3d_ext.idl
++++ b/include/vkd3d_device_vkd3d_ext.idl
+@@ -120,6 +120,19 @@
+ }
+ 
+ [
++    uuid(4f8c9462-8197-4dc1-a584-cf44d9706db2),
++    object,
++    local,
++    pointer_default(unique)
++]
++interface ID3D12DXVKInteropDeviceD4R2 : ID3D12DXVKInteropDeviceD4R
++{
++    /* Retain external resources until the allocator can safely reset or is
++     * destroyed. Pending submissions already retain the allocator. */
++    HRESULT RetainExternalResources(ID3D12CommandList *list, IUnknown *resources);
++}
++
++[
+     uuid(f3112584-41f9-348d-a59b-00b7e1d285d6),
+     object,
+     local,
+diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
+--- a/libs/vkd3d/command.c
++++ b/libs/vkd3d/command.c
+@@ -2791,6 +2791,24 @@
+     }
+ }
+ 
++HRESULT d3d12_command_list_retain_external_resources(struct d3d12_command_list *list, IUnknown *resources)
++{
++    struct d3d12_command_allocator *allocator = list->allocator;
++    size_t i;
++
++    if (!list->is_recording || !allocator)
++        return E_INVALIDARG;
++    for (i = 0; i < allocator->external_resources_count; i++)
++        if (allocator->external_resources[i] == resources)
++            return S_OK;
++    if (!vkd3d_array_reserve((void **)&allocator->external_resources, &allocator->external_resources_size,
++            allocator->external_resources_count + 1, sizeof(*allocator->external_resources)))
++        return E_OUTOFMEMORY;
++    IUnknown_AddRef(resources);
++    allocator->external_resources[allocator->external_resources_count++] = resources;
++    return S_OK;
++}
++
+ static bool d3d12_command_allocator_add_view(struct d3d12_command_allocator *allocator,
+         struct vkd3d_view *view)
+ {
+@@ -2858,6 +2876,10 @@
+     struct d3d12_device *device = allocator->device;
+     const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
+     unsigned int i;
++
++    for (i = 0; i < allocator->external_resources_count; ++i)
++        IUnknown_Release(allocator->external_resources[i]);
++    allocator->external_resources_count = 0;
+ 
+     for (i = 0; i < allocator->buffer_view_count; ++i)
+     {
+@@ -2993,6 +3015,7 @@
+             d3d12_command_list_allocator_destroyed(allocator->current_command_list);
+ 
+         d3d12_command_allocator_free_resources(allocator);
++        vkd3d_free(allocator->external_resources);
+         vkd3d_free(allocator->buffer_views);
+         vkd3d_free(allocator->views);
+         vkd3d_free(allocator->pipelines);
+diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c
+--- a/libs/vkd3d/device.c
++++ b/libs/vkd3d/device.c
+@@ -4658,7 +4658,8 @@
+             || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice1)
+             || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice2)
+             || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice3)
+-            || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R))
++            || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R)
++            || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R2))
+     {
+         d3d12_dxvk_interop_device_AddRef(&device->ID3D12DXVKInteropDevice_iface);
+         *object = &device->ID3D12DXVKInteropDevice_iface;
+@@ -11380,7 +11381,7 @@
+ }
+ 
+ extern CONST_VTBL struct ID3D12DeviceExt5Vtbl d3d12_device_vkd3d_ext_vtbl;
+-extern CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl;
++extern CONST_VTBL struct ID3D12DXVKInteropDeviceD4R2Vtbl d3d12_dxvk_interop_device_vtbl;
+ extern CONST_VTBL struct ID3DLowLatencyDeviceVtbl d3d_low_latency_device_vtbl;
+ extern CONST_VTBL struct IAmdExtAntiLagApiVtbl d3d_amd_ext_anti_lag_vtbl;
+ 
+diff --git a/libs/vkd3d/device_vkd3d_ext.c b/libs/vkd3d/device_vkd3d_ext.c
+--- a/libs/vkd3d/device_vkd3d_ext.c
++++ b/libs/vkd3d/device_vkd3d_ext.c
+@@ -1157,7 +1157,16 @@
+     return d3d12_command_list_split_for_external_wait(cmd_list, (VkSemaphore)vk_semaphore, value);
+ }
+ 
+-CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl =
++static HRESULT STDMETHODCALLTYPE d3d12_dxvk_interop_device_RetainExternalResources(
++        d3d12_dxvk_interop_device_iface *iface, ID3D12CommandList *list, IUnknown *resources)
++{
++    struct d3d12_command_list *cmd_list = d3d12_command_list_from_iface(list);
++    if (!cmd_list || !resources)
++        return E_INVALIDARG;
++    return d3d12_command_list_retain_external_resources(cmd_list, resources);
++}
++
++CONST_VTBL struct ID3D12DXVKInteropDeviceD4R2Vtbl d3d12_dxvk_interop_device_vtbl =
+ {
+     /* IUnknown methods */
+     d3d12_dxvk_interop_device_QueryInterface,
+@@ -1192,6 +1201,9 @@
+ 
+     /* ID3D12DXVKInteropDeviceD4R methods */
+     d3d12_dxvk_interop_device_SplitCommandListForExternalWait,
++
++    /* ID3D12DXVKInteropDeviceD4R2 methods */
++    d3d12_dxvk_interop_device_RetainExternalResources,
+ };
+ 
+ static inline struct d3d12_device *d3d12_device_from_ID3DLowLatencyDevice(d3d_low_latency_device_iface *iface)
+diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
+--- a/libs/vkd3d/vkd3d_private.h
++++ b/libs/vkd3d/vkd3d_private.h
+@@ -2758,6 +2758,10 @@
+     LONG refcount;
+     LONG internal_refcount;
+ 
++    IUnknown **external_resources;
++    size_t external_resources_size;
++    size_t external_resources_count;
++
+     D3D12_COMMAND_LIST_TYPE type;
+ 
+     struct vkd3d_view **views;
+@@ -5474,7 +5478,9 @@
+ HRESULT d3d12_command_list_split_for_external_wait(struct d3d12_command_list *list,
+         VkSemaphore semaphore, uint64_t value);
+ 
+-typedef ID3D12DXVKInteropDeviceD4R d3d12_dxvk_interop_device_iface;
++HRESULT d3d12_command_list_retain_external_resources(struct d3d12_command_list *list, IUnknown *resources);
++
++typedef ID3D12DXVKInteropDeviceD4R2 d3d12_dxvk_interop_device_iface;
+ 
+ /* ID3DLowLatencyDevice */
+ typedef ID3DLowLatencyDevice d3d_low_latency_device_iface;

patches/zluda-llvm/0001-backport-gfx1154.patch

@@ -0,0 +1,128 @@
+diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
+index e619b186d..dd3e9d92b 100644
+--- a/llvm/include/llvm/BinaryFormat/ELF.h
++++ b/llvm/include/llvm/BinaryFormat/ELF.h
+@@ -856,7 +856,7 @@ enum : unsigned {
+   EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC   = 0x054,
+   EF_AMDGPU_MACH_AMDGCN_GFX1152         = 0x055,
+   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X56   = 0x056,
+-  EF_AMDGPU_MACH_AMDGCN_RESERVED_0X57   = 0x057,
++  EF_AMDGPU_MACH_AMDGCN_GFX1154         = 0x057,
+   EF_AMDGPU_MACH_AMDGCN_GFX1153         = 0x058,
+   EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC   = 0x059,
+   EF_AMDGPU_MACH_AMDGCN_GFX1251         = 0x05a,
+diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
+index aad985926..c0aaa465b 100644
+--- a/llvm/include/llvm/TargetParser/TargetParser.h
++++ b/llvm/include/llvm/TargetParser/TargetParser.h
+@@ -109,6 +109,7 @@ enum GPUKind : uint32_t {
+   GK_GFX1151,
+   GK_GFX1152,
+   GK_GFX1153,
++  GK_GFX1154,
+ 
+   GK_GFX1200,
+   GK_GFX1201,
+diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
+index f9fda2346..70ddaae22 100644
+--- a/llvm/lib/Object/ELFObjectFile.cpp
++++ b/llvm/lib/Object/ELFObjectFile.cpp
+@@ -591,6 +591,8 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const {
+     return "gfx1152";
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153:
+     return "gfx1153";
++  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154:
++    return "gfx1154";
+ 
+   // AMDGCN GFX12.
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200:
+diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
+index d9cce1eb7..7936876f4 100644
+--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
++++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
+@@ -632,6 +632,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1151, EF_AMDGPU_MACH);
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1152, EF_AMDGPU_MACH);
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1153, EF_AMDGPU_MACH);
++    BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1154, EF_AMDGPU_MACH);
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1200, EF_AMDGPU_MACH);
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1201, EF_AMDGPU_MACH);
+     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1250, EF_AMDGPU_MACH);
+diff --git a/llvm/lib/Target/AMDGPU/GCNProcessors.td b/llvm/lib/Target/AMDGPU/GCNProcessors.td
+index c6fb31f9c..5d21894a3 100644
+--- a/llvm/lib/Target/AMDGPU/GCNProcessors.td
++++ b/llvm/lib/Target/AMDGPU/GCNProcessors.td
+@@ -304,7 +304,12 @@ def : ProcessorModel<"gfx1153", GFX11SpeedModel,
+   FeatureISAVersion11_5_3.Features
+ >;
+ 
+-// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153]
++// Backported from LLVM 7a0829e41228: actual gfx1154 features and identity.
++def : ProcessorModel<"gfx1154", GFX11SpeedModel,
++  FeatureISAVersion11_5_Common.Features
++>;
++
++// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153, gfx1154]
+ def : ProcessorModel<"gfx11-generic", GFX11SpeedModel,
+   FeatureISAVersion11_Generic.Features
+ >;
+diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+index 5a0857322..c506fc308 100644
+--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
++++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+@@ -114,6 +114,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) {
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: AK = GK_GFX1151; break;
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: AK = GK_GFX1152; break;
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153: AK = GK_GFX1153; break;
++  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154: AK = GK_GFX1154; break;
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break;
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break;
+   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250: AK = GK_GFX1250; break;
+@@ -201,6 +202,7 @@ unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {
+   case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151;
+   case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152;
+   case GK_GFX1153: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153;
++  case GK_GFX1154: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154;
+   case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;
+   case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;
+   case GK_GFX1250: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250;
+diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
+index 34b09b14b..39acbb942 100644
+--- a/llvm/lib/TargetParser/TargetParser.cpp
++++ b/llvm/lib/TargetParser/TargetParser.cpp
+@@ -172,6 +172,7 @@ constexpr GPUInfo AMDGCNGPUs[] = {
+     {{"gfx1151"},   {"gfx1151"}, GK_GFX1151, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+     {{"gfx1152"},   {"gfx1152"}, GK_GFX1152, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+     {{"gfx1153"},   {"gfx1153"}, GK_GFX1153, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
++    {{"gfx1154"},   {"gfx1154"}, GK_GFX1154, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+     {{"gfx1200"},   {"gfx1200"}, GK_GFX1200, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+     {{"gfx1201"},   {"gfx1201"}, GK_GFX1201, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+     {{"gfx1250"},   {"gfx1250"}, GK_GFX1250, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32},
+@@ -328,6 +329,7 @@ AMDGPU::IsaVersion AMDGPU::getIsaVersion(StringRef GPU) {
+   case GK_GFX1151: return {11, 5, 1};
+   case GK_GFX1152: return {11, 5, 2};
+   case GK_GFX1153: return {11, 5, 3};
++  case GK_GFX1154: return {11, 5, 4};
+   case GK_GFX1200: return {12, 0, 0};
+   case GK_GFX1201: return {12, 0, 1};
+   case GK_GFX1250: return {12, 5, 0};
+@@ -475,6 +477,7 @@ static void fillAMDGCNFeatureMap(StringRef GPU, const Triple &T,
+     Features["atomic-fmin-fmax-global-f32"] = true;
+     break;
+   case GK_GFX1153:
++  case GK_GFX1154:
+   case GK_GFX1152:
+   case GK_GFX1151:
+   case GK_GFX1150:
+diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
+index ab9331690..0ade2d412 100644
+--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
++++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
+@@ -1655,6 +1655,7 @@ const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"),                          \
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1152, "gfx1152"),                          \
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1153, "gfx1153"),                          \
++  ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1154, "gfx1154"),                          \
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"),                          \
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"),                          \
+   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1250, "gfx1250"),                          \

patches/zluda-llvm/README.md

@@ -0,0 +1,24 @@
+# LLVM source patches for the Windows compiler
+
+Base: ZLUDA's LLVM submodule `ff4dc1f7c9e1c64d4d69e40f4ed30c2280a96dfd`
+(LLVM 22). `build-zluda-llvm.ps1` applies these patches idempotently before
+building. ZLUDA's parent source patches remain in `patches/zluda`.
+
+`0001-backport-gfx1154.patch` adapts official LLVM
+[`7a0829e41228`](https://github.com/llvm/llvm-project/commit/7a0829e41228513299c5108685b0bc127463c6a1)
+to the older parser/ELF structures. It adds actual gfx1154 processor features,
+ISA 11.5.4, parser identity and ELF machine 0x57. It does not map gfx1154 onto
+another CPU, patch binary headers or alter existing processors' features.
+LLVM's gfx1154 uses `FeatureISAVersion11_5_Common` and the GFX11 speed model.
+LLVM code remains Apache-2.0 WITH LLVM-exception (see the dependency license).
+
+The prior offline compiler aborts on a public PTX pattern for gfx1154 with
+unrecognized-processor diagnostics and exit `0xC0000409`. After rebuilding
+LLVM, rebuild/relink ZLUDA and its offline emitter; changing native Clang
+objects alone cannot fix the translated PTX path. Run
+`scripts/windows/test-zluda-target-compilation.ps1` to compile the public
+Driver API pattern and FP16 MMA fixtures (strict/reference and WMMA modes)
+for every target without GPU work. All 33 compilations pass with correct ELF
+targets and wave32. Strict gfx12 F16 reference retains its prior FP64 helper;
+WMMA mode here is a compilation check, not a default arithmetic change.
+This checks code generation and ELF identity, not hardware execution/numerics.

patches/zluda/0004-native-overrides-and-gpu-handoffs.patch

@@ -1225,9 +1225,10 @@ index e6f22a6..4256ae5 100644
      Kernel(SpirvWord),
  }
  
-+// d4r: D4R_ZLUDA_IGNORE_DENORMAL=1 drops per-instruction denormal (.ftz) requirements, so a kernel
-+// keeps one denormal mode instead of switching it around ftz instructions (differences only for
-+// values below ~1e-38). Not part of the JIT cache key: use a fresh cache when toggling it.
++// d4r: D4R_ZLUDA_IGNORE_DENORMAL=1 drops only FP32 denormal (.ftz) requirements to avoid mode switches.
++// Keep FP16/FP64 requirements: flushing FP16 values below 2^-14 corrupts K's output and can make its
++// translated attention layers non-finite. The runtime fingerprints code-generation switches in the
++// JIT cache key; native texture code objects must also be rebuilt after changing this policy.
 +fn ignore_denormal_modes() -> bool {
 +    static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
 +    *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_IGNORE_DENORMAL").map_or(false, |v| v.trim() == "1"))
@@ -1237,7 +1238,6 @@ index e6f22a6..4256ae5 100644
 +    let mut modes = get_modes_inner(inst);
 +    if ignore_denormal_modes() {
 +        modes.denormal_f32 = None;
-+        modes.denormal_f16f64 = None;
 +    }
 +    // d4r: D4R_ZLUDA_DEBUG_MODES=1 prints instructions that need a non-default rounding mode
 +    if std::env::var("D4R_ZLUDA_DEBUG_MODES").map_or(false, |v| v == "1") {

patches/zluda/0006-k8-wmma-prep-slots-wave64.patch

@@ -0,0 +1,444 @@
+diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
+index 5d8c0a5..b2870e5 100644
+--- a/llvm_zluda/src/compile.rs
++++ b/llvm_zluda/src/compile.rs
+@@ -70,8 +70,10 @@ fn create_oclc_constants(ctx: &Context, gcn_arch: &str) -> Result<Module, String
+     let module = Module::new(ctx, c"oclc_constants");
+ 
+     // used by ockl
+-    add_constant(ctx, &module, c"__oclc_wavefrontsize64", 0);
+-    add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", 5);
++    // d4r: D4R_ZLUDA_WAVE64=1 (see ptx::pass::wave64_requested)
++    let wave64 = std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1");
++    add_constant(ctx, &module, c"__oclc_wavefrontsize64", if wave64 { 1 } else { 0 });
++    add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", if wave64 { 6 } else { 5 });
+     add_constant(ctx, &module, c"__oclc_ABI_version", 500);
+     add_constant(
+         ctx,
+@@ -174,7 +176,13 @@ pub fn compile(
+ 
+     let triple = c"amdgcn-amd-amdhsa";
+     let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
+-    let features = c"-wavefrontsize64,+cumode";
++    // functions without their own target features (the linked helper libraries) take these, so under
++    // D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
++    let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
++        c"-wavefrontsize32,+wavefrontsize64,+cumode"
++    } else {
++        c"-wavefrontsize64,+cumode"
++    };
+ 
+     let mut target = unsafe { std::mem::zeroed() };
+     let mut err = std::ptr::null_mut();
+diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
+index 88848b1..0d1f9cf 100644
+--- a/llvm_zluda/src/lib.cpp
++++ b/llvm_zluda/src/lib.cpp
+@@ -869,10 +869,111 @@ unsigned LLVMZludaSplitWmmaM16n16k16(LLVMModuleRef ModuleRef)
+     return static_cast<unsigned>(Calls.size());
+ }
+ 
++// d4r: an m16n8k8 f16 MMA is an m16n8k16 one whose second K half is zero (its A fragment is the k16
++// A's registers 0-1, its B the k16 B's register 0), and adding zero products is exact. Each k8 call
++// becomes a k16 call so the WMMA rewrite below lowers it too. Zero halves are loads of one zeroed
++// register, so the operands stay "loads of plain registers" and two k8 MMAs that share A still fuse
++// into one WMMA. Returns the number of calls converted.
++static unsigned widenMmaK8(Module *M)
++{
++    Function *K8 = M->getFunction("__zluda_ptx_impl_mma_sync_aligned_m16n8k8_row_col_f16_f16_f16_f16");
++    if (K8 == nullptr || K8->use_empty())
++        return 0;
++    LLVMContext &Context = M->getContext();
++    Type *I32 = Type::getInt32Ty(Context);
++    Type *V2 = FixedVectorType::get(I32, 2);
++    Type *V4 = FixedVectorType::get(I32, 4);
++    AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
++                                                  AttrBuilder(Context, K8->getAttributes().getFnAttrs()));
++    FunctionCallee K16 = M->getOrInsertFunction(kMmaF16, Attributes, V2, V4, V2, V2);
++    SmallVector<CallInst *, 64> Calls;
++    for (User *U : K8->users())
++        if (auto *Call = dyn_cast<CallInst>(U); Call != nullptr && Call->getCalledFunction() == K8)
++            Calls.push_back(Call);
++    DenseMap<Function *, AllocaInst *> Zeros;
++    for (CallInst *Call : Calls)
++    {
++        Function *F = Call->getFunction();
++        AllocaInst *&Zero = Zeros[F];
++        if (Zero == nullptr)
++        {
++            // an alloca like ZLUDA's register allocas (same address space), stored once at entry
++            AllocaInst *Like = nullptr;
++            for (Instruction &I : F->getEntryBlock())
++                if ((Like = dyn_cast<AllocaInst>(&I)) != nullptr)
++                    break;
++            IRBuilder<> Entry(&*F->getEntryBlock().getFirstInsertionPt());
++            Zero = Entry.CreateAlloca(I32, Like != nullptr ? Like->getAddressSpace() : 5, nullptr, "d4r.k8.zero");
++            Entry.CreateStore(Entry.getInt32(0), Zero);
++        }
++        IRBuilder<> Builder(Call);
++        SmallVector<AllocaInst *, 2> ARegisters, BRegisters;
++        auto build = [&](ArrayRef<Value *> Elements) {
++            Value *Vector = UndefValue::get(FixedVectorType::get(I32, Elements.size()));
++            for (unsigned Index = 0; Index < Elements.size(); ++Index)
++                Vector = Builder.CreateInsertElement(Vector, Elements[Index], Builder.getInt8(Index));
++            return Vector;
++        };
++        auto zero = [&]() { return Builder.CreateLoad(I32, Zero); };
++        // the registers may be reloaded at the call only if no store reaches them after their original loads
++        auto unchangedSinceLoads = [&](Value *Vector) {
++            for (Value *V = Vector; auto *Insert = dyn_cast<InsertElementInst>(V); V = Insert->getOperand(0))
++            {
++                auto *Load = cast<LoadInst>(Insert->getOperand(1));
++                if (Load->getParent() != Call->getParent() ||
++                    storesBetween(Load, Call, {cast<AllocaInst>(Load->getPointerOperand())}))
++                    return false;
++            }
++            return true;
++        };
++        // B is one register: a <1 x i32> vector or a plain i32
++        Value *BArg = Call->getArgOperand(1);
++        auto bRegister = [&]() {
++            if (BArg->getType()->isVectorTy())
++                return matchLoadedVector(BArg, 1, BRegisters) && unchangedSinceLoads(BArg);
++            auto *Load = dyn_cast<LoadInst>(BArg);
++            auto *Register = Load != nullptr ? dyn_cast<AllocaInst>(Load->getPointerOperand()) : nullptr;
++            if (Register == nullptr || !Register->getAllocatedType()->isIntegerTy(32) || !isPlainRegister(Register) ||
++                Load->getParent() != Call->getParent() || storesBetween(Load, Call, {Register}))
++                return false;
++            BRegisters.assign(1, Register);
++            return true;
++        };
++        auto element = [&](Value *V, unsigned Index) -> Value * {
++            return V->getType()->isVectorTy() ? Builder.CreateExtractElement(V, Builder.getInt8(Index)) : V;
++        };
++        Value *A, *B;
++        if (matchLoadedVector(Call->getArgOperand(0), 2, ARegisters) && bRegister() &&
++            unchangedSinceLoads(Call->getArgOperand(0)))
++        {
++            // reload the same registers right at the call (nothing lies between these loads and it)
++            Value *A0 = Builder.CreateLoad(I32, ARegisters[0]), *A1 = Builder.CreateLoad(I32, ARegisters[1]);
++            Value *B0 = Builder.CreateLoad(I32, BRegisters[0]);
++            A = build({A0, A1, zero(), zero()});
++            B = build({B0, zero()});
++        }
++        else
++        {
++            Value *A0 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(0));
++            Value *A1 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(1));
++            Value *B0 = element(BArg, 0);
++            A = build({A0, A1, Builder.getInt32(0), Builder.getInt32(0)});
++            B = build({B0, Builder.getInt32(0)});
++        }
++        CallInst *Wide = Builder.CreateCall(K16, {A, B, Call->getArgOperand(2)});
++        Call->replaceAllUsesWith(Wide);
++        Call->eraseFromParent();
++    }
++    return static_cast<unsigned>(Calls.size());
++}
++
+ // Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
+ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+ {
+     Module *M = unwrap(ModuleRef);
++    // D4R_ZLUDA_WMMA_K8=0 keeps m16n8k8 on the dot-product fallback
++    if (const char *K8Env = getenv("D4R_ZLUDA_WMMA_K8"); K8Env == nullptr || K8Env[0] != '0')
++        widenMmaK8(M);
+     Function *Mma = M->getFunction(kMmaF16);
+     Function *MmaE4m3 = (Flags & 1) != 0 ? M->getFunction(kMmaE4m3) : nullptr;
+     if (Mma != nullptr && Mma->use_empty())
+diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
+index cdefdf1..74cf36f 100644
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -127,9 +127,32 @@ extern "C"
+ {
+     extern "C" __attribute__((constant)) uint32_t __oclc_ISA_version __device__;
+ 
++    // d4r: kernels may be compiled as wave64 (D4R_ZLUDA_WAVE64, offline texture-kernel builds, linked with
++    // the zluda_ptx_impl*_w64.bc helpers built with -DD4R_WAVE64), where each wave holds two CUDA warps:
++    // lanes 0-31 and 32-63. Warp-level helpers then act on the lane's own 32-lane half.
++#ifdef D4R_WAVE64
++    __device__ static inline uint32_t d4r_wave_lane()
++    {
++        return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
++    }
++    __device__ static inline uint32_t d4r_warp_base()
++    {
++        return d4r_wave_lane() & 32u;
++    }
++#else
++    __device__ static inline uint32_t d4r_warp_base()
++    {
++        return 0u;
++    }
++#endif
++
+     uint32_t FUNC(activemask)()
+     {
++#ifdef D4R_WAVE64
++        return (uint32_t)(__builtin_amdgcn_read_exec() >> d4r_warp_base());
++#else
+         return __builtin_amdgcn_read_exec_lo();
++#endif
+     }
+ 
+     size_t __ockl_get_local_id(uint32_t) __device__;
+@@ -158,7 +181,11 @@ extern "C"
+ 
+     uint32_t FUNC(sreg_laneid)()
+     {
++#ifdef D4R_WAVE64
++        return d4r_wave_lane() & 31u;
++#else
+         return __lane_id();
++#endif
+     }
+ 
+     uint32_t FUNC(sreg_lanemask_le)()
+@@ -322,7 +349,7 @@ extern "C"
+         int32_t section_mask = (opts >> 8) & 0b11111;                                                                                           \
+         int32_t warp_end = opts & 0b11111;                                                                                                      \
+         delta &= 0b11111;                                                                                                                       \
+-        int32_t self = (int32_t)__lane_id();                                                                                                    \
++        int32_t self = (int32_t)__zluda_ptx_impl_sreg_laneid();                                                                                 \
+         int32_t subsection = section_mask & self;                                                                                               \
+         int32_t subsection_end = subsection | (~section_mask & warp_end);                                                                       \
+         int32_t idx = calculate_index;                                                                                                          \
+@@ -331,7 +358,7 @@ extern "C"
+         {                                                                                                                                       \
+             idx = self;                                                                                                                         \
+         }                                                                                                                                       \
+-        int32_t output = __builtin_amdgcn_ds_bpermute(idx << 2, (int32_t)input);                                                                \
++        int32_t output = __builtin_amdgcn_ds_bpermute((int32_t)(d4r_warp_base() | (uint32_t)idx) << 2, (int32_t)input);                         \
+         return {(uint32_t)output, uint32_t(!out_of_bounds)};                                                                                    \
+     }                                                                                                                                           \
+                                                                                                                                                 \
+@@ -850,7 +877,11 @@ extern "C"
+     __device__ static inline uint32_t ballot(bool value, bool negate)
+     {
+         __builtin_amdgcn_wave_barrier();
++#ifdef D4R_WAVE64
++        return (uint32_t)(__builtin_amdgcn_ballot_w64(negate ? !value : value) >> d4r_warp_base());
++#else
+         return __builtin_amdgcn_ballot_w32(negate ? !value : value);
++#endif
+     }
+ 
+     bool FUNC(vote_sync_any_pred)(bool value, uint32_t membermask __attribute__((unused)))
+@@ -866,7 +897,7 @@ extern "C"
+     // IMPORTANT: exec mask must be a subset of membermask, the behavior is undefined otherwise
+     bool FUNC(vote_sync_all_pred)(bool value, uint32_t membermask __attribute__((unused)))
+     {
+-        return ballot(value, false) == __builtin_amdgcn_read_exec_lo();
++        return ballot(value, false) == __zluda_ptx_impl_activemask();
+     }
+ 
+     // also known as "none"
+diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
+index 8294bb4..6af2495 100644
+--- a/ptx/src/pass/llvm/emit.rs
++++ b/ptx/src/pass/llvm/emit.rs
+@@ -397,7 +397,12 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
+ 
+     fn emit_target_features(&mut self, fn_: LLVMValueRef) {
+         let value = format!(
+-            "+wavefrontsize32,-wavefrontsize64,+cumode{}",
++            "{},+cumode{}",
++            if crate::pass::wave64_requested() {
++                "-wavefrontsize32,+wavefrontsize64"
++            } else {
++                "+wavefrontsize32,-wavefrontsize64"
++            },
+             if cfg!(debug_assertions) {
+                 ",+precise-memory"
+             } else {
+diff --git a/ptx/src/pass/mod.rs b/ptx/src/pass/mod.rs
+index 63aa98c..e3baa72 100644
+--- a/ptx/src/pass/mod.rs
++++ b/ptx/src/pass/mod.rs
+@@ -37,6 +37,18 @@ mod test;
+ static ZLUDA_PTX_IMPL: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl.bc");
+ static ZLUDA_PTX_IMPL_CONSTRAINED: &'static [u8] =
+     include_bytes!("../../lib/zluda_ptx_impl_constrained.bc");
++static ZLUDA_PTX_IMPL_W64: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl_w64.bc");
++static ZLUDA_PTX_IMPL_CONSTRAINED_W64: &'static [u8] =
++    include_bytes!("../../lib/zluda_ptx_impl_constrained_w64.bc");
++
++/// d4r: D4R_ZLUDA_WAVE64=1 compiles kernels as wave64: each wave runs two CUDA warps (lanes 0-31 and
++/// 32-63; the warp-level helpers act on the lane's own half) and RDNA3 issues most FP32 VALU work for all
++/// 64 lanes at once. Meant for offline builds of texture kernels whose warp operations are limited to
++/// lane ids, shuffles and votes; not for MMA/matrix kernels (their helpers assume one warp per wave).
++pub fn wave64_requested() -> bool {
++    static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
++    *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1"))
++}
+ const ZLUDA_PTX_PREFIX: &'static str = "__zluda_ptx_impl_";
+ 
+ quick_error! {
+@@ -194,10 +206,11 @@ pub struct Module {
+ 
+ impl Module {
+     pub fn linked_bitcode(&self) -> &'static [u8] {
+-        if self.constrained_fp {
+-            ZLUDA_PTX_IMPL_CONSTRAINED
+-        } else {
+-            ZLUDA_PTX_IMPL
++        match (self.constrained_fp, wave64_requested()) {
++            (true, false) => ZLUDA_PTX_IMPL_CONSTRAINED,
++            (false, false) => ZLUDA_PTX_IMPL,
++            (true, true) => ZLUDA_PTX_IMPL_CONSTRAINED_W64,
++            (false, true) => ZLUDA_PTX_IMPL_W64,
+         }
+     }
+ }
+diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
+index bacb93d..033d9a0 100644
+--- a/zluda/src/impl/function.rs
++++ b/zluda/src/impl/function.rs
+@@ -18,9 +18,14 @@ pub(crate) struct Function {
+     /// d4r native override: persistent kernels launch this many workgroups in x (0 = as launched)
+     pub(crate) grid_x: u32,
+     /// d4r native override: byte offset of the u64 parameter that keys the prep kernel's output
+-    /// (0 = run the prep before every launch) and the key of the last prep launched
+-    pub(crate) prep_key: u32,
++    /// (None = run the prep before every launch) and the key of the last prep launched
++    pub(crate) prep_key: Option<u32>,
+     pub(crate) prep_last: std::sync::atomic::AtomicU64,
++    /// d4r native override: with N > 0 prep slots the kernel keeps one prepared image per key for
++    /// up to N keys, so the prep runs only for keys not seen yet (and for every key beyond N, which
++    /// the kernel prepares into a shared overflow slot). 0 = only the last key is kept.
++    pub(crate) prep_slots: u32,
++    pub(crate) prep_seen: std::sync::Mutex<Vec<u64>>,
+ }
+ 
+ impl<'a, E: zluda_common::CudaErrorType> zluda_common::FromCuda<'a, CUfunction, E>
+@@ -140,7 +145,7 @@ pub(crate) fn launch_kernel(
+         translated_extra_storage.as_mut_ptr()
+     };
+     // the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
+-    let prep_key = if f.prep.is_some() && f.prep_key != 0 {
++    let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
+         let mut block: *const u8 = std::ptr::null();
+         if !extra.is_null() {
+             let mut i = 0usize;
+@@ -160,12 +165,27 @@ pub(crate) fn launch_kernel(
+         if block.is_null() {
+             0
+         } else {
+-            unsafe { std::ptr::read_unaligned(block.add(f.prep_key as usize) as *const u64) }
++            unsafe { std::ptr::read_unaligned(block.add(key_offset as usize) as *const u64) }
+         }
+     } else {
+         0
+     };
+-    let run_prep = prep_key == 0 || f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key;
++    let run_prep = if prep_key == 0 {
++        true
++    } else if f.prep_slots == 0 {
++        f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key
++    } else {
++        // prepared once per key while slots last; keys beyond them are prepared on every launch
++        let mut seen = f.prep_seen.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
++        if seen.contains(&prep_key) {
++            false
++        } else {
++            if seen.len() < f.prep_slots as usize {
++                seen.push(prep_key);
++            }
++            true
++        }
++    };
+     if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
+         unsafe {
+             hipModuleLaunchKernel(
+diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
+index 5db5ace..c5a39c9 100644
+--- a/zluda/src/impl/module.rs
++++ b/zluda/src/impl/module.rs
+@@ -37,12 +37,12 @@ impl ModuleMutable {
+         Ok(match self.functions.entry(name) {
+             hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
+             hash_map::Entry::Vacant(entry) => {
+-                let (func_handle, prep, block_z, grid_x, prep_key) = match native_override(entry.key()) {
++                let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
+                     Some(native) => native,
+                     None => {
+                         let mut func_handle = unsafe { std::mem::zeroed() };
+                         unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
+-                        (func_handle, None, 0, 0, 0)
++                        (func_handle, None, 0, 0, None, 0)
+                     }
+                 };
+                 let func = Box::new(Function {
+@@ -54,6 +54,8 @@ impl ModuleMutable {
+                     grid_x,
+                     prep_key,
+                     prep_last: std::sync::atomic::AtomicU64::new(0),
++                    prep_slots,
++                    prep_seen: std::sync::Mutex::new(Vec::new()),
+                 });
+                 &*entry.insert(func)
+             }
+@@ -67,7 +69,9 @@ impl ModuleMutable {
+ /// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
+ /// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
+ /// (persistent kernels, which derive the original grid from their parameters) the grid.
+-fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, u32)> {
++fn native_override(
++    name: &CStr,
++) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
+     let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
+     let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
+     let image = fs::read(&path).ok()?;
+@@ -103,34 +107,38 @@ fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t,
+     } else {
+         None
+     };
+-    let read_u32 = |symbol: &CStr| -> u32 {
++    let read_u32_opt = |symbol: &CStr| -> Option<u32> {
+         let mut ptr = unsafe { mem::zeroed() };
+         let mut size = 0usize;
+         let mut value = 0u32;
+-        if unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
++        (unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
+             && size == 4
+-            && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok()
+-        {
+-            value
+-        } else {
+-            0
+-        }
++            && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok())
++        .then_some(value)
+     };
++    let read_u32 = |symbol: &CStr| -> u32 { read_u32_opt(symbol).unwrap_or(0) };
+     let block_z = read_u32(c"d4r_block_z");
+     let grid_x = read_u32(c"d4r_grid_x");
+     // byte offset of a u64 in the parameter block that fully determines the prep kernel's output
+-    // (the weights pointer): the prep is skipped while it repeats (0 = prep before every launch)
+-    let prep_key = read_u32(c"d4r_prep_key_offset");
++    // (the weights pointer): the prep is skipped while it repeats. d4r_prep_key_at holds offset + 1
++    // (so a key at offset 0 is possible); the older d4r_prep_key_offset uses 0 for "no key".
++    let prep_key = match read_u32_opt(c"d4r_prep_key_at") {
++        Some(at) if at > 0 => Some(at - 1),
++        _ => Some(read_u32(c"d4r_prep_key_offset")).filter(|&offset| offset != 0),
++    };
++    // d4r_prep_key_slots: the kernel keeps one prepared image per key (see Function::prep_slots)
++    let prep_slots = read_u32(c"d4r_prep_key_slots");
+     eprintln!(
+-        "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {})",
++        "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {:?}, prep_slots: {})",
+         name,
+         path.display(),
+         prep.map(|p| p.1),
+         block_z,
+         grid_x,
+-        prep_key
++        prep_key,
++        prep_slots
+     );
+-    Some((func, prep, block_z, grid_x, prep_key))
++    Some((func, prep, block_z, grid_x, prep_key, prep_slots))
+ }
+ 
+ impl ZludaObject for Module {

patches/zluda/0007-gfx12-wmma-native-fp8.patch

@@ -0,0 +1,551 @@
+--- a/llvm_zluda/src/compile.rs
++++ b/llvm_zluda/src/compile.rs
+@@ -113,9 +113,57 @@
+     *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_PARAM_PROVENANCE").map_or(true, |value| value.trim() != "0"))
+ }
+ 
+-fn wmma_rewrite_enabled(gcn_arch: &str) -> bool {
+-    // The helpers assume RDNA3's wave32 WMMA register layout.
+-    wmma_rewrite_requested() && gcn_arch.starts_with("gfx11")
++/// D4R_ZLUDA_WMMA_FP8_NATIVE=1: on gfx12 (RDNA4), e4m3 MMAs use the native FP8 WMMA instead of widening
++/// to f16. Ignored on every other GPU (gfx11 has no FP8 WMMA).
++pub fn wmma_fp8_native_requested() -> bool {
++    static VALUE: OnceLock<bool> = OnceLock::new();
++    *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WMMA_FP8_NATIVE").map_or(false, |value| value.trim() == "1"))
++}
++
++/// The WMMA register layout family of a GPU: 11 (RDNA3) or 12 (RDNA4); the MMA helpers exist for these.
++/// D4R_ZLUDA_WMMA_LAYOUT=12 on gfx11 is for tests only: it takes the gfx12 helper names, which then must
++/// come from a helper library built with D4R_WMMA12_SHIM (gfx12 data movement, gfx11 WMMAs).
++fn wmma_family(gcn_arch: &str) -> Option<u32> {
++    let family = if gcn_arch.starts_with("gfx11") {
++        11
++    } else if gcn_arch.starts_with("gfx12") {
++        12
++    } else {
++        return None;
++    };
++    if family == 11 && std::env::var("D4R_ZLUDA_WMMA_LAYOUT").map_or(false, |value| value.trim() == "12") {
++        return Some(12);
++    }
++    Some(family)
++}
++
++/// Whether e4m3 MMAs compiled for this GPU use native FP8 WMMA (the setting only acts on gfx12).
++pub fn wmma_fp8_native_effective(gcn_arch: &str) -> bool {
++    wmma_rewrite_flags(gcn_arch).map_or(false, |flags| flags & 4 != 0)
++}
++
++/// Flags for LLVMZludaRewriteMmaF16Wmma, or None to keep the dot-product fallback (logged once when the
++/// rewrite was requested but this GPU has no WMMA lowering).
++fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
++    if !wmma_rewrite_requested() {
++        return None;
++    }
++    let Some(family) = wmma_family(gcn_arch) else {
++        static LOGGED: OnceLock<()> = OnceLock::new();
++        LOGGED.get_or_init(|| {
++            eprintln!("[ZLUDA] D4R_ZLUDA_WMMA: no WMMA lowering for {gcn_arch}; MMAs use the dot-product fallback")
++        });
++        return None;
++    };
++    let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
++    if family == 12 {
++        flags |= 2;
++        // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
++        if flags & 1 != 0 && wmma_fp8_native_requested() {
++            flags |= 4;
++        }
++    }
++    Some(flags)
+ }
+ 
+ pub fn compile(
+@@ -139,8 +187,7 @@
+ 
+     // wmma.mma m16n16k16 always becomes m16n8k16 MMAs (there is no helper for it).
+     unsafe { crate::ffi::LLVMZludaSplitWmmaM16n16k16(main.get()) };
+-    if wmma_rewrite_enabled(gcn_arch) {
+-        let flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
++    if let Some(flags) = wmma_rewrite_flags(gcn_arch) {
+         unsafe { crate::ffi::LLVMZludaRewriteMmaF16Wmma(main.get(), flags) };
+     }
+     linked.link(main)?;
+--- a/llvm_zluda/src/lib.cpp
++++ b/llvm_zluda/src/lib.cpp
+@@ -968,6 +968,9 @@
+ }
+ 
+ // Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
++// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
++// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
++// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
+ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+ {
+     Module *M = unwrap(ModuleRef);
+@@ -990,33 +993,39 @@
+     Type *V8 = FixedVectorType::get(I32, 8);
+     AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
+                                                   AttrBuilder(Context, Reference->getAttributes().getFnAttrs()));
++    // arch-specific helpers (operand gathers, WMMAs), e4m3 MMAs, e4m3 fragment widening; the conversions
++    // between NVIDIA fragments, the transposed layout and f32 shadows are shared
++    const bool Gfx12 = (Flags & 2) != 0, NativeFp8 = Gfx12 && (Flags & 4) != 0;
++    const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
++    const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
++    const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
+     Helpers H;
+-    H.AFromB = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b", Attributes, V8, V2);
+-    H.BFromA = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_b_from_a", Attributes, V8, V4);
+-    H.Mma = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16", Attributes, V2, V8, V8, V2);
++    H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
++    H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
++    H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
+     H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
+     H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
+-    H.AFromBPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b_pair", Attributes, V8, V2, V2);
+-    H.MmaPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16_pair", Attributes, V4, V8, V8, V2, V2);
++    H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
++    H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
+     {
+         Type *F = Type::getFloatTy(Context);
+         Type *V4F = FixedVectorType::get(F, 4);
+         Type *V8F = FixedVectorType::get(F, 8);
+         H.SFromT = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_from_t", Attributes, V4F, V2, F);
+         H.TFromS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_s", Attributes, V2, V4F, F);
+-        H.MmaS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16", Attributes, V4F, V8, V8, V4F, F, F);
+-        H.MmaPairS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16_pair", Attributes, V8F, V8, V8, V4F, V4F, F, F);
+-        H.MmaE4m3S = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3", Attributes, V4F, V8, V8, V8, V8, V4F);
++        H.MmaS = M->getOrInsertFunction((Arch + "s_mma_f16").c_str(), Attributes, V4F, V8, V8, V4F, F, F);
++        H.MmaPairS = M->getOrInsertFunction((Arch + "s_mma_f16_pair").c_str(), Attributes, V8F, V8, V8, V4F, V4F, F, F);
++        H.MmaE4m3S = M->getOrInsertFunction((E4m3 + "s_mma_e4m3").c_str(), Attributes, V4F, V8, V8, V8, V8, V4F);
+         H.MmaE4m3PairS =
+-            M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3_pair", Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
++            M->getOrInsertFunction((E4m3 + "s_mma_e4m3_pair").c_str(), Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
+     }
+     if (MmaE4m3 != nullptr)
+     {
+-        H.E4m3AToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_a_to_f16", Attributes, V8, V4);
+-        H.E4m3BToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_b_to_f16", Attributes, V4, V2);
+-        H.MmaE4m3 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3", Attributes, V2, V8, V8, V8, V8, V2);
++        H.E4m3AToF16 = M->getOrInsertFunction((Widen + "e4m3_a_to_f16").c_str(), Attributes, V8, V4);
++        H.E4m3BToF16 = M->getOrInsertFunction((Widen + "e4m3_b_to_f16").c_str(), Attributes, V4, V2);
++        H.MmaE4m3 = M->getOrInsertFunction((E4m3 + "t_mma_e4m3").c_str(), Attributes, V2, V8, V8, V8, V8, V2);
+         H.MmaE4m3Pair =
+-            M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3_pair", Attributes, V4, V8, V8, V8, V8, V2, V2);
++            M->getOrInsertFunction((E4m3 + "t_mma_e4m3_pair").c_str(), Attributes, V4, V8, V8, V8, V8, V2, V2);
+     }
+     unsigned Rewritten = 0;
+     for (Function &F : *M)
+--- a/ptx/examples/d4r_emit.rs
++++ b/ptx/examples/d4r_emit.rs
+@@ -1,9 +1,13 @@
+ use std::{env, fs, path::PathBuf};
+ use ptx_parser::parse_module_checked;
+ 
++// usage: d4r_emit PTX OUT_DIR [GPU_ARCH]: translates and compiles a PTX module offline (no GPU needed) for
++// GPU_ARCH (default gfx1101), with the same D4R_ZLUDA_* code-generation switches as the runtime, and writes
++// the IR stages, asm.s and module.hsaco to OUT_DIR
+ fn main() {
+     let input = env::args().nth(1).expect("PTX path");
+     let outdir = PathBuf::from(env::args().nth(2).unwrap_or_else(|| "/tmp/d4r-autoexposure-emitted".into()));
++    let arch = env::args().nth(3).unwrap_or_else(|| "gfx1101".into());
+     fs::create_dir_all(&outdir).expect("create output directory");
+     let bytes = fs::read(&input).expect("read PTX");
+     let text = std::str::from_utf8(&bytes).expect("PTX UTF-8").trim_end_matches('\0');
+@@ -18,13 +22,14 @@
+             "opt.ll" => "opt.ll",
+             "asm" => "asm.s",
+             "o" => "object.o",
++            "elf" => "linked.elf", // the code object compile() returns (written again as module.hsaco)
+             other => panic!("unexpected stage: {other}"),
+         };
+         fs::write(outdir.join(filename), data).expect("write compiler stage");
+     };
+     let elf = llvm_zluda::compile(
+         &module.context,
+-        "gfx1101",
++        &arch,
+         module.llvm_ir,
+         linked_helpers,
+         module.attributes_ir,
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -1696,6 +1696,349 @@
+     }
+ }
+ 
++// gfx12 (RDNA4) lowering of the same MMAs (LLVMZludaRewriteMmaF16Wmma with the gfx12 flag: helpers
++// d4r_wmma12_*, and d4r_wmma12f8_* for the e4m3 ones with native FP8 WMMA). The contract is the gfx11 one
++// above: NVIDIA fragments in, the transposed accumulator layout (lane l holds D[l % 16][2v + l / 16] as two
++// f16x2 words, or four floats for the f32 shadows), so the rewrite only switches helper names.
++//
++// A gfx12 WMMA operand holds 8 of the 16 K values per lane: half h = lane / 16 its own 8 (4 dwords, returned
++// in the low half of a wmma_uint8). With A' = B^T and B' = A^T as above, dword j of half h holds the k-pair of
++// NVIDIA thread t = 2h + (j >> 1): the low pair (k 2t, 2t + 1) for even j, the high pair (k 2t + 8, 2t + 9)
++// for odd j. Then every lane of a gather reads the same source register (4 bpermutes per operand; B' also
++// needs a DPP swap per dword for rows 8..15). For e4m3 fragments the same slots hold, after the widening
++// above, e4m3 k 8h .. 8h + 7 in order.
++// The result D' has VGPR i = row n = i + 8h (columns m = lane % 16): unpaired, rows 0..7 are the h = 0 lanes
++// and the h = 1 lanes are padding; paired (A' rows 8..15 = B1^T), D1 is in the h = 1 lanes. Converting from
++// and to the transposed layout takes 2 permlanex16 each way (4 for f32 shadows).
++//
++// D4R_WMMA12_SHIM (tests only): these helpers compiled for gfx11, every gfx12 WMMA done by the gfx11 one on
++// operands rebuilt in the gfx11 helpers' K order (and for e4m3 with the same scaled widening), so the results
++// are bit-identical to the gfx11 helpers and the gfx12 data movement can be checked on RDNA3.
++typedef _Float16 wmma_half8 __attribute__((ext_vector_type(8)));
++typedef uint32_t wmma_uint4 __attribute__((ext_vector_type(4)));
++typedef int wmma_int2 __attribute__((ext_vector_type(2)));
++#ifndef D4R_WMMA12_SHIM
++__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_f16(wmma_half8 a, wmma_half8 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16");
++__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_fp8_fp8(wmma_int2 a, wmma_int2 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32");
++#endif
++
++__device__ static uint32_t wmma12_half()
++{
++    return wmma_lane() / 16;
++}
++
++__device__ static uint32_t wmma12_other_half(uint32_t value)
++{
++    return uint32_t(__builtin_amdgcn_permlanex16(int(value), int(value), 0x76543210, 0xfedcba98, false, false));
++}
++
++__device__ static float wmma12_other_half_f(float value)
++{
++    return std::bit_cast<float>(wmma12_other_half(std::bit_cast<uint32_t>(value)));
++}
++
++// dword j of an operand gathered from two candidate registers: rows r < 8 take `low` from lane 4r + t, rows
++// r >= 8 take `high` from lane 4(r - 8) + t (t = 2h + (j >> 1)). The source lane is in the first half-wave,
++// while t differs between the two destination half-waves, so the two candidates must be gathered independently.
++__device__ static uint32_t wmma12_gather2(uint32_t low, uint32_t high, uint32_t j)
++{
++    const uint32_t row = wmma_lane() % 16;
++    const uint32_t t = (j >> 1) + 2 * wmma12_half();
++    const uint32_t lo = wmma_from_lane(row * 4 + t, low);
++    const uint32_t hi = wmma_from_lane((row % 8) * 4 + t, high);
++    return row < 8 ? lo : hi;
++}
++
++__device__ static wmma_uint4 wmma12_low4(wmma_uint8 v)
++{
++    return {v[0], v[1], v[2], v[3]};
++}
++
++// f32 D' = A' B' + C' (f16 operands in the slot order above)
++__device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
++{
++#ifndef D4R_WMMA12_SHIM
++    return __zluda_wmma12_f32_16x16x16_f16(std::bit_cast<wmma_half8>(wmma12_low4(a)), std::bit_cast<wmma_half8>(wmma12_low4(b)), c);
++#else
++    // gfx11 operand dword p (K 2p, 2p + 1 in the gfx11 helpers' order) is gfx12 dword jp of half hp
++    const bool high = wmma12_half() != 0;
++    wmma_uint8 a11, b11;
++    for (uint32_t p = 0; p < 8; p++)
++    {
++        const uint32_t hp = (p & 3) >> 1, jp = 2 * (p & 1) + (p >> 2);
++        const uint32_t ao = wmma12_other_half(a[jp]), bo = wmma12_other_half(b[jp]);
++        a11[p] = (hp != 0) == high ? a[jp] : ao;
++        b11[p] = (hp != 0) == high ? b[jp] : bo;
++    }
++    // accumulators: gfx12 VGPR i = row i + 8h, gfx11 VGPR i = row 2i + h
++    wmma_float8 co, c11;
++    for (uint32_t i = 0; i < 8; i++)
++        co[i] = wmma12_other_half_f(c[i]);
++    for (uint32_t i = 0; i < 8; i++)
++        c11[i] = high ? (i < 4 ? co[2 * i + 1] : c[2 * i - 7]) : (i < 4 ? c[2 * i] : co[2 * i - 8]);
++    wmma_float8 d11 = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a11), std::bit_cast<wmma_half16>(b11), c11);
++    wmma_float8 dout, d;
++    for (uint32_t i = 0; i < 8; i++)
++        dout[i] = wmma12_other_half_f(d11[i]);
++    for (uint32_t i = 0; i < 8; i++)
++        d[i] = high ? ((i & 1) ? d11[(i >> 1) + 4] : dout[(i >> 1) + 4]) : ((i & 1) ? dout[i >> 1] : d11[i >> 1]);
++    return d;
++#endif
++}
++
++// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
++// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
++__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
++{
++    const wmma_int2 a8 = {int(__builtin_amdgcn_perm(a[1], a[0], 0x05040100u)), int(__builtin_amdgcn_perm(a[3], a[2], 0x05040100u))};
++    const wmma_int2 b8 = {int(__builtin_amdgcn_perm(b[1], b[0], 0x05040100u)), int(__builtin_amdgcn_perm(b[3], b[2], 0x05040100u))};
++#ifndef D4R_WMMA12_SHIM
++    return __zluda_wmma12_f32_16x16x16_fp8_fp8(a8, b8, c);
++#else
++    // the gfx11 helpers' scaled widening (value * 2^-8 per operand, C * 2^-16): exact powers of two
++    auto widen = [](wmma_int2 q) {
++        auto lo = e4m3x4_to_scaled_f16x2x2(uint32_t(q.x)), hi = e4m3x4_to_scaled_f16x2x2(uint32_t(q.y));
++        return wmma_uint8{lo.x, lo.y, hi.x, hi.y, 0u, 0u, 0u, 0u};
++    };
++    return wmma12_f16(widen(a8), widen(b8), c * 0x1p-16f) * 0x1p16f;
++#endif
++}
++
++// transposed f16 accumulator(s) <-> D' / C'
++__device__ static wmma_float8 wmma12_c_from_t(uint2::Native_vec_ c_t)
++{
++    // lane h = 0 needs rows 0..7: its own rows 0, 2, 4, 6 and the other half's 1, 3, 5, 7 (h = 1 lanes: padding)
++    const uint32_t o0 = wmma12_other_half(c_t.x), o1 = wmma12_other_half(c_t.y);
++    f16x2 e01 = std::bit_cast<f16x2>(c_t.x), e23 = std::bit_cast<f16x2>(c_t.y);
++    f16x2 d01 = std::bit_cast<f16x2>(o0), d23 = std::bit_cast<f16x2>(o1);
++    return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
++}
++
++__device__ static wmma_float8 wmma12_c_from_t_pair(uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
++{
++    // h = 0: C0 rows 0..7 (own even, other odd); h = 1: C1 rows 0..7 (other even, own odd)
++    const bool high = wmma12_half() != 0;
++    const uint2::Native_vec_ own = high ? c1_t : c0_t, send = high ? c0_t : c1_t;
++    const uint32_t r0 = wmma12_other_half(send.x), r1 = wmma12_other_half(send.y);
++    f16x2 o01 = std::bit_cast<f16x2>(own.x), o23 = std::bit_cast<f16x2>(own.y);
++    f16x2 p01 = std::bit_cast<f16x2>(r0), p23 = std::bit_cast<f16x2>(r1);
++    f16x2 e01 = high ? p01 : o01, e23 = high ? p23 : o23, d01 = high ? o01 : p01, d23 = high ? o23 : p23;
++    return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
++}
++
++__device__ static void wmma12_rows_f16(wmma_float8 d, uint2::Native_vec_ &even, uint2::Native_vec_ &odd)
++{
++    f16x2 e01{static_cast<f16>(d[0]), static_cast<f16>(d[2])}, e23{static_cast<f16>(d[4]), static_cast<f16>(d[6])};
++    f16x2 o01{static_cast<f16>(d[1]), static_cast<f16>(d[3])}, o23{static_cast<f16>(d[5]), static_cast<f16>(d[7])};
++    even = {std::bit_cast<uint32_t>(e01), std::bit_cast<uint32_t>(e23)};
++    odd = {std::bit_cast<uint32_t>(o01), std::bit_cast<uint32_t>(o23)};
++}
++
++__device__ static uint2::Native_vec_ wmma12_t_from_d(wmma_float8 d)
++{
++    // h = 0 keeps rows 0, 2, 4, 6; h = 1 takes rows 1, 3, 5, 7 from the other half
++    uint2::Native_vec_ even, odd;
++    wmma12_rows_f16(d, even, odd);
++    const uint2::Native_vec_ other = {wmma12_other_half(odd.x), wmma12_other_half(odd.y)};
++    return wmma12_half() != 0 ? other : even;
++}
++
++__device__ static uint4::Native_vec_ wmma12_t_from_d_pair(wmma_float8 d)
++{
++    // h = 0 holds D0 rows 0..7, h = 1 D1 rows 0..7; lane h wants rows 2v + h of both
++    const bool high = wmma12_half() != 0;
++    uint2::Native_vec_ even, odd;
++    wmma12_rows_f16(d, even, odd);
++    const uint2::Native_vec_ send = high ? even : odd;
++    const uint2::Native_vec_ recv = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
++    return high ? uint4::Native_vec_{recv.x, recv.y, odd.x, odd.y} : uint4::Native_vec_{even.x, even.y, recv.x, recv.y};
++}
++
++// the same for f32 shadows (element v = row 2v + h)
++__device__ static wmma_float8 wmma12_c_from_s(wmma_float4 c)
++{
++    wmma_float4 o;
++    for (uint32_t v = 0; v < 4; v++)
++        o[v] = wmma12_other_half_f(c[v]);
++    return {c[0], o[0], c[1], o[1], c[2], o[2], c[3], o[3]};
++}
++
++__device__ static wmma_float8 wmma12_c_from_s_pair(wmma_float4 c0, wmma_float4 c1)
++{
++    const bool high = wmma12_half() != 0;
++    wmma_float8 r;
++    for (uint32_t v = 0; v < 4; v++)
++    {
++        const float own = high ? c1[v] : c0[v], recv = wmma12_other_half_f(high ? c0[v] : c1[v]);
++        r[2 * v] = high ? recv : own;
++        r[2 * v + 1] = high ? own : recv;
++    }
++    return r;
++}
++
++__device__ static wmma_float4 wmma12_s_from_d(wmma_float8 d)
++{
++    const bool high = wmma12_half() != 0;
++    wmma_float4 r;
++    for (uint32_t v = 0; v < 4; v++)
++    {
++        const float odd = wmma12_other_half_f(d[2 * v + 1]);
++        r[v] = high ? odd : d[2 * v];
++    }
++    return r;
++}
++
++__device__ static wmma_float8 wmma12_s_from_d_pair(wmma_float8 d)
++{
++    const bool high = wmma12_half() != 0;
++    wmma_float8 r;
++    for (uint32_t v = 0; v < 4; v++)
++    {
++        const float recv = wmma12_other_half_f(high ? d[2 * v] : d[2 * v + 1]);
++        r[v] = high ? recv : d[2 * v];
++        r[4 + v] = high ? d[2 * v + 1] : recv;
++    }
++    return r;
++}
++
++extern "C"
++{
++    wmma_uint8 FUNC(d4r_wmma12_t_a_from_b)(uint2::Native_vec_ b)
++    {
++        __builtin_amdgcn_sched_barrier(0);
++        uint32_t n = wmma_lane() % 16;
++        wmma_uint8 result = {};
++        for (uint32_t j = 0; j < 4; j++)
++        {
++            uint32_t value = wmma_from_lane((n % 8) * 4 + (j >> 1) + 2 * wmma12_half(), (j & 1) ? b.y : b.x);
++            result[j] = n < 8 ? value : 0u;
++        }
++        return result;
++    }
++
++    wmma_uint8 FUNC(d4r_wmma12_t_b_from_a)(uint4::Native_vec_ a)
++    {
++        __builtin_amdgcn_sched_barrier(0);
++        wmma_uint8 result = {};
++        for (uint32_t j = 0; j < 4; j++)
++            result[j] = wmma12_gather2((j & 1) ? a.z : a.x, (j & 1) ? a.w : a.y, j);
++        return result;
++    }
++
++    wmma_uint8 FUNC(d4r_wmma12_t_a_from_b_pair)(uint2::Native_vec_ b0, uint2::Native_vec_ b1)
++    {
++        __builtin_amdgcn_sched_barrier(0);
++        wmma_uint8 result = {};
++        for (uint32_t j = 0; j < 4; j++)
++            result[j] = wmma12_gather2((j & 1) ? b0.y : b0.x, (j & 1) ? b1.y : b1.x, j);
++        return result;
++    }
++
++    uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
++    {
++        return wmma12_t_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_t(c_t)));
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
++    {
++        return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
++    }
++
++    wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
++    {
++        return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
++    }
++
++    wmma_float8 FUNC(d4r_wmma12_s_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c0, wmma_float4 c1, float up,
++                                                float down)
++    {
++        return wmma12_s_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_s_pair(c0 * up, c1 * up))) * down;
++    }
++
++    // e4m3 with the f16 widening (d4r_wmma_e4m3_a_to_f16 / b_to_f16 above): two chained f16 WMMAs in the
++    // 2^-16 scaled domain, like the gfx11 helpers
++    uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                   wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
++    {
++        wmma_float8 c = wmma12_c_from_t(c_t) * 0x1p-16f;
++        c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
++        return wmma12_t_from_d(c * 0x1p16f);
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                        wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
++                                                        uint2::Native_vec_ c1_t)
++    {
++        wmma_float8 c = wmma12_c_from_t_pair(c0_t, c1_t) * 0x1p-16f;
++        c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
++        return wmma12_t_from_d_pair(c * 0x1p16f);
++    }
++
++    wmma_float4 FUNC(d4r_wmma12_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                            wmma_uint8 b_t_high, wmma_float4 c)
++    {
++        return wmma12_s_from_d(wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s(c))));
++    }
++
++    wmma_float8 FUNC(d4r_wmma12_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                 wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
++    {
++        return wmma12_s_from_d_pair(
++            wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s_pair(c0, c1))));
++    }
++
++    // Native FP8 (d4r_wmma12f8_*, D4R_ZLUDA_WMMA_FP8_NATIVE): the "widening" keeps the e4m3 bytes, two per
++    // dword in the low 16 bits and in the same slots the f16 widening fills, so the f16 gathers above move them;
++    // the MMA packs this half's 8 bytes (e4m3 k 8h .. 8h + 7 in order) into the FP8 operand. OCP e4m3 (gfx12)
++    // is NVIDIA's e4m3, so the bytes need no conversion. Shadows keep the 2^-16 scaled domain of the e4m3
++    // functions (exact power-of-two rescaling around the WMMA).
++    __device__ static uint2::Native_vec_ e4m3x4_split(uint32_t packed)
++    {
++        return {__builtin_amdgcn_perm(0u, packed, 0x0c0c0100u), __builtin_amdgcn_perm(0u, packed, 0x0c0c0302u)};
++    }
++
++    wmma_uint8 FUNC(d4r_wmma12f8_e4m3_a_to_f16)(uint4::Native_vec_ a)
++    {
++        auto x = e4m3x4_split(a.x), y = e4m3x4_split(a.y), z = e4m3x4_split(a.z), w = e4m3x4_split(a.w);
++        return {x.x, y.x, x.y, y.y, z.x, w.x, z.y, w.y};
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12f8_e4m3_b_to_f16)(uint2::Native_vec_ b)
++    {
++        auto x = e4m3x4_split(b.x), y = e4m3x4_split(b.y);
++        return {x.x, x.y, y.x, y.y};
++    }
++
++    uint2::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                     wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
++    {
++        return wmma12_t_from_d(wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t(c_t))));
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                          wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
++                                                          uint2::Native_vec_ c1_t)
++    {
++        return wmma12_t_from_d_pair(
++            wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t_pair(c0_t, c1_t))));
++    }
++
++    wmma_float4 FUNC(d4r_wmma12f8_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                              wmma_uint8 b_t_high, wmma_float4 c)
++    {
++        wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s(c * 0x1p16f)));
++        return wmma12_s_from_d(d) * 0x1p-16f;
++    }
++
++    wmma_float8 FUNC(d4r_wmma12f8_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
++                                                   wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
++    {
++        wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high,
++                                    wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s_pair(c0 * 0x1p16f, c1 * 0x1p16f)));
++        return wmma12_s_from_d_pair(d) * 0x1p-16f;
++    }
++}
++
+ template <typename T, typename A, typename B, int BOffset, typename OcklFunc>
+ __device__ T dp2a_impl(T a, T b, T c, OcklFunc ockl_func)
+ {
+--- a/zluda/src/impl/module.rs
++++ b/zluda/src/impl/module.rs
+@@ -356,6 +356,7 @@
+         wmma: llvm_zluda::compile::wmma_rewrite_requested(),
+         wmma_fp8: llvm_zluda::compile::wmma_fp8_rewrite_requested(),
+         param_provenance: llvm_zluda::compile::param_provenance_enabled(),
++        wmma_fp8_native: llvm_zluda::compile::wmma_fp8_native_effective(gcn_arch),
+     };
+     let mut cache_with_key = match (text, global_state.cache_path.as_ref()) {
+         (Some(text), Some(p)) => (|| {
+@@ -397,6 +398,10 @@
+     wmma: bool,
+     wmma_fp8: bool,
+     param_provenance: bool,
++    // d4r: native FP8 WMMA where it takes effect (gfx12); absent from the key everywhere else, so the setting
++    // (d4r.ini NativeFp8) never forces a recompile on GPUs that ignore it
++    #[serde(skip_serializing_if = "std::ops::Not::not")]
++    wmma_fp8_native: bool,
+ }
+ 
+ fn get_hip_properties<'a>() -> Result<hipDeviceProp_tR0600, CUerror> {
+@@ -430,6 +435,7 @@
+                     && !matches!(
+                         key.as_str(),
+                         "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
++                            | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
+                     )
+             })
+             .map(|(key, value)| format!("{key}={}", value.trim()))

patches/zluda/0008-native-windows-build.patch

@@ -0,0 +1,203 @@
+Windows LLVM-MinGW build, portable helper assertions and optional diagnostic UI.
+Apply after upstream d4r patches 0002 through 0007, on ZLUDA ee2f25a.
+
+diff --git a/ext/highs-sys/build.rs b/ext/highs-sys/build.rs
+--- a/ext/highs-sys/build.rs
++++ b/ext/highs-sys/build.rs
+@@ -66,11 +66,12 @@
+     let linux = target.contains("linux");
+     let mingw = target.contains("pc-windows-gnu");
+     if apple {
+         println!("cargo:rustc-link-lib=dylib=c++");
+     } else if linux || mingw {
+-        println!("cargo:rustc-link-lib=dylib=stdc++");
++        let cxx_library = env::var("CXXSTDLIB").unwrap_or_else(|_| "stdc++".to_owned());
++        println!("cargo:rustc-link-lib=dylib={cxx_library}");
+     }
+     println!("cargo:rerun-if-changed=../HiGHS/highs/interfaces/highs_c_api.h");
+ 
+     true
+ }
+diff --git a/ext/hip_runtime-sys/build.rs b/ext/hip_runtime-sys/build.rs
+--- a/ext/hip_runtime-sys/build.rs
++++ b/ext/hip_runtime-sys/build.rs
+@@ -1,8 +1,12 @@
+ fn main() {
+     if cfg!(windows) {
+-        println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
++        if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
++            println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
++        } else {
++            println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
++        }
+     } else {
+         println!("cargo:rustc-link-lib=dylib=amdhip64");
+         println!("cargo:rustc-link-search=native=/opt/rocm/lib/");
+     }
+ }
+diff --git a/llvm_zluda/build.rs b/llvm_zluda/build.rs
+--- a/llvm_zluda/build.rs
++++ b/llvm_zluda/build.rs
+@@ -19,10 +19,18 @@
+ 
+ fn main() {
+     let bc_path1 = "src/device-libs/ockl.bc";
+     println!("cargo:rerun-if-changed={}", bc_path1);
+     check_lfs_file(bc_path1);
++
++    println!("cargo:rerun-if-env-changed=ZLUDA_LLVM_BUILD_DIR");
++    if let Some(directory) = env::var_os("ZLUDA_LLVM_BUILD_DIR") {
++        let llvm_build_path = PathBuf::from(directory);
++        println!("cargo:build-path={}", llvm_build_path.display());
++        link_prebuilt_llvm(&llvm_build_path);
++        return;
++    }
+ 
+     let mut cmake = Config::new(r"../ext/llvm-project/llvm");
+     try_use_sccache(&mut cmake);
+     try_use_ninja(&mut cmake);
+     cmake
+@@ -57,16 +65,31 @@
+         cmake.build();
+     }
+     let cmake_profile = cmake.get_profile();
+     let llvm_build_path = get_llvm_build_path(&llvm_dir, cmake_profile);
+     println!("cargo:build-path={}", llvm_build_path.display());
++    link_prebuilt_llvm(&llvm_build_path);
++}
++
++fn link_prebuilt_llvm(llvm_build_path: &PathBuf) {
+     let (cxxflags, ldflags, libdir, lib_names, system_libs) =
+         llvm_config(&llvm_build_path.join("bin/llvm-config")).unwrap();
+     println!("cargo:rustc-link-arg={ldflags}");
+     println!("cargo:rustc-link-search=native={libdir}");
+     for lib in system_libs.split_ascii_whitespace() {
+-        println!("cargo:rustc-link-arg={lib}");
++        if let Some(name) = lib.strip_prefix("-l") {
++            println!("cargo:rustc-link-lib={name}");
++        } else {
++            println!("cargo:rustc-link-arg={lib}");
++        }
++    }
++    // llvm-config omits system libraries on Windows. Propagate these through
++    // this rlib to its final cdylib consumer (link-arg alone cannot do that).
++    if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") {
++        for library in ["psapi", "shell32", "ole32", "uuid", "advapi32", "ws2_32", "ntdll"] {
++            println!("cargo:rustc-link-lib={library}");
++        }
+     }
+     link_lld_components();
+     link_llvm_components(lib_names);
+     compile_cxx_lib(cxxflags, &llvm_build_path);
+ }
+diff --git a/zluda/build.rs b/zluda/build.rs
+--- a/zluda/build.rs
++++ b/zluda/build.rs
+@@ -1,11 +1,15 @@
+ use vergen_gix::{Emitter, Gix};
+ 
+ fn main() {
+     if cfg!(windows) {
+-        println!("cargo:rustc-link-arg=delayimp.lib");
+-        println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
++        if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
++            println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
++        } else {
++            println!("cargo:rustc-link-arg=delayimp.lib");
++            println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
++        }
+         let dll_path = "bin/nvcudart_hybrid64.dll";
+         println!("cargo:rerun-if-changed={}", dll_path);
+         check_lfs_file(dll_path);
+     }
+     let git = Gix::builder().sha(false).build();
+diff --git a/zluda_windows/src/lib.rs b/zluda_windows/src/lib.rs
+--- a/zluda_windows/src/lib.rs
++++ b/zluda_windows/src/lib.rs
+@@ -11,27 +11,27 @@
+ use uuid::uuid;
+ use widestring::{u16str, U16Str, U16String};
+ use windows::{
+     core::{w, HRESULT, PCWSTR},
+     Win32::{
+-        Foundation::{HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
++        Foundation::{FreeLibrary, HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
+         System::{
+             JobObjects::{
+                 AssignProcessToJobObject, JobObjectExtendedLimitInformation,
+                 SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+                 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
+             },
+             LibraryLoader::{
+-                GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, LoadLibraryExW,
++                GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, GetProcAddress, LoadLibraryExW,
+                 GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
+                 GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS,
+-                LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR,
++                LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, LOAD_LIBRARY_SEARCH_SYSTEM32,
+             },
+         },
+         UI::{
+             Controls::{
+-                TaskDialogIndirect, TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
++                TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
+                 TASKDIALOG_NOTIFICATIONS, TDCBF_CLOSE_BUTTON, TDF_ENABLE_HYPERLINKS,
+                 TDN_HYPERLINK_CLICKED, TD_ERROR_ICON,
+             },
+             WindowsAndMessaging::IDCLOSE,
+         },
+@@ -487,11 +487,23 @@
+             pszFooter: PCWSTR::null(),
+             pfCallback: Some(task_dialog_callback),
+             lpCallbackData: 0,
+             cxWidth: 0,
+         };
+-        TaskDialogIndirect(&config, None, None, None).ok();
++        // A CUDA DLL may be loaded by an executable with no Common Controls
++        // v6 activation manifest. Do not make its diagnostic UI an eager DLL
++        // import: comctl32 v5 has no TaskDialogIndirect export.
++        if let Ok(controls) = LoadLibraryExW(w!("comctl32.dll"), None, LOAD_LIBRARY_SEARCH_SYSTEM32) {
++            if let Some(raw) = GetProcAddress(controls, windows::core::s!("TaskDialogIndirect")) {
++                let show: unsafe extern "system" fn(*const TASKDIALOGCONFIG, *mut i32, *mut i32, *mut i32) -> HRESULT =
++                    mem::transmute(raw);
++                let _ = show(&config, ptr::null_mut(), ptr::null_mut(), ptr::null_mut());
++            } else {
++                eprintln!("ZLUDA failed to load {}; set HIP_PATH to the selected HIP SDK", redirect_names[0]);
++            }
++            let _ = FreeLibrary(controls);
++        }
+     }
+     result
+ }
+ 
+ unsafe extern "system" fn task_dialog_callback(
+diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -405,11 +405,26 @@
+                             uint32_t line,
+                             uint64_t function,
+                             uint64_t char_size)
+     {
+         (void)char_size;
++#if defined(_WIN32) || defined(_WIN64)
++        // HIP on Windows only supplies the wide _wassert entry point. Keep the
++        // CUDA assertion's ASCII diagnostics and trap through public OCKL APIs.
++        auto output = __ockl_fprintf_stderr_begin();
++        const char* parts[] = {"CUDA assertion: ", (const char*)message, " at ", (const char*)file,
++                               " in ", (const char*)function, "\n"};
++        for (const char* part : parts) {
++            const char* end = part;
++            while (*end) ++end;
++            output = __ockl_fprintf_append_string_n(output, part, end - part, 0);
++        }
++        __ockl_fprintf_append_args(output, 1, line, 0, 0, 0, 0, 0, 0, 1);
++        __builtin_trap();
++#else
+         [[clang::noinline]] __assert_fail((const char *)message, (const char *)file, line, (const char *)function);
++#endif
+     }
+ 
+     // * Smallest denormal is 1.4 × 10^-45
+     // * Smallest normal is ~1.175494351 × 10^(-38)
+     // * Now, 1.175494351×10^-38 / 1.4 × 10^-45  = 8396388 + 31/140

patches/zluda/0009-resource-descriptor-queries.patch

@@ -0,0 +1,124 @@
+Implement the CUDA texture resource descriptor query required by NGX Evaluate.
+Regression: tools/windows/cuda_image_probe.cpp in the d4r Windows port.
+
+diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
+--- a/zluda/src/lib.rs
++++ b/zluda/src/lib.rs
+@@ -67,12 +67,14 @@
+ cuda_macros::cuda_function_declarations!(
+     unimplemented,
+     implemented
+         <= [
+             cuArray3DCreate_v2,
++            cuArray3DGetDescriptor_v2,
+             cuArrayCreate_v2,
+             cuArrayDestroy,
++            cuArrayGetDescriptor_v2,
+             cuCtxCreate_v2,
+             cuCtxCreate_v4,
+             cuCtxDestroy_v2,
+             cuCtxGetApiVersion,
+             cuCtxGetCurrent,
+@@ -203,10 +205,11 @@
+             cuSurfObjectCreate,
+             cuSurfObjectDestroy,
+             cuSurfObjectGetResourceDesc,
+             cuTexObjectCreate,
+             cuTexObjectDestroy,
++            cuTexObjectGetResourceDesc,
+             cuTexRefSetAddress_v2,
+             cuTexRefSetAddressMode,
+             cuTexRefSetArray,
+             cuTexRefSetFilterMode,
+             cuTexRefSetFlags,
+diff --git a/zluda/src/impl/tex.rs b/zluda/src/impl/tex.rs
+--- a/zluda/src/impl/tex.rs
++++ b/zluda/src/impl/tex.rs
+@@ -1,19 +1,58 @@
+ use crate::r#impl::hipfix;
+ use hip_runtime_sys::*;
++use std::collections::HashMap;
++use std::sync::{Mutex, OnceLock};
++
++fn object_resources() -> &'static Mutex<HashMap<usize, Vec<u8>>> {
++    static RESOURCES: OnceLock<Mutex<HashMap<usize, Vec<u8>>>> = OnceLock::new();
++    RESOURCES.get_or_init(|| Mutex::new(HashMap::new()))
++}
+ 
+ pub(crate) unsafe fn object_create(
+     p_tex_object: *mut hipTextureObject_t,
+     p_res_desc: *const HIP_RESOURCE_DESC,
+     p_tex_desc: *const HIP_TEXTURE_DESC,
+     p_res_view_desc: *const HIP_RESOURCE_VIEW_DESC,
+ ) -> hipError_t {
+-    hipTexObjectCreate(p_tex_object, p_res_desc, p_tex_desc, p_res_view_desc)
++    let output = p_tex_object.as_mut().ok_or(hipErrorCode_t::InvalidValue)?;
++    let descriptor = p_res_desc.as_ref().ok_or(hipErrorCode_t::InvalidValue)?;
++    hipTexObjectCreate(output, descriptor, p_tex_desc, p_res_view_desc)?;
++    let bytes = std::slice::from_raw_parts(p_res_desc.cast::<u8>(), std::mem::size_of::<HIP_RESOURCE_DESC>()).to_vec();
++    match object_resources().lock() {
++        Ok(mut resources) => {
++            resources.insert(*output as usize, bytes);
++            Ok(())
++        }
++        Err(_) => {
++            let _ = hipDestroyTextureObject(*output);
++            *output = std::ptr::null_mut();
++            Err(hipErrorCode_t::Unknown)
++        }
++    }
+ }
+ 
+ pub(crate) unsafe fn object_destroy(tex_object: hipTextureObject_t) -> hipError_t {
+-    hipDestroyTextureObject(tex_object)
++    let mut resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
++    if !resources.contains_key(&(tex_object as usize)) { return Err(hipErrorCode_t::InvalidValue); }
++    hipDestroyTextureObject(tex_object)?;
++    resources.remove(&(tex_object as usize));
++    Ok(())
++}
++
++pub(crate) unsafe fn object_get_resource_desc(
++    descriptor: &mut HIP_RESOURCE_DESC,
++    tex_object: hipTextureObject_t,
++) -> hipError_t {
++    // NGX probes surface handles as textures before trying the surface API.
++    // Passing that foreign handle to HIP dereferences invalid host metadata.
++    // Return a CUDA error for unknown handles and preserve the complete CUDA
++    // descriptor (HIP's query leaves the flags/reserved bytes unspecified).
++    let resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
++    let bytes = resources.get(&(tex_object as usize)).ok_or(hipErrorCode_t::InvalidValue)?;
++    std::ptr::copy_nonoverlapping(bytes.as_ptr(), (descriptor as *mut HIP_RESOURCE_DESC).cast::<u8>(), bytes.len());
++    Ok(())
+ }
+ 
+ pub(crate) unsafe fn ref_set_array(
+     texref: *mut textureReference,
+     array: hipArray_t,
+diff --git a/zluda/src/impl/array.rs b/zluda/src/impl/array.rs
+--- a/zluda/src/impl/array.rs
++++ b/zluda/src/impl/array.rs
+@@ -70,5 +70,23 @@
+     handle: *mut hipArray_t,
+     desc: *const HIP_ARRAY3D_DESCRIPTOR,
+ ) -> hipError_t {
+     hipArray3DCreate(handle, desc)
+ }
++
++pub(crate) unsafe fn get_descriptor_v2(
++    output: &mut HIP_ARRAY_DESCRIPTOR,
++    array: hipArray_t,
++) -> hipError_t {
++    hipArrayGetDescriptor(output, array)?;
++    if let Some(format) = original_format(array)? {
++        output.Format = hipArray_Format(format);
++    }
++    Ok(())
++}
++
++pub(crate) unsafe fn _3d_get_descriptor_v2(
++    output: &mut HIP_ARRAY3D_DESCRIPTOR,
++    array: hipArray_t,
++) -> hipError_t {
++    hipArray3DGetDescriptor(output, array)
++}

patches/zluda/0010-native-identity-replay-windows-cache.patch

@@ -0,0 +1,440 @@
+Subject: [PATCH] Guard and own native overrides; capture K replay on Windows
+
+Native replacements require the upstream PTX identity manifest. Preserve native
+module/image ownership, bypass redundant JIT for completely replaced modules,
+include the loaded Windows DLL fingerprint in cache identity, and expose opt-in
+private real-weight replay captures with explicit native/translated launch logs.
+
+diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
+new file mode 100644
+index 0000000..875b2e3
+--- /dev/null
++++ b/zluda/src/impl/d4r_replay.rs
+@@ -0,0 +1,109 @@
++//! Opt-in, synchronous diagnostic captures. Never enabled in the game fast path.
++//! Captures can contain proprietary network weights; keep them out of releases.
++use hip_runtime_sys::*;
++use std::{collections::HashMap, ffi::{c_void, CStr}, fs, path::PathBuf, sync::{Mutex, OnceLock}};
++
++struct Config { directory: PathBuf, filter: String, limit: u32 }
++static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
++static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
++
++pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
++
++fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
++    eprintln!("[d4r-replay] capture failed: {error}");
++    hipErrorCode_t::r#Unknown
++}
++
++impl Capture {
++    fn save(&self, destination: &std::path::Path) -> hipError_t {
++        fs::create_dir_all(destination).map_err(io_error)?;
++        for (id, &(base, size)) in self.allocations.iter().enumerate() {
++            let mut bytes = vec![0u8; size];
++            unsafe { hipMemcpyDtoH(bytes.as_mut_ptr().cast(), hipDeviceptr_t(base as *mut c_void), size) }?;
++            fs::write(destination.join(format!("alloc-{id}.bin")), bytes).map_err(io_error)?;
++        }
++        Ok(())
++    }
++
++    pub(crate) fn finish(self, stream: hipStream_t) -> hipError_t {
++        unsafe { hipStreamSynchronize(stream) }?;
++        self.save(&self.directory.join("gpu-output"))?;
++        eprintln!("[d4r-replay] complete path={}", self.directory.display());
++        Ok(())
++    }
++}
++
++pub(crate) unsafe fn begin(
++    name: &CStr, stream: hipStream_t, parameters: *mut *mut c_void,
++    extra: *mut *mut c_void, launch: [u32; 7],
++) -> Result<Option<Capture>, hipErrorCode_t> {
++    let Some(config) = CONFIG.get_or_init(|| std::env::var_os("D4R_CUDA_REPLAY_DUMP_DIR").map(|directory| Config {
++        directory: directory.into(),
++        filter: std::env::var("D4R_CUDA_REPLAY_DUMP_FILTER").unwrap_or_default(),
++        limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
++    })) else { return Ok(None) };
++    let name = name.to_str().map_err(io_error)?;
++    // Only the audited, single 176-byte K argument ABI is captured here.
++    if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
++        return Ok(None);
++    }
++    let mut block = std::ptr::null::<u8>();
++    let mut bytes = None;
++    if !extra.is_null() {
++        for i in (0..6).step_by(2) {
++            match *extra.add(i) as usize {
++                1 => block = (*extra.add(i + 1)).cast(),
++                2 => bytes = Some(*(*extra.add(i + 1) as *const usize)),
++                0 => break,
++                _ => return Err(hipErrorCode_t::r#InvalidValue),
++            }
++        }
++    } else if !parameters.is_null() {
++        block = (*parameters).cast();
++        bytes = Some(176);
++    }
++    if block.is_null() || bytes != Some(176) {
++        return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
++    }
++    let sequence = {
++        let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
++        let count = counts.entry(name.to_owned()).or_default();
++        if *count >= config.limit { return Ok(None); }
++        *count += 1;
++        *count
++    };
++    hipStreamSynchronize(stream)?;
++    let arguments = std::slice::from_raw_parts(block, 176);
++    let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
++    fs::create_dir_all(&directory).map_err(io_error)?;
++    fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
++    let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
++        launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
++    let mut allocations = Vec::<(usize, usize)>::new();
++    for offset in (0..176).step_by(8) {
++        let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
++        if pointer == 0 { continue; }
++        let mut base = hipDeviceptr_t(std::ptr::null_mut());
++        let mut size = 0;
++        if hipMemGetAddressRange(&mut base, &mut size, hipDeviceptr_t(pointer as *mut c_void)).is_err() { continue; }
++        let base = base.0 as usize;
++        if pointer < base || pointer - base >= size { continue; }
++        if size > 512 * 1024 * 1024 { return Err(io_error("allocation exceeds diagnostic 512 MiB limit")); }
++        let id = match allocations.iter().position(|&(b, _)| b == base) {
++            Some(id) => id,
++            None => {
++                let id = allocations.len();
++                allocations.push((base, size));
++                manifest.push_str(&format!("alloc {id} 0x{base:x} {size}\n"));
++                id
++            }
++        };
++        manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
++    }
++    if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
++    fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
++    let capture = Capture { directory, allocations };
++    capture.save(&capture.directory)?;
++    eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
++    Ok(Some(capture))
++}
+diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
+index 033d9a0..7a7470a 100644
+--- a/zluda/src/impl/function.rs
++++ b/zluda/src/impl/function.rs
+@@ -7,6 +7,8 @@ use hip_runtime_sys::*;
+ use std::mem;
+ 
+ pub(crate) struct Function {
++    pub(crate) name: std::ffi::CString,
++    pub(crate) native_owner: Option<super::module::NativeModule>,
+     pub(crate) base: hipFunction_t,
+     pub(crate) sm_version: u32,
+     pub(crate) explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
+@@ -144,6 +146,8 @@ pub(crate) fn launch_kernel(
+         }
+         translated_extra_storage.as_mut_ptr()
+     };
++    let replay = unsafe { super::d4r_replay::begin(&f.name, stream, kernel_params, extra,
++        [grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y, block_dim_z, shared_mem_bytes]) }?;
+     // the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
+     let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
+         let mut block: *const u8 = std::ptr::null();
+@@ -206,7 +210,11 @@ pub(crate) fn launch_kernel(
+     }
+     let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
+     let (grid_dim_x, grid_dim_y, grid_dim_z) = if f.grid_x != 0 { (f.grid_x, 1, 1) } else { (grid_dim_x, grid_dim_y, grid_dim_z) };
+-    unsafe {
++    if std::env::var_os("D4R_ZLUDA_VERBOSE").is_some() {
++        eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
++            f.name, if f.native_owner.is_some() { "native" } else { "translated" });
++    }
++    let result = unsafe {
+         hipModuleLaunchKernel(
+             f.base,
+             grid_dim_x,
+@@ -220,7 +228,10 @@ pub(crate) fn launch_kernel(
+             kernel_params,
+             extra_for_hip,
+         )
+-    }
++    };
++    result?;
++    if let Some(replay) = replay { replay.finish(stream)?; }
++    Ok(())
+ }
+ 
+ pub(crate) unsafe fn set_attribute(
+diff --git a/zluda/src/impl/library.rs b/zluda/src/impl/library.rs
+index 98be33e..1319792 100644
+--- a/zluda/src/impl/library.rs
++++ b/zluda/src/impl/library.rs
+@@ -21,8 +21,8 @@ impl Library {
+         let module_lock = self.modules.get(device).ok_or(CUerror::INVALID_DEVICE)?;
+         *module_lock.get_or_init(|| match self.data {
+             LibraryData::Lazy(lib) => {
+-                let (module, sm_version, meta32) = module::load_hip_module(lib)?;
+-                Ok(Module::new(module, sm_version, meta32).wrap())
++                let (module, sm_version, meta32, ptx_hash, native_image) = module::load_hip_module(lib)?;
++                Ok(Module::new(module, sm_version, meta32, ptx_hash, native_image).wrap())
+             }
+             LibraryData::Eager(()) => Err(CUerror::NOT_SUPPORTED),
+         })
+diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
+index 5329d32..14e0aee 100644
+--- a/zluda/src/impl/mod.rs
++++ b/zluda/src/impl/mod.rs
+@@ -5,6 +5,7 @@ pub(super) mod array;
+ pub(super) mod context;
+ pub(super) mod device;
+ pub(super) mod driver;
++pub(super) mod d4r_replay;
+ pub(super) mod event;
+ pub(super) mod function;
+ pub(super) mod graph;
+diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
+index 9f72f2f..5a79368 100644
+--- a/zluda/src/impl/module.rs
++++ b/zluda/src/impl/module.rs
+@@ -19,6 +19,8 @@ pub(crate) struct Module {
+     pub(crate) base: hipModule_t,
+     pub(crate) sm_version: u32,
+     pub(crate) bit32: Option<Metadata32Bit>,
++    ptx_hash: Option<u64>,
++    _native_base_image: Option<Vec<u8>>,
+     mutable: Mutex<ModuleMutable>,
+ }
+ 
+@@ -33,12 +35,14 @@ impl ModuleMutable {
+         sm_version: u32,
+         name: CString,
+         explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
++        ptx_hash: Option<u64>,
+     ) -> Result<&Function, CUerror> {
+         Ok(match self.functions.entry(name) {
+             hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
+             hash_map::Entry::Vacant(entry) => {
+-                let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
+-                    Some(native) => native,
++                let mut native_owner = None;
++                let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key(), ptx_hash) {
++                    Some((native, owner)) => { native_owner = Some(owner); native },
+                     None => {
+                         let mut func_handle = unsafe { std::mem::zeroed() };
+                         unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
+@@ -46,6 +50,8 @@ impl ModuleMutable {
+                     }
+                 };
+                 let func = Box::new(Function {
++                    name: entry.key().clone(),
++                    native_owner,
+                     base: func_handle,
+                     sm_version,
+                     explicit_args_size_align,
+@@ -69,19 +75,48 @@ impl ModuleMutable {
+ /// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
+ /// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
+ /// (persistent kernels, which derive the original grid from their parameters) the grid.
++pub(crate) struct NativeModule {
++    module: hipModule_t,
++    _image: Vec<u8>,
++}
++
++impl Drop for NativeModule {
++    fn drop(&mut self) {
++        if self.module.0.is_null() { return; }
++        if let Err(error) = unsafe { hipModuleUnload(self.module) } {
++            eprintln!("[d4r-native] unload failed: {error:?}");
++        }
++    }
++}
++
+ fn native_override(
+     name: &CStr,
+-) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
++    ptx_hash: Option<u64>,
++) -> Option<((hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32), NativeModule)> {
+     let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
++    let manifest = std::path::Path::new(&dir).join("d4r-kernels.txt");
++    let allowed = fs::read_to_string(&manifest).ok().is_some_and(|text| text.lines().any(|line| {
++        let mut fields = line.split_whitespace();
++        fields.next() == name.to_str().ok()
++            && fields.next().and_then(|s| u64::from_str_radix(s, 16).ok()) == ptx_hash
++            && ptx_hash.is_some()
++    }));
++    if !allowed {
++        eprintln!("[d4r-native] miss kernel={name:?} reason=ptx_identity hash={ptx_hash:016x?} fallback=translated");
++        return None;
++    }
+     let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
+-    let image = fs::read(&path).ok()?;
+-    // HIP may reference the image after loading; the override lives for the whole process.
+-    let image: &'static [u8] = Box::leak(image.into_boxed_slice());
++    let image = match fs::read(&path) {
++        Ok(image) => image,
++        Err(error) => { eprintln!("[d4r-native] miss kernel={name:?} reason=object_missing error={error} fallback=translated"); return None; }
++    };
+     let mut native_module = unsafe { mem::zeroed() };
+     if unsafe { hipModuleLoadData(&mut native_module, image.as_ptr().cast()) }.is_err() {
+         eprintln!("[d4r-native] failed to load {}", path.display());
+         return None;
+     }
++    // Keep the image and module owned by the CUfunction; release at module unload.
++    let owner = NativeModule { module: native_module, _image: image };
+     let mut func = unsafe { mem::zeroed() };
+     if unsafe { hipModuleGetFunction(&mut func, native_module, name.as_ptr()) }.is_err() {
+         eprintln!("[d4r-native] {} has no kernel {:?}", path.display(), name);
+@@ -138,7 +173,7 @@ fn native_override(
+         prep_key,
+         prep_slots
+     );
+-    Some((func, prep, block_z, grid_x, prep_key, prep_slots))
++    Some(((func, prep, block_z, grid_x, prep_key, prep_slots), owner))
+ }
+ 
+ impl ZludaObject for Module {
+@@ -229,7 +264,7 @@ pub(crate) struct Global32Bit {
+ }
+ 
+ impl Module {
+-    pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>) -> Self {
++    pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>, ptx_hash: Option<u64>, native_base_image: Option<Vec<u8>>) -> Self {
+         Self {
+             base,
+             sm_version,
+@@ -237,6 +272,8 @@ impl Module {
+                 functions: FxHashMap::default(),
+             }),
+             bit32,
++            ptx_hash,
++            _native_base_image: native_base_image,
+         }
+     }
+ 
+@@ -248,7 +285,7 @@ impl Module {
+                 .map(Vec::clone)
+         });
+         mutable
+-            .get_function(self.base, self.sm_version, name.to_owned(), explicit_args)
++            .get_function(self.base, self.sm_version, name.to_owned(), explicit_args, self.ptx_hash)
+             .map(|f| unsafe { (f as *const Function).as_ref().unwrap() })
+     }
+ }
+@@ -256,7 +293,7 @@ impl Module {
+ fn get_best_ptx_and_compile(
+     global_state: &GlobalState,
+     image: CodeLibraryRef<'_>,
+-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
++) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
+     let mut ptx_modules = Vec::new();
+     unsafe {
+         CodeLibraryRef::iterate_modules(image, |_, module| match module {
+@@ -280,7 +317,7 @@ fn get_best_ptx_and_compile(
+         let dump_dir = std::path::PathBuf::from(dump_dir);
+         let _ = fs::create_dir_all(&dump_dir);
+         for (index, source) in ptx_modules.iter().enumerate() {
+-            let stem = format!("module-{index:04}");
++            let stem = format!("module-{:016x}-{index:04}", ptx_identity(source));
+             let _ = fs::write(dump_dir.join(format!("{stem}.ptx")), source.as_bytes());
+             let diagnostic = match ptx_parser::parse_module_checked(source.as_ref()) {
+                 Ok(ast) => format!(
+@@ -345,6 +382,32 @@ fn get_best_ptx_and_compile(
+             )
+         }
+     };
++    let ptx_hash = text.map(|source| ptx_identity(source));
++    // A fully replaced, single-entry module needs no translated binary. LDS
++    // declarations belong to the replaced kernel; exported globals do not.
++    let only_kernel_and_lds = module.directives.iter().all(|directive| match directive {
++        ptx_parser::Directive::Method(_, method) => method.func_directive.name.is_kernel(),
++        ptx_parser::Directive::Variable(_, variable) => matches!(variable.info.state_space, ptx_parser::StateSpace::Shared),
++    });
++    let entries: Vec<_> = module.directives.iter().filter_map(|directive| match directive {
++        ptx_parser::Directive::Method(_, method) => Some(method), _ => None,
++    }).collect();
++    if module.address_size == 64 && only_kernel_and_lds && entries.len() == 1 {
++        let method = entries[0];
++        {
++            if method.func_directive.name.is_kernel() {
++                if let Ok(name) = CString::new(method.func_directive.name.text()) {
++                    if let Some((_, mut owner)) = native_override(&name, ptx_hash) {
++                        let base = owner.module;
++                        owner.module = hipModule_t(std::ptr::null_mut());
++                        let image = mem::take(&mut owner._image);
++                        eprintln!("[d4r-native] module_direct kernel={name:?} translated_jit=skipped");
++                        return Ok((base, module.sm_version, None, ptx_hash, Some(image)));
++                    }
++                }
++            }
++        }
++    }
+     // TODO: get this information on initialization
+     let hip_properties = get_hip_properties()?;
+     let gcn_arch = get_gcn_arch(&hip_properties)?;
+@@ -372,7 +435,12 @@ fn get_best_ptx_and_compile(
+         .or_else(|_| compile_and_cache(gcn_arch, attributes, module, &mut cache_with_key))?;
+     let mut hip_module = unsafe { mem::zeroed() };
+     unsafe { hipModuleLoadData(&mut hip_module, elf_module.as_ptr().cast()) }?;
+-    Ok((hip_module, sm_version, zluda32))
++    Ok((hip_module, sm_version, zluda32, ptx_hash, None))
++}
++
++fn ptx_identity(text: &str) -> u64 {
++    text.trim_end_matches('\0').bytes().fold(0xcbf29ce484222325u64, |hash, byte|
++        (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3))
+ }
+ 
+ fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
+@@ -384,7 +452,7 @@ fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
+ 
+ pub(crate) fn load_hip_module(
+     library: CodeLibraryRef,
+-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
++) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
+     let global_state = driver::global_state()?;
+     get_best_ptx_and_compile(global_state, library)
+ }
+@@ -434,7 +502,7 @@ fn zluda_build_version() -> &'static str {
+                 key.starts_with("D4R_ZLUDA_")
+                     && !matches!(
+                         key.as_str(),
+-                        "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
++                        "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
+                             | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
+                     )
+             })
+@@ -469,7 +537,10 @@ fn loaded_library_fingerprint() -> Option<String> {
+ 
+ #[cfg(windows)]
+ fn loaded_library_fingerprint() -> Option<String> {
+-    None
++    let path = zluda_windows::get_module_path_for_function(zluda_build_version as *const () as usize)?;
++    let metadata = fs::metadata(path).ok()?;
++    let timestamp = metadata.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?;
++    Some(format!("{:x}.{:x}.{:x}", metadata.len(), timestamp.as_secs(), timestamp.subsec_nanos()))
+ }
+ 
+ fn get_cache_key<'a, 'b>(
+@@ -577,16 +648,16 @@ pub(crate) fn load(module: &mut CUmodule, fname: &CStr) -> CUresult {
+     image.push(0);
+     let library = unsafe { CodeLibraryRef::try_load(image.as_ptr() as *const std::ffi::c_void) }
+         .map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
+-    let (hip_module, sm_version, meta32) = load_hip_module(library)?;
+-    *module = Module::new(hip_module, sm_version, meta32).wrap();
++    let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
++    *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
+     Ok(())
+ }
+ 
+ pub(crate) fn load_data(module: &mut CUmodule, image: &std::ffi::c_void) -> CUresult {
+     let library =
+         unsafe { CodeLibraryRef::try_load(image) }.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
+-    let (hip_module, sm_version, meta32) = load_hip_module(library)?;
+-    *module = Module::new(hip_module, sm_version, meta32).wrap();
++    let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
++    *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
+     Ok(())
+ }
+ 

patches/zluda/0011-audited-k-m-windows-replay.patch

@@ -0,0 +1,93 @@
+diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
+index 875b2e3..d491ae2 100644
+--- a/zluda/src/impl/d4r_replay.rs
++++ b/zluda/src/impl/d4r_replay.rs
+@@ -7,6 +7,18 @@ struct Config { directory: PathBuf, filter: String, limit: u32 }
+ static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
+ static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
+ 
++fn parameter_size(name: &str) -> Option<usize> {
++    match name {
++        "dltss_pwin_enc0_layer" | "dltss_pwin_enc1_layer" | "dltss_pwin_enc2_layer" |
++        "dltss_pwin_enc3_layer" | "dltss_pwin_enc4_layer" | "dltss_pwin_dec5_layer" |
++        "dltss_pwin_dec4_layer" | "dltss_pwin_dec3_layer" | "dltss_pwin_dec2_layer" |
++        "dltss_pwin_dec1_layer" | "dltss_pwin_dec0_layer" => Some(176),
++        "rrlite_enc1_4x4" | "rrlite_enc2_4x4" | "rrlite_dec2_4x4" | "rrlite_dec1_4x4" => Some(56),
++        "rrlite_enc3_tube_4x4" => Some(88),
++        _ => None,
++    }
++}
++
+ pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
+ 
+ fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
+@@ -43,8 +55,10 @@ pub(crate) unsafe fn begin(
+         limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
+     })) else { return Ok(None) };
+     let name = name.to_str().map_err(io_error)?;
+-    // Only the audited, single 176-byte K argument ABI is captured here.
+-    if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
++    // Only audited single-by-value K/M argument blocks. Never infer the size
++    // from a prefix: reading a different kernel's parameters can overrun host memory.
++    let Some(parameter_bytes) = parameter_size(name) else { return Ok(None); };
++    if !name.contains(&config.filter) {
+         return Ok(None);
+     }
+     let mut block = std::ptr::null::<u8>();
+@@ -60,10 +74,10 @@ pub(crate) unsafe fn begin(
+         }
+     } else if !parameters.is_null() {
+         block = (*parameters).cast();
+-        bytes = Some(176);
++        bytes = Some(parameter_bytes);
+     }
+-    if block.is_null() || bytes != Some(176) {
+-        return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
++    if block.is_null() || bytes != Some(parameter_bytes) {
++        return Err(io_error(format!("unexpected parameter ABI for {name}: {bytes:?}, expected {parameter_bytes}")));
+     }
+     let sequence = {
+         let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
+@@ -73,14 +87,14 @@ pub(crate) unsafe fn begin(
+         *count
+     };
+     hipStreamSynchronize(stream)?;
+-    let arguments = std::slice::from_raw_parts(block, 176);
++    let arguments = std::slice::from_raw_parts(block, parameter_bytes);
+     let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
+     fs::create_dir_all(&directory).map_err(io_error)?;
+     fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
+-    let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
++    let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs {parameter_bytes}\n",
+         launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
+     let mut allocations = Vec::<(usize, usize)>::new();
+-    for offset in (0..176).step_by(8) {
++    for offset in (0..parameter_bytes).step_by(8) {
+         let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
+         if pointer == 0 { continue; }
+         let mut base = hipDeviceptr_t(std::ptr::null_mut());
+@@ -100,10 +114,23 @@ pub(crate) unsafe fn begin(
+         };
+         manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
+     }
+-    if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
++    if allocations.is_empty() { return Err(io_error("no allocation pointers resolved")); }
+     fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
+     let capture = Capture { directory, allocations };
+     capture.save(&capture.directory)?;
+     eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
+     Ok(Some(capture))
+ }
++
++#[cfg(test)]
++mod tests {
++    #[test]
++    fn only_audited_argument_sizes() {
++        assert_eq!(super::parameter_size("dltss_pwin_enc0_layer"), Some(176));
++        assert_eq!(super::parameter_size("rrlite_dec2_4x4"), Some(56));
++        assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4"), Some(88));
++        assert_eq!(super::parameter_size("dltss_pwin_enc0_layer_prep"), None);
++        assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4_prep"), None);
++        assert_eq!(super::parameter_size("dltss_pwin_unknown_layer"), None);
++    }
++}

patches/zluda/0012-configurable-windows-jit-cache.patch

@@ -0,0 +1,65 @@
+diff --git a/zluda_cache/src/lib.rs b/zluda_cache/src/lib.rs
+index acd262e..c3ac1be 100644
+--- a/zluda_cache/src/lib.rs
++++ b/zluda_cache/src/lib.rs
+@@ -22,18 +22,36 @@ pub struct ModuleCache(SqliteConnection);
+ 
+ impl ModuleCache {
+     pub fn create_cache_dir_and_get_path() -> Option<String> {
+-        let mut cache_dir = dirs::cache_dir()?;
+-        cache_dir.extend(["zluda", "ComputeCache"]);
++        let override_dir = std::env::var_os("ZLUDA_CACHE_DIR");
++        let cache_dir = match override_dir {
++            Some(path) => {
++                let path = std::path::PathBuf::from(path);
++                if !path.is_absolute() {
++                    eprintln!("[zluda-cache] ZLUDA_CACHE_DIR must be absolute; cache disabled");
++                    return None;
++                }
++                path
++            }
++            None => dirs::cache_dir()?.join("zluda").join("ComputeCache"),
++        };
++        Self::cache_path(&cache_dir)
++    }
++
++    fn cache_path(cache_dir: &std::path::Path) -> Option<String> {
+         // We ensure that the cache directory exists
+-        std::fs::create_dir_all(&cache_dir).ok()?;
++        if let Err(error) = std::fs::create_dir_all(cache_dir) {
++            eprintln!("[zluda-cache] cannot create {}: {error}; cache disabled", cache_dir.display());
++            return None;
++        }
+         // No need to create the file, it will be created by SQLite on first access
+         // zluda.db might be in use by older versions of ZLUDA
+-        cache_dir.push("zluda2.db");
+-        Some(cache_dir.to_string_lossy().into())
++        Some(cache_dir.join("zluda2.db").to_string_lossy().into())
+     }
+ 
+     pub fn open(file_path: &str) -> Option<Self> {
+-        let mut conn = SqliteConnection::establish(file_path).ok()?;
++        let mut conn = SqliteConnection::establish(file_path).map_err(|error| {
++            eprintln!("[zluda-cache] cannot open {file_path}: {error}; cache disabled");
++        }).ok()?;
+         conn.batch_execute("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")
+             .ok()?;
+         conn.run_pending_migrations(MIGRATIONS).ok()?;
+@@ -86,6 +104,17 @@ mod tests {
+     use arrayvec::ArrayString;
+     use diesel::prelude::*;
+ 
++    #[test]
++    fn explicit_directory_creates_persistent_database() {
++        let directory = std::env::temp_dir().join(format!("zluda-cache-test-{}-{}", std::process::id(), ModuleCache::time_now()));
++        let path = ModuleCache::cache_path(&directory).unwrap();
++        assert_eq!(std::path::Path::new(&path), directory.join("zluda2.db"));
++        let cache = ModuleCache::open(&path).unwrap();
++        drop(cache);
++        assert!(std::path::Path::new(&path).is_file());
++        std::fs::remove_dir_all(directory).unwrap();
++    }
++
+     #[derive(Queryable, Selectable)]
+     #[diesel(table_name = crate::schema::modules)]
+     #[diesel(check_for_backend(diesel::sqlite::Sqlite))]

patches/zluda/0013-gfx12-f16-reference-accumulation.patch

@@ -0,0 +1,151 @@
+diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
+index 5a54417..90c0034 100644
+--- a/llvm_zluda/src/compile.rs
++++ b/llvm_zluda/src/compile.rs
+@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
+     let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+     if family == 12 {
+         flags |= 2;
++        // Explicit diagnostic/correctness mode; retain upstream's fast default.
++        if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") {
++            flags |= 8;
++        }
+         // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
+         if flags & 1 != 0 && wmma_fp8_native_requested() {
+             flags |= 4;
+diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
+index 07238db..d262fc6 100644
+--- a/llvm_zluda/src/lib.cpp
++++ b/llvm_zluda/src/lib.cpp
+@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M)
+ // Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
+ // Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
+ // Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
++// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs.
+ // The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
+ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+ {
+@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
+     const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
+     const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
+     const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
++    const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16");
+     Helpers H;
+     H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
+     H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
+-    H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
++    H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2);
+     H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
+     H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
+     H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
+-    H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
++    H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
+     {
+         Type *F = Type::getFloatTy(Context);
+         Type *V4F = FixedVectorType::get(F, 4);
+diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
+index 4ff25dd..36bd6b1 100644
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8
+ #endif
+ }
+ 
++// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may
++// reach the other f16 neighbour around cancellation-heavy rounding midpoints.
++// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and
++// round directly to f16 once, matching d4r's independent NumPy/PTX reference.
++// The e4m3 and f32-shadow paths continue using native WMMA.
++__device__ static double wmma12_half_value(uint32_t bits)
++{
++    const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10;
++    const double value = exponent == 0 ? double(magnitude) * 0x1p-24 :
++        double(std::bit_cast<float>(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) :
++                                     ((magnitude << 13) + 0x38000000u)));
++    return bits & 0x8000u ? -value : value;
++}
++
++// Preserve f16 subnormal result bits even when the target's default f16
++// arithmetic mode flushes them. No intermediate f32 rounding is involved.
++__device__ static uint32_t wmma12_half_round(double value)
++{
++    const uint64_t bits = std::bit_cast<uint64_t>(value);
++    const uint32_t sign = uint32_t(bits >> 48) & 0x8000u;
++    const uint32_t exponent = uint32_t((bits >> 52) & 2047u);
++    const uint64_t fraction = bits & 0xfffffffffffffu;
++    if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u);
++    const int power = int(exponent) - 1023;
++    if (power < -25) return sign;
++    if (power > 15) return sign | 0x7c00u;
++    const unsigned shift = power < -14 ? unsigned(28 - power) : 42u;
++    const uint64_t significand = fraction | 0x10000000000000ull;
++    uint32_t code = uint32_t(significand >> shift);
++    const uint64_t tail = significand & ((uint64_t(1) << shift) - 1);
++    const uint64_t midpoint = uint64_t(1) << (shift - 1);
++    code += tail > midpoint || (tail == midpoint && (code & 1u));
++    if (power >= -14) code += uint32_t(power + 14) << 10;
++    return sign | code;
++}
++
++__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired)
++{
++    const bool high = wmma12_half() != 0;
++    const uint2::Native_vec_ own = paired && high ? c1 : c0;
++    const uint2::Native_vec_ send = paired && !high ? c1 : c0;
++    const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
++    const uint2::Native_vec_ even = paired && high ? other : own;
++    const uint2::Native_vec_ odd = paired && high ? own : other;
++    return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u),
++            (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)};
++}
++
++__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c)
++{
++    // x/y: packed even rows, z/w: packed odd rows, in this half-wave.
++    wmma_uint4 result = {};
++    const uint32_t half = wmma12_half();
++    for (uint32_t i = 0; i < 8; ++i)
++    {
++        const uint32_t row = i + 8 * half;
++        double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu);
++        for (uint32_t khalf = 0; khalf < 2; ++khalf)
++            for (uint32_t j = 0; j < 4; ++j)
++            {
++                const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]);
++                const uint32_t other = wmma12_other_half(b[j]);
++                const uint32_t bv = half == khalf ? b[j] : other;
++                sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu);
++                sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16);
++            }
++        // Do not round through f32: that can reintroduce the midpoint error.
++        result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1));
++    }
++    return result;
++}
++
+ // f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
+ // e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
+ __device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+@@ -1959,6 +2031,22 @@ extern "C"
+         return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
+     }
+ 
++    uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
++    {
++        const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false));
++        const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)};
++        return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y};
++    }
++
++    uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
++    {
++        const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true));
++        const bool high = wmma12_half() != 0;
++        const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z);
++        const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w);
++        return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1};
++    }
++
+     wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
+     {
+         return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;

patches/zluda/0014-windows-kernel-profiling.patch

@@ -0,0 +1,148 @@
+From b0161a4166652eee86a8113a38d042b02403db6e Mon Sep 17 00:00:00 2001
+From: Codex <codex@localhost>
+Date: Thu, 1 Oct 2026 18:07:07 +0300
+Subject: [PATCH] Add opt-in Windows HIP event and occupancy diagnostics
+
+---
+ zluda/src/impl/d4r_profile.rs | 60 +++++++++++++++++++++++++++++++++++
+ zluda/src/impl/function.rs    |  9 ++++++
+ zluda/src/impl/mod.rs         |  1 +
+ zluda/src/impl/module.rs      |  2 +-
+ 4 files changed, 71 insertions(+), 1 deletion(-)
+ create mode 100644 zluda/src/impl/d4r_profile.rs
+
+diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
+new file mode 100644
+index 0000000..33ba5f6
+--- /dev/null
++++ b/zluda/src/impl/d4r_profile.rs
+@@ -0,0 +1,60 @@
++//! Optional, deliberately serializing HIP-event diagnostics. Not a production
++//! scheduler or a claim about application FPS / measured hardware occupancy.
++use hip_runtime_sys::*;
++use std::{ffi::CStr, ptr, time::Instant};
++
++pub(super) struct Launch {
++    start: hipEvent_t,
++    end: hipEvent_t,
++    host: Instant,
++}
++
++impl Drop for Launch {
++    fn drop(&mut self) {
++        unsafe {
++            if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
++            if !self.start.is_null() { let _ = hipEventDestroy(self.start); }
++        }
++    }
++}
++
++impl Launch {
++    pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
++        if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
++        let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
++        hipEventCreateWithFlags(&mut result.start, 0)?;
++        hipEventCreateWithFlags(&mut result.end, 0)?;
++        hipEventRecord(result.start, stream)?;
++        result.host = Instant::now();
++        Ok(Some(result))
++    }
++
++    pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
++        phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
++        let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
++        hipEventRecord(self.end, stream)?;
++        hipEventSynchronize(self.end)?;
++        let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
++        let mut gpu_ms = 0.;
++        hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
++        // Module function handles use the Driver API attribute query. The
++        // plural Runtime API expects a registered host function address.
++        let mut lds = 0;
++        let mut regs = 0;
++        let mut private = 0;
++        let mut max_threads = 0;
++        hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
++        hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
++        hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
++        hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
++        let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
++            .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
++        let mut blocks_per_multiprocessor = 0;
++        hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
++            function, threads as i32, shared as usize)?;
++        eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
++            grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
++            regs, private, max_threads);
++        Ok(())
++    }
++}
+diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
+index 7a7470a..c456dfe 100644
+--- a/zluda/src/impl/function.rs
++++ b/zluda/src/impl/function.rs
+@@ -191,6 +191,7 @@ pub(crate) fn launch_kernel(
+         }
+     };
+     if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
++        let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
+         unsafe {
+             hipModuleLaunchKernel(
+                 prep,
+@@ -206,6 +207,9 @@ pub(crate) fn launch_kernel(
+                 extra_for_hip,
+             )
+         }?;
++        if let Some(profile) = profile {
++            unsafe { profile.finish(prep, &f.name, "native", "prep", [blocks,1,1], [128,1,1], 0, stream) }?;
++        }
+         f.prep_last.store(prep_key, std::sync::atomic::Ordering::Relaxed);
+     }
+     let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
+@@ -214,6 +218,7 @@ pub(crate) fn launch_kernel(
+         eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
+             f.name, if f.native_owner.is_some() { "native" } else { "translated" });
+     }
++    let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
+     let result = unsafe {
+         hipModuleLaunchKernel(
+             f.base,
+@@ -230,6 +235,10 @@ pub(crate) fn launch_kernel(
+         )
+     };
+     result?;
++    if let Some(profile) = profile {
++        unsafe { profile.finish(f.base, &f.name, if f.native_owner.is_some() { "native" } else { "translated" },
++            "main", [grid_dim_x, grid_dim_y, grid_dim_z], [block_dim_x, block_dim_y, block_dim_z], shared_mem_bytes, stream) }?;
++    }
+     if let Some(replay) = replay { replay.finish(stream)?; }
+     Ok(())
+ }
+diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
+index 14e0aee..380eb6e 100644
+--- a/zluda/src/impl/mod.rs
++++ b/zluda/src/impl/mod.rs
+@@ -6,6 +6,7 @@ pub(super) mod context;
+ pub(super) mod device;
+ pub(super) mod driver;
+ pub(super) mod d4r_replay;
++pub(super) mod d4r_profile;
+ pub(super) mod event;
+ pub(super) mod function;
+ pub(super) mod graph;
+diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
+index 5a79368..232f7d4 100644
+--- a/zluda/src/impl/module.rs
++++ b/zluda/src/impl/module.rs
+@@ -502,7 +502,7 @@ fn zluda_build_version() -> &'static str {
+                 key.starts_with("D4R_ZLUDA_")
+                     && !matches!(
+                         key.as_str(),
+-                        "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
++                        "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
+                             | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
+                     )
+             })
+-- 
+2.55.0.windows.5
+

patches/zluda/0015-nonserializing-cuda-api-profile.patch

@@ -0,0 +1,83 @@
+From a1c506fc956cfc347c56c44311669700c0d500ac Mon Sep 17 00:00:00 2001
+From: Codex <codex@localhost>
+Date: Thu, 1 Oct 2026 22:16:19 +0300
+Subject: [PATCH] Profile CUDA host API duration without serializing GPU
+ launches
+
+---
+ zluda/src/d4r_api_profile.rs | 34 ++++++++++++++++++++++++++++++++++
+ zluda/src/lib.rs             |  3 +++
+ 2 files changed, 37 insertions(+)
+ create mode 100644 zluda/src/d4r_api_profile.rs
+
+diff --git a/zluda/src/d4r_api_profile.rs b/zluda/src/d4r_api_profile.rs
+new file mode 100644
+index 0000000..ef2da1b
+--- /dev/null
++++ b/zluda/src/d4r_api_profile.rs
+@@ -0,0 +1,34 @@
++//! Host API timings only: no HIP events, stream waits, or altered scheduling.
++use std::{cell::RefCell, collections::BTreeMap, sync::OnceLock, time::Instant};
++
++#[derive(Default)]
++struct Row { calls: u64, total_ms: f64, max_ms: f64 }
++struct Samples { since: Instant, rows: BTreeMap<&'static str, Row> }
++thread_local! {
++    static SAMPLES: RefCell<Samples> = RefCell::new(Samples { since: Instant::now(), rows: BTreeMap::new() });
++}
++pub(crate) struct Call { name: &'static str, start: Instant }
++impl Call {
++    pub(crate) fn begin(name: &'static str) -> Option<Self> {
++        static ENABLED: OnceLock<bool> = OnceLock::new();
++        if !*ENABLED.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_API").is_some()) { return None; }
++        Some(Self { name, start: Instant::now() })
++    }
++}
++impl Drop for Call {
++    fn drop(&mut self) {
++        let ms = self.start.elapsed().as_secs_f64() * 1000.;
++        SAMPLES.with(|samples| {
++            let mut samples = samples.borrow_mut();
++            let row = samples.rows.entry(self.name).or_default();
++            row.calls += 1; row.total_ms += ms; row.max_ms = row.max_ms.max(ms);
++            let period_ms = samples.since.elapsed().as_secs_f64() * 1000.;
++            if period_ms < 2000. { return; }
++            for (name, row) in &samples.rows {
++                eprintln!("D4R_CUDA_API_PROFILE thread={:?} api={name} calls={} total_ms={:.6} max_ms={:.6} period_ms={period_ms:.6} serializing=0",
++                    std::thread::current().id(), row.calls, row.total_ms, row.max_ms);
++            }
++            samples.rows.clear(); samples.since = Instant::now();
++        });
++    }
++}
+diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
+index ecdeb3e..31bfa42 100644
+--- a/zluda/src/lib.rs
++++ b/zluda/src/lib.rs
+@@ -2,6 +2,7 @@ use cuda_types::cuda::CUerror;
+ use std::sync::atomic::{AtomicBool, Ordering};
+ 
+ pub(crate) mod r#impl;
++mod d4r_api_profile;
+ #[cfg_attr(windows, path = "os_win.rs")]
+ #[cfg_attr(not(windows), path = "os_unix.rs")]
+ mod os;
+@@ -40,6 +41,7 @@ macro_rules! implemented {
+                 if !initialized() {
+                     return Err(CUerror::DEINITIALIZED);
+                 }
++                let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
+                 cuda_macros::cuda_normalize_fn!( crate::r#impl::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
+                 Ok(())
+             }
+@@ -57,6 +59,7 @@ macro_rules! implemented_in_function {
+                 if !initialized() {
+                     return Err(CUerror::DEINITIALIZED);
+                 }
++                let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
+                 cuda_macros::cuda_normalize_fn!( crate::r#impl::function::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
+                 Ok(())
+             }
+-- 
+2.55.0.windows.5
+

patches/zluda/0016-deferred-kernel-profile.patch

@@ -0,0 +1,282 @@
+From 26a43f7047ab933a4afb5ebdd26d5ba9e51e58a3 Mon Sep 17 00:00:00 2001
+From: xdfnx-dev <[email protected]>
+Date: Fri, 2 Oct 2026 11:50:45 +0300
+Subject: [PATCH 16/17] Sample HIP kernel events without per-launch completion
+ waits
+
+---
+ zluda/src/impl/context.rs     |   4 +-
+ zluda/src/impl/d4r_profile.rs | 204 ++++++++++++++++++++++++++++------
+ zluda/src/impl/module.rs      |   1 +
+ 3 files changed, 171 insertions(+), 38 deletions(-)
+
+diff --git a/zluda/src/impl/context.rs b/zluda/src/impl/context.rs
+index 9ccb372..374d383 100644
+--- a/zluda/src/impl/context.rs
++++ b/zluda/src/impl/context.rs
+@@ -165,7 +165,7 @@ pub(crate) fn set_cache_config(config: CUfunc_cache) -> CUresult {
+ }
+ 
+ pub(crate) fn synchronize() -> hipError_t {
+-    unsafe { hipDeviceSynchronize() }
++    unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
+ }
+ 
+ pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
+@@ -177,7 +177,7 @@ pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
+     } else {
+         None
+     };
+-    unsafe { hipDeviceSynchronize() }
++    unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
+ }
+ 
+ struct SetDeviceOnDrop(i32);
+diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
+index 33ba5f6..1b9d248 100644
+--- a/zluda/src/impl/d4r_profile.rs
++++ b/zluda/src/impl/d4r_profile.rs
+@@ -1,15 +1,29 @@
+-//! Optional, deliberately serializing HIP-event diagnostics. Not a production
+-//! scheduler or a claim about application FPS / measured hardware occupancy.
++//! Optional HIP-event diagnostics. Deferred mode never waits for its events,
++//! skips the legacy default stream and bounds outstanding events per thread.
++//! Timestamp insertion still adds overhead; this is not a production scheduler.
+ use hip_runtime_sys::*;
+-use std::{ffi::CStr, ptr, time::Instant};
++use std::{cell::RefCell, ffi::{CStr, CString}, ptr, sync::OnceLock, time::Instant};
+ 
+-pub(super) struct Launch {
+-    start: hipEvent_t,
+-    end: hipEvent_t,
+-    host: Instant,
++#[derive(Clone, Copy)]
++enum Mode { Off, Serial, Deferred(u64) }
++fn mode() -> Mode {
++    static MODE: OnceLock<Mode> = OnceLock::new();
++    *MODE.get_or_init(|| {
++        if std::env::var_os("D4R_ZLUDA_PROFILE_DEFERRED").is_some() {
++            let every = std::env::var("D4R_ZLUDA_PROFILE_EVERY").ok()
++                .and_then(|v| v.parse::<u64>().ok()).filter(|n| *n > 0).unwrap_or(17);
++            Mode::Deferred(every)
++        } else if std::env::var_os("D4R_ZLUDA_PROFILE").is_some() { Mode::Serial }
++        else { Mode::Off }
++    })
++}
++fn allow_legacy() -> bool {
++    static ALLOW: OnceLock<bool> = OnceLock::new();
++    *ALLOW.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_ALLOW_LEGACY").is_some())
+ }
+ 
+-impl Drop for Launch {
++struct Events { start: hipEvent_t, end: hipEvent_t }
++impl Drop for Events {
+     fn drop(&mut self) {
+         unsafe {
+             if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
+@@ -17,44 +31,162 @@ impl Drop for Launch {
+         }
+     }
+ }
++impl Events {
++    unsafe fn new() -> Result<Self, hipErrorCode_t> {
++        let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut() };
++        hipEventCreateWithFlags(&mut result.start, 0)?;
++        hipEventCreateWithFlags(&mut result.end, 0)?;
++        Ok(result)
++    }
++}
++
++struct Description {
++    name: CString, backend: String, phase: String, grid: [u32;3], block: [u32;3],
++    shared: u32, lds: i32, regs: i32, private: i32, max_threads: i32, blocks_per_multiprocessor: i32,
++}
++impl Description {
++    unsafe fn new(function: hipFunction_t, name: &CStr, backend: &str, phase: &str,
++        grid: [u32;3], block: [u32;3], shared: u32) -> Result<Self, hipErrorCode_t> {
++        let mut d = Self { name: name.to_owned(), backend: backend.to_owned(), phase: phase.to_owned(),
++            grid, block, shared, lds: 0, regs: 0, private: 0, max_threads: 0, blocks_per_multiprocessor: 0 };
++        // Copy metadata while the function/module is alive. Collection never
++        // dereferences a function handle that the application may have unloaded.
++        hipFuncGetAttribute(&mut d.lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
++        hipFuncGetAttribute(&mut d.regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
++        hipFuncGetAttribute(&mut d.private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
++        hipFuncGetAttribute(&mut d.max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
++        let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
++            .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
++        hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut d.blocks_per_multiprocessor,
++            function, threads as i32, shared as usize)?;
++        Ok(d)
++    }
++    fn print(&self, gpu_ms: f32, enqueue_ms: f64, completion_ms: f64, serializing: bool, stream: usize) {
++        if !gpu_ms.is_finite() || gpu_ms < 0. {
++            eprintln!("D4R_KERNEL_PROFILE_INVALID kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} reason=invalid_elapsed_time serializing={} deferred={} stream=0x{stream:x}",
++                self.name, self.backend, self.phase, u32::from(serializing), u32::from(!serializing));
++            return;
++        }
++        eprintln!("D4R_KERNEL_PROFILE kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={} serializing={} deferred={} stream=0x{stream:x} legacy_stream={}",
++            self.name, self.backend, self.phase, self.grid[0], self.grid[1], self.grid[2],
++            self.block[0], self.block[1], self.block[2], self.lds, self.shared, self.regs,
++            self.private, self.max_threads, self.blocks_per_multiprocessor,
++            u32::from(serializing), u32::from(!serializing), u32::from(stream <= 1));
++    }
++}
++
++struct Pending { launch: Launch, description: Description, enqueue_ms: f64, stream: usize }
++const CAPACITY: usize = 64;
++struct Deferred {
++    calls: u64, skipped: u64, pending: Vec<Pending>, free: Vec<(i32, Events)>,
++}
++thread_local! {
++    static DEFERRED: RefCell<Deferred> = RefCell::new(Deferred {
++        calls: 0, skipped: 0, pending: Vec::new(), free: Vec::new(),
++    });
++}
++
++impl Deferred {
++    unsafe fn collect(&mut self, device: i32) -> hipError_t {
++        let mut index = 0;
++        while index < self.pending.len() {
++            let p = &self.pending[index];
++            if p.launch.device != device { index += 1; continue; }
++            match hipEventQuery(p.launch.events.end) {
++                Err(e) if e == hipErrorCode_t::NotReady => { index += 1; continue; }
++                result => result?,
++            }
++            let mut gpu_ms = 0.;
++            hipEventElapsedTime(&mut gpu_ms, p.launch.events.start, p.launch.events.end)?;
++            p.description.print(gpu_ms, p.enqueue_ms, p.launch.host.elapsed().as_secs_f64() * 1000., false, p.stream);
++            let p = self.pending.swap_remove(index);
++            // Re-record only after the old timestamps have been collected.
++            self.free.push((device, p.launch.events));
++        }
++        Ok(())
++    }
++}
++
++/// Called only after the application's existing context synchronization, or
++/// before a selected launch. No event/stream/device wait is added here.
++pub(super) unsafe fn collect_completed() -> hipError_t {
++    if !matches!(mode(), Mode::Deferred(_)) { return Ok(()); }
++    DEFERRED.with(|state| {
++        let mut state = state.borrow_mut();
++        if state.pending.is_empty() { return Ok(()); }
++        let mut device = 0; hipGetDevice(&mut device)?;
++        state.collect(device)
++    })
++}
++
++pub(super) struct Launch {
++    events: Events,
++    host: Instant,
++    device: i32,
++    deferred: bool,
++}
+ 
+ impl Launch {
+     pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
+-        if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
+-        let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
+-        hipEventCreateWithFlags(&mut result.start, 0)?;
+-        hipEventCreateWithFlags(&mut result.end, 0)?;
+-        hipEventRecord(result.start, stream)?;
+-        result.host = Instant::now();
+-        Ok(Some(result))
++        let mode = mode();
++        if matches!(mode, Mode::Off) { return Ok(None); }
++        if let Mode::Deferred(every) = mode {
++            return DEFERRED.with(|state| {
++                let mut state = state.borrow_mut();
++                state.calls = state.calls.wrapping_add(1);
++                if (state.calls - 1) % every != 0 { return Ok(None); }
++                // HIP timing events on the legacy default stream can order
++                // other streams. Do not introduce that dependency in this mode.
++                if stream.0 as usize <= 1 && !allow_legacy() {
++                    state.skipped += 1;
++                    if state.skipped.is_power_of_two() {
++                        eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=legacy_default_stream count={} serializing=0", state.skipped);
++                    }
++                    return Ok(None);
++                }
++                let mut device = 0; hipGetDevice(&mut device)?;
++                state.collect(device)?;
++                let mut capture = hipStreamCaptureStatus::hipStreamCaptureStatusNone;
++                hipStreamIsCapturing(stream, &mut capture)?;
++                if capture != hipStreamCaptureStatus::hipStreamCaptureStatusNone {
++                    state.skipped += 1;
++                    eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=stream_capture count={} serializing=0", state.skipped);
++                    return Ok(None);
++                }
++                let events = if let Some(index) = state.free.iter().position(|(d, _)| *d == device) {
++                    state.free.swap_remove(index).1
++                } else if state.pending.len() + state.free.len() < CAPACITY {
++                    Events::new()?
++                } else {
++                    state.skipped += 1;
++                    eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=capacity count={} capacity={CAPACITY} serializing=0", state.skipped);
++                    return Ok(None);
++                };
++                hipEventRecord(events.start, stream)?;
++                Ok(Some(Self { events, host: Instant::now(), device, deferred: true }))
++            });
++        }
++        let events = Events::new()?;
++        hipEventRecord(events.start, stream)?;
++        Ok(Some(Self { events, host: Instant::now(), device: -1, deferred: false }))
+     }
+ 
+     pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
+         phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
+         let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
+-        hipEventRecord(self.end, stream)?;
+-        hipEventSynchronize(self.end)?;
++        hipEventRecord(self.events.end, stream)?;
++        let description = Description::new(function, name, backend, phase, grid, block, shared)?;
++        if self.deferred {
++            DEFERRED.with(|state| state.borrow_mut().pending.push(Pending {
++                launch: self, description, enqueue_ms, stream: stream.0 as usize,
++            }));
++            return Ok(());
++        }
++        hipEventSynchronize(self.events.end)?;
+         let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
+         let mut gpu_ms = 0.;
+-        hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
+-        // Module function handles use the Driver API attribute query. The
+-        // plural Runtime API expects a registered host function address.
+-        let mut lds = 0;
+-        let mut regs = 0;
+-        let mut private = 0;
+-        let mut max_threads = 0;
+-        hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
+-        hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
+-        hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
+-        hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
+-        let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
+-            .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
+-        let mut blocks_per_multiprocessor = 0;
+-        hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
+-            function, threads as i32, shared as usize)?;
+-        eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
+-            grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
+-            regs, private, max_threads);
++        hipEventElapsedTime(&mut gpu_ms, self.events.start, self.events.end)?;
++        description.print(gpu_ms, enqueue_ms, completion_ms, true, stream.0 as usize);
+         Ok(())
+     }
+ }
+diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
+index 232f7d4..8d43abf 100644
+--- a/zluda/src/impl/module.rs
++++ b/zluda/src/impl/module.rs
+@@ -503,6 +503,7 @@ fn zluda_build_version() -> &'static str {
+                     && !matches!(
+                         key.as_str(),
+                         "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
++                            | "D4R_ZLUDA_PROFILE_API" | "D4R_ZLUDA_PROFILE_DEFERRED" | "D4R_ZLUDA_PROFILE_EVERY" | "D4R_ZLUDA_PROFILE_ALLOW_LEGACY"
+                             | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
+                     )
+             })
+-- 
+2.55.0.windows.5
+

patches/zluda/0017-asynchronous-2d-array-copies.patch

@@ -0,0 +1,50 @@
+From 67127dde599879ada022d6439383d682e09e0b36 Mon Sep 17 00:00:00 2001
+From: xdfnx-dev <[email protected]>
+Date: Fri, 2 Oct 2026 11:50:45 +0300
+Subject: [PATCH 17/17] Implement asynchronous 2D array copies for Windows
+ input batching
+
+---
+ zluda/src/impl/memory.rs | 13 +++++++++++++
+ zluda/src/lib.rs         |  1 +
+ 2 files changed, 14 insertions(+)
+
+diff --git a/zluda/src/impl/memory.rs b/zluda/src/impl/memory.rs
+index b724942..42facb3 100644
+--- a/zluda/src/impl/memory.rs
++++ b/zluda/src/impl/memory.rs
+@@ -469,6 +469,19 @@ pub(crate) unsafe fn copy_2d_v2(memcpy: hip_Memcpy2D) -> CUresult {
+     Ok(())
+ }
+ 
++pub(crate) unsafe fn copy_2d_async_v2(memcpy: hip_Memcpy2D, stream: hipStream_t) -> CUresult {
++    // The format-80 emulation in copy_2d_v2 uses temporary CPU storage.
++    // Reject it explicitly here: an asynchronous copy cannot outlive those
++    // temporaries, and the Windows fast path uses canonical float arrays.
++    let src_packed = memcpy.srcMemoryType == hipMemoryType::hipMemoryTypeArray
++        && array::original_format(memcpy.srcArray)? == Some(80);
++    let dst_packed = memcpy.dstMemoryType == hipMemoryType::hipMemoryTypeArray
++        && array::original_format(memcpy.dstArray)? == Some(80);
++    if src_packed || dst_packed { return Err(CUerror::NOT_SUPPORTED); }
++    hipMemcpyParam2DAsync(&memcpy, stream)?;
++    Ok(())
++}
++
+ pub(crate) unsafe fn copy_2d_unaligned_v2(memcpy: hip_Memcpy2D) -> CUresult {
+     hipDrvMemcpy2DUnaligned(&memcpy)?;
+     Ok(())
+diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
+index 31bfa42..939981e 100644
+--- a/zluda/src/lib.rs
++++ b/zluda/src/lib.rs
+@@ -150,6 +150,7 @@ cuda_macros::cuda_function_declarations!(
+             cuMemAlloc_v2,
+             cuMemAllocPitch_v2,
+             cuMemcpy2D_v2,
++            cuMemcpy2DAsync_v2,
+             cuMemcpy2DUnaligned_v2,
+             cuMemcpy3D_v2,
+             cuMemcpyAsync,
+-- 
+2.55.0.windows.5
+

patches/zluda/0018-opt-in-rdna-wgp-placement.patch

@@ -0,0 +1,93 @@
+From fb0adc8714f57647320f0601533ec50331e378b0 Mon Sep 17 00:00:00 2001
+From: xdfnx-dev <[email protected]>
+Date: Fri, 2 Oct 2026 14:40:56 +0300
+Subject: [PATCH] Add opt-in RDNA CU/WGP placement for matched native
+ experiments
+
+---
+ llvm_zluda/src/compile.rs | 42 ++++++++++++++++++++++++++++++++++-----
+ ptx/src/pass/llvm/emit.rs |  3 ++-
+ 2 files changed, 39 insertions(+), 6 deletions(-)
+
+diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
+index 90c0034..6a2f452 100644
+--- a/llvm_zluda/src/compile.rs
++++ b/llvm_zluda/src/compile.rs
+@@ -24,6 +24,37 @@ const OCKL_MODULE: &[u8] = include_bytes!("device-libs/ockl.bc");
+ // https://llvm.org/docs/AMDGPUUsage.html#address-spaces
+ const CONSTANT_ADDRESS_SPACE: u32 = 4;
+ 
++/// Opt-in RDNA scheduling experiment; changes placement, not FP semantics.
++pub fn wgp_requested() -> bool {
++    std::env::var("D4R_ZLUDA_WGP").is_ok_and(|value| value.trim() == "1")
++}
++
++fn target_features(wave64: bool, wgp: bool) -> &'static CStr {
++    match (wave64, wgp) {
++        (false, false) => c"-wavefrontsize64,+cumode",
++        (true, false) => c"-wavefrontsize32,+wavefrontsize64,+cumode",
++        (false, true) => c"-wavefrontsize64,-cumode",
++        (true, true) => c"-wavefrontsize32,+wavefrontsize64,-cumode",
++    }
++}
++
++#[cfg(test)]
++mod shader_mode_tests {
++    use super::target_features;
++
++    #[test]
++    fn wgp_changes_only_cu_placement_in_both_wave_modes() {
++        for wave64 in [false, true] {
++            let cu = target_features(wave64, false).to_str().unwrap();
++            let wgp = target_features(wave64, true).to_str().unwrap();
++            assert_eq!(cu.replace("+cumode", "-cumode"), wgp);
++            assert_eq!(cu.contains("+wavefrontsize64"), wave64);
++            assert_eq!(wgp.contains("+wavefrontsize64"), wave64);
++        }
++        assert_eq!(target_features(false, false), c"-wavefrontsize64,+cumode");
++    }
++}
++
+ fn load_module(ctx: &Context, bc: &[u8], name: &std::ffi::CStr) -> Result<Module, String> {
+     let module = Module::try_from_bitcode(ctx, bc, Some(name))
+         .ok_or(("Failed to parse bitcode").to_string())?;
+@@ -229,11 +260,12 @@ pub fn compile(
+     let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
+     // functions without their own target features (the linked helper libraries) take these, so under
+     // D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
+-    let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
+-        c"-wavefrontsize32,+wavefrontsize64,+cumode"
+-    } else {
+-        c"-wavefrontsize64,+cumode"
+-    };
++    let wgp = wgp_requested();
++    if wgp && !gcn_arch.starts_with("gfx11") && !gcn_arch.starts_with("gfx12") {
++        return Err(format!("D4R_ZLUDA_WGP requires RDNA3/4, got {gcn_arch}"));
++    }
++    let features = target_features(
++        std::env::var("D4R_ZLUDA_WAVE64").is_ok_and(|v| v.trim() == "1"), wgp);
+ 
+     let mut target = unsafe { std::mem::zeroed() };
+     let mut err = std::ptr::null_mut();
+diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
+index 6af2495..8b67c93 100644
+--- a/ptx/src/pass/llvm/emit.rs
++++ b/ptx/src/pass/llvm/emit.rs
+@@ -397,12 +397,13 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
+ 
+     fn emit_target_features(&mut self, fn_: LLVMValueRef) {
+         let value = format!(
+-            "{},+cumode{}",
++            "{},{}{}",
+             if crate::pass::wave64_requested() {
+                 "-wavefrontsize32,+wavefrontsize64"
+             } else {
+                 "+wavefrontsize32,-wavefrontsize64"
+             },
++            if llvm_zluda::compile::wgp_requested() { "-cumode" } else { "+cumode" },
+             if cfg!(debug_assertions) {
+                 ",+precise-memory"
+             } else {
+-- 
+2.55.0.windows.5
+

scripts/build_d3d12_dlss_harness.sh

@@ -5,7 +5,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 MINGW_CXX="${MINGW_CXX:-x86_64-w64-mingw32-g++}"
 
 "$ROOT/scripts/build_d4r_nvngx_shim.sh"
-"$MINGW_CXX" -std=c++20 -O2 -Wall -Wextra -static -static-libgcc -static-libstdc++ \
+"$MINGW_CXX" -std=c++20 -O2 -Wall -Wextra -I"$ROOT/build/vulkan-include" -static -static-libgcc -static-libstdc++ \
   "$ROOT/tools/d3d12_dlss_harness.cpp" "$ROOT/tools/d4r_ngx_param_host.cpp" "$ROOT/build/d4r_ngx_param_msvc.obj" \
   -o "$ROOT/build/d3d12_dlss_harness.exe"
 printf 'Built %s\n' "$ROOT/build/d3d12_dlss_harness.exe"

scripts/build_d4r_motion_kernels.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Build d4r-owned motion processing kernels and embed them in the Wine bridge.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+OUT="${1:-$ROOT/build/wine-nvcuda/generated}"
+ROCM="${D4R_ROCM_DIR:-/opt/rocm}"
+CLANG="$ROCM/lib/llvm/bin/clang++"
+[[ -x "$CLANG" ]] || { echo "motion kernels need ROCm clang (D4R_ROCM_DIR)" >&2; exit 2; }
+mkdir -p "$OUT"
+for arch in gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201; do
+    "$CLANG" -x hip -nogpuinc -nogpulib --offload-device-only --offload-arch="$arch" \
+        --no-gpu-bundle-output -O3 -mno-wavefrontsize64 -o "$OUT/$arch.hsaco" \
+        "$ROOT/kernels/support/motion_dilate.hip"
+done
+python3 - "$OUT" <<'PY'
+from pathlib import Path
+import sys
+root = Path(sys.argv[1])
+lines = ['/* Generated from d4r-owned HIP source; do not edit. */', '#include <string.h>']
+for p in sorted(root.glob('gfx*.hsaco')):
+    data = p.read_bytes()
+    lines.append(f'static const unsigned char motion_{p.stem}[] __attribute__((aligned(64))) = {{')
+    lines.extend(','.join(f'0x{x:02x}' for x in data[i:i+24]) + ',' for i in range(0,len(data),24))
+    lines.append('};')
+lines.append('static const void* d4r_motion_image(const char* arch) {')
+for p in sorted(root.glob('gfx*.hsaco')):
+    lines.append(f'  if (strcmp(arch, "{p.stem}") == 0) return motion_{p.stem};')
+lines.append('  return NULL;\n}')
+(root/'d4r_motion_kernels.h').write_text('\n'.join(lines)+'\n')
+PY

scripts/build_d4r_nvngx_shim.sh

@@ -16,7 +16,7 @@ ln -sfn "$VULKAN_INCLUDE/vulkan" "$ROOT/build/vulkan-include/vulkan"
 ln -sfn "$VULKAN_INCLUDE/vk_video" "$ROOT/build/vulkan-include/vk_video"
 PARAM_OBJECT="$ROOT/build/d4r_ngx_param_msvc.obj"
 "$CLANG_CL" --target=x86_64-pc-windows-msvc /nologo /std:c++20 /O2 /c /GS- /GR- /EHs-c- /Zl \
-  "/Fo$PARAM_OBJECT" "$ROOT/tools/d4r_ngx_param_msvc.cpp"
+  "/Fo$PARAM_OBJECT" -- "$ROOT/tools/d4r_ngx_param_msvc.cpp"
 "$MINGW_CXX" -std=c++20 -O2 -Wall -Wextra -Wno-missing-field-initializers -I"$ROOT/build/vulkan-include" -shared -static -static-libgcc -static-libstdc++ \
   "$ROOT/tools/d4r_nvngx_shim.cpp" "$ROOT/tools/d4r_ngx_param_host.cpp" "$PARAM_OBJECT" \
   -o "$ROOT/build/d4r_nvngx.dll"

scripts/build_release_glibc241.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Run inside packaging/build/Dockerfile.glibc241 with a recent official Rust toolchain and ROCm mounted.
+# Supply isolated, patched ZLUDA/vkd3d checkouts and the usual package_release.sh inputs.
+set -euo pipefail
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+: "${D4R_RELEASE_BUILD_ROOT:?set the build output directory}"
+: "${D4R_ZLUDA_SRC:?set a build-local ZLUDA checkout with patches 0002-0007 applied}"
+: "${D4R_VKD3D_SRC:?set a build-local vkd3d-proton checkout with patches 0001 and 0002 applied}"
+: "${D4R_ROCM_DIR:?set ROCm with clang and device libraries}"
+BUILD="$(realpath -m "$D4R_RELEASE_BUILD_ROOT")"
+[[ "$(getconf GNU_LIBC_VERSION)" == 'glibc 2.41' ]] || {
+    echo "This release build requires GLIBC 2.41; use the pinned Debian build image" >&2; exit 2;
+}
+# Helper generation modifies its source checkout. Refuse the external source-cache directory.
+case "$(realpath "$D4R_ZLUDA_SRC")" in "$BUILD"/*) ;; *) echo "ZLUDA source must be inside $BUILD" >&2; exit 2 ;; esac
+case "$(realpath "$D4R_VKD3D_SRC")" in "$BUILD"/*) ;; *) echo "vkd3d source must be inside $BUILD" >&2; exit 2 ;; esac
+mkdir -p "$BUILD/logs"
+for source in "$D4R_ZLUDA_SRC/ext/llvm-project/llvm/CMakeLists.txt" "$D4R_ZLUDA_SRC/ext/HiGHS/CMakeLists.txt"; do
+    [[ -f "$source" ]] || { echo "missing pinned source submodule: $source" >&2; exit 2; }
+done
+export CARGO_TARGET_DIR="$BUILD/zluda-target"
+export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-8}"
+export CMAKE_BUILD_PARALLEL_LEVEL="$CARGO_BUILD_JOBS"
+export LIBRARY_PATH="$D4R_ROCM_DIR/lib${D4R_ROCM_LINK_STUBS:+:$D4R_ROCM_LINK_STUBS}"
+export LD_LIBRARY_PATH="$D4R_ROCM_DIR/lib"
+export HIP_PATH="$D4R_ROCM_DIR"
+export ROCM_PATH="$D4R_ROCM_DIR"
+export HIP_CLANG_PATH="$D4R_ROCM_DIR/lib/llvm/bin"
+{
+    getconf GNU_LIBC_VERSION
+    rustc --version
+    cargo --version
+    gcc --version | head -1
+    winegcc --version | head -1
+    "$D4R_ROCM_DIR/lib/llvm/bin/clang++" --version | head -1
+} > "$BUILD/toolchain.txt"
+
+ZLUDA_SOURCE_ROOT="$D4R_ZLUDA_SRC" ROCM_ROOT="$D4R_ROCM_DIR" \
+    D4R_ZLUDA_PTX_HELPER_WORK="$BUILD/helpers" "$ROOT/scripts/build_zluda_ptx_helpers.sh" \
+    > "$BUILD/logs/helpers.log" 2>&1
+echo 'Device helpers rebuilt'
+(cd "$D4R_ZLUDA_SRC" && cargo build --offline --locked --release -p zluda \
+    && cargo build --offline --locked --release -p ptx --example d4r_emit) \
+    > "$BUILD/logs/zluda.log" 2>&1
+export D4R_ZLUDA_DIR="$CARGO_TARGET_DIR/release"
+export D4R_ZLUDA_EMIT="$D4R_ZLUDA_DIR/examples/d4r_emit"
+"$ROOT/scripts/check_glibc_compat.sh" 2.41 "$D4R_ZLUDA_DIR/libnvcuda.so"
+echo 'ZLUDA and the offline emitter rebuilt; GLIBC check passed'
+# A minimal ROCm compiler extraction may omit lld. Use the matching LLVM built above so
+# all GPU targets link with the same AMDGPU/ELF support, inside the compatible environment.
+for llvm_build in "$CARGO_TARGET_DIR"/release/build/llvm_zluda-*/out/build; do
+    [[ -f "$llvm_build/build.ninja" ]] || continue
+    ninja -j "$CARGO_BUILD_JOBS" -C "$llvm_build" lld > "$BUILD/logs/lld.log" 2>&1
+    export PATH="$llvm_build/bin:$PATH"
+    break
+done
+command -v lld >/dev/null || { echo 'missing AMDGPU linker' >&2; exit 2; }
+
+"$ROOT/scripts/build_d4r_nvngx_shim.sh" > "$BUILD/logs/shim.log" 2>&1
+"$ROOT/scripts/build_wine_nvcuda_bridge.sh" > "$BUILD/logs/bridge.log" 2>&1
+"$ROOT/scripts/check_glibc_compat.sh" 2.41 "$ROOT/build/wine-nvcuda/x86_64-unix/nvcuda.dll.so"
+if [[ ! -f "$BUILD/vkd3d-build/build.ninja" ]]; then
+    meson setup --cross-file "$D4R_VKD3D_SRC/build-win64.txt" --buildtype release \
+        -Denable_tests=false "$BUILD/vkd3d-build" "$D4R_VKD3D_SRC" > "$BUILD/logs/vkd3d.log" 2>&1
+fi
+ninja -j "$CARGO_BUILD_JOBS" -C "$BUILD/vkd3d-build" >> "$BUILD/logs/vkd3d.log" 2>&1
+export D4R_VKD3D_DIR="$BUILD/vkd3d"
+mkdir -p "$D4R_VKD3D_DIR"
+x86_64-w64-mingw32-strip -o "$D4R_VKD3D_DIR/d3d12.dll" "$BUILD/vkd3d-build/libs/d3d12/d3d12.dll"
+x86_64-w64-mingw32-strip -o "$D4R_VKD3D_DIR/d3d12core.dll" "$BUILD/vkd3d-build/libs/d3d12core/d3d12core.dll"
+echo 'Shim, bridge and vkd3d-proton rebuilt'
+
+export D4R_BUNDLE_TEX="$BUILD/kernels"
+export D4R_DLSS_PTX_DIR="$BUILD/ptx"
+python3 "$ROOT/kernels/tools/extract_dlss_ptx.py" "$D4R_BUNDLE_DLSS" "$D4R_DLSS_PTX_DIR" > "$BUILD/logs/extract.log"
+build_textures() {
+    local arch="$1" folder fp8 accuracy out
+    folders=("$arch")
+    [[ "$arch" == gfx12* ]] && folders+=("$arch-fp8")
+    for folder in "${folders[@]}"; do
+        fp8=0
+        [[ "$folder" == *-fp8 ]] && fp8=1
+        for accuracy in 0 1; do
+            out="$D4R_BUNDLE_TEX/$folder"
+            [[ "$accuracy" == 1 ]] && out="$D4R_BUNDLE_TEX/accuracy/$folder"
+            D4R_PREFER_ACCURACY="$accuracy" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" \
+                D4R_DLSS_DLL="$D4R_BUNDLE_DLSS" "$ROOT/kernels/build.sh" tex "$out" \
+                > "$BUILD/logs/tex-$folder-$accuracy.log" 2>&1
+            echo "Texture kernels rebuilt: $folder accuracy=$accuracy"
+        done
+    done
+}
+pending=()
+failed=0
+for arch in ${D4R_GPU_ARCHS:-gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201}; do
+    build_textures "$arch" &
+    pending+=("$!")
+    if (( ${#pending[@]} >= 4 )); then
+        wait "${pending[0]}" || failed=1
+        pending=("${pending[@]:1}")
+    fi
+done
+for pid in "${pending[@]}"; do wait "$pid" || failed=1; done
+[[ "$failed" == 0 ]] || { echo 'Texture target build failed' >&2; exit 1; }
+python3 -m unittest discover -s "$ROOT/tests" -v > "$BUILD/logs/tests.log" 2>&1
+export D4R_SKIP_BUILD=1
+export D4R_MAX_GLIBC=2.41
+export D4R_BUILD_INFO="$BUILD/toolchain.txt"
+export D4R_PACKAGE_KERNEL_JOBS=4
+"$ROOT/scripts/package_release.sh" "$BUILD/dist" > "$BUILD/logs/package.log" 2>&1
+echo "Release packaged in $BUILD/dist"

scripts/build_wine_nvcuda_bridge.sh

@@ -7,7 +7,13 @@ WINEBUILD="${WINEBUILD:-winebuild}"
 BRIDGE_ROOT="$ROOT/build/wine-nvcuda"
 
 mkdir -p "$BRIDGE_ROOT/x86_64-unix" "$BRIDGE_ROOT/x86_64-windows"
+MOTION_FLAGS=()
+if [[ -x "${D4R_ROCM_DIR:-/opt/rocm}/lib/llvm/bin/clang++" ]]; then
+  "$ROOT/scripts/build_d4r_motion_kernels.sh" "$BRIDGE_ROOT/generated"
+  MOTION_FLAGS=(-DD4R_MOTION_KERNELS -I"$BRIDGE_ROOT/generated")
+fi
 "$WINEGCC" -m64 -shared -o "$BRIDGE_ROOT/x86_64-unix/nvcuda.dll" \
+  "${MOTION_FLAGS[@]}" \
   "$ROOT/tools/wine_nvcuda_bridge.c" \
   -Wb,--export="$ROOT/tools/wine_nvcuda_bridge.spec" -ldl -lpthread
 "$WINEBUILD" --dll --fake-module -E "$ROOT/tools/wine_nvcuda_bridge.spec" \

scripts/build_zluda_ptx_helpers.sh

@@ -9,6 +9,9 @@ LLVM_DIS="${LLVM_DIS:-$ROCM_ROOT/lib/llvm/bin/llvm-dis}"
 LLVM_AS="${LLVM_AS:-$ROCM_ROOT/lib/llvm/bin/llvm-as}"
 OCML="${OCML:-$ROCM_ROOT/amdgcn/bitcode/ocml.bc}"
 WORK_ROOT="${D4R_ZLUDA_PTX_HELPER_WORK:-${TMPDIR:-/tmp}/d4r-zluda-ptx-helpers}"
+# D4R_ZLUDA_HELPER_FLAGS: extra compiler flags, e.g. -DD4R_WMMA12_SHIM for the gfx12-layout test build (never
+# ship it: its gfx12 helpers use gfx11 WMMAs)
+read -r -a EXTRA_FLAGS <<< "${D4R_ZLUDA_HELPER_FLAGS:-}"
 
 if [[ -z "$ZLUDA_SOURCE_ROOT" || ! -f "$ZLUDA_SOURCE_ROOT/ptx/lib/zluda_ptx_impl.cpp" ]]; then
   printf 'Set ZLUDA_SOURCE_ROOT to a ZLUDA source checkout\n' >&2
@@ -25,9 +28,10 @@ mkdir -p "$WORK_ROOT"
 SOURCE="$ZLUDA_SOURCE_ROOT/ptx/lib/zluda_ptx_impl.cpp"
 PTX_LIB="$ZLUDA_SOURCE_ROOT/ptx/lib"
 COMMON_FLAGS=(-DHIP_ENABLE_WARP_SYNC_BUILTINS -std=c++20 -Xclang -fdenormal-fp-math=dynamic
+  --rocm-path="$ROCM_ROOT" -I"$ROCM_ROOT/include"
   -Wall -Wextra -Wsign-compare -Wconversion -x hip "$SOURCE" -nogpulib -O3
   -mno-wavefrontsize64 --offload-device-only --offload-arch=gfx1030 -emit-llvm -c
-  -Xclang -mlink-bitcode-file -Xclang "$OCML")
+  -Xclang -mlink-bitcode-file -Xclang "$OCML" "${EXTRA_FLAGS[@]}")
 
 build_one() {
   local suffix="$1"
@@ -55,3 +59,6 @@ build_one() {
 
 build_one ""
 build_one "_constrained" -ffp-model=strict -ffp-exception-behavior=ignore
+# wave64 helpers (D4R_ZLUDA_WAVE64: two CUDA warps per wave; see d4r_warp_base in zluda_ptx_impl.cpp)
+build_one "_w64" -mwavefrontsize64 -DD4R_WAVE64
+build_one "_constrained_w64" -ffp-model=strict -ffp-exception-behavior=ignore -mwavefrontsize64 -DD4R_WAVE64

scripts/check_glibc_compat.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Reject Linux release libraries that need a newer GLIBC than the target runtime.
+# Usage: check_glibc_compat.sh MAX_GLIBC LIBRARY [LIBRARY ...]
+set -euo pipefail
+
+if [[ $# -lt 2 || ! "$1" =~ ^[0-9]+\.[0-9]+$ ]]; then
+  printf 'usage: %s MAX_GLIBC LIBRARY [LIBRARY ...]\n' "$0" >&2
+  exit 2
+fi
+max="GLIBC_$1"
+shift
+command -v readelf >/dev/null || { echo 'readelf is required to check GLIBC compatibility' >&2; exit 2; }
+
+failed=0
+for library in "$@"; do
+  if [[ ! -f "$library" ]]; then
+    printf 'missing library: %s\n' "$library" >&2
+    failed=1
+    continue
+  fi
+  if ! versions="$(readelf -W --version-info "$library" 2>&1)"; then
+    printf 'cannot inspect %s: %s\n' "$library" "$versions" >&2
+    failed=1
+    continue
+  fi
+  required="$(printf '%s\n' "$versions" | grep -Eo 'GLIBC_[0-9]+\.[0-9]+' | sort -Vu | tail -1 || true)"
+  if [[ -n "$required" && "$(printf '%s\n%s\n' "$max" "$required" | sort -Vu | tail -1)" != "$max" ]]; then
+    printf '%s requires %s (target: at most %s)\n' "$library" "$required" "$max" >&2
+    failed=1
+  fi
+done
+exit "$failed"

scripts/d4r_config.py

@@ -75,6 +75,7 @@ def main(argv):
         return
 
     env = {}
+    env["D4R_PREFER_ACCURACY"] = "1" if flag("Kernels", "PreferAccuracy") else "0"
 
     model = get("DLSS", "Model")
     if model is not None:
@@ -129,18 +130,27 @@ def main(argv):
         value = flag("Kernels", key)
         if value is not None:
             env[var] = "1" if value else "0"
+    # RDNA4's native FP8 WMMA (on unless set off; ZLUDA and the bridge ignore it on other GPUs)
+    value = flag("Kernels", "NativeFp8")
+    env["D4R_ZLUDA_WMMA_FP8_NATIVE"] = "0" if value is False else "1"
+    value = flag("Kernels", "NativeSwinEncoders")
+    env["D4R_NATIVE_SWIN_ENCODERS"] = "0" if value is False else "1"
     value = get("Kernels", "ImplicitMaxBlock")
     if value:
         env["D4R_ZLUDA_IMPLICIT_MAX_BLOCK"] = value
 
     for key, var in (("VramInterop", "D4R_SHIM_VRAM_INTEROP"), ("InputSync", "D4R_SHIM_INPUT_SYNC"),
-                     ("GpuWait", "D4R_SHIM_GPU_WAIT"), ("LinearInputs", "D4R_SHIM_LINEAR_INPUTS"),
+                     ("LinearInputs", "D4R_SHIM_LINEAR_INPUTS"),
                      ("DirectOutput", "D4R_SHIM_OUTPUT_DIRECT"), ("ElideNgxSync", "D4R_ELIDE_NGX_SYNC"),
                      ("EvalSync", "D4R_SHIM_EVAL_SYNC")):
         value = flag("Interop", key)
         if value is not None:
             env[var] = "1" if value else "0"
     value = get("Interop", "MarkerPollUs")
+    dilation = get("Interop", "MotionVectorDilation")
+    if dilation is not None and dilation not in ("0", "1", "2"):
+        fail("[Interop] MotionVectorDilation must be 0, 1 or 2")
+    env["D4R_MOTION_DILATION"] = dilation or "0"
     if value:
         if not value.isdigit():
             fail(f"[Interop] MarkerPollUs must be a number of microseconds, not {value!r}")

scripts/d4r_proton_env.sh

@@ -22,8 +22,8 @@ export D4R_ZLUDA_LIBCUDA="$D4R_ZLUDA_DIR/libcuda.so"
 # most 256-thread blocks, as CUDA itself allows 255 registers for them; the
 # default 1024 bound caps gfx11 waves at 96 VGPRs and makes them spill.
 export D4R_ZLUDA_IMPLICIT_MAX_BLOCK="${D4R_ZLUDA_IMPLICIT_MAX_BLOCK:-256}"
-# f16 tensor-core MMAs (DLSS's convolutions) run on gfx11 WMMA instead of a
-# dot-product fallback (patches/zluda/0003); 0 selects the fallback.
+# f16 tensor-core MMAs (DLSS's convolutions) run on gfx11/gfx12 WMMA instead of a
+# dot-product fallback (patches/zluda/0003 and 0007); 0 selects the fallback.
 export D4R_ZLUDA_WMMA="${D4R_ZLUDA_WMMA:-1}"
 # DLSS frames in flight. The presented upscaled image is this many game frames
 # old at most: 3 = highest frame rate (Townfall ~86 fps, age 2-3), 2 = age 2

scripts/package_release.sh

@@ -13,15 +13,24 @@
 #   D4R_ROCM_DIR     ROCm with clang, for the kernels                       default /opt/rocm
 #   D4R_ROCM_RUNTIME ROCm runtime bundled as d4r/rocm (scripts/fetch_rocm_runtime.sh)
 #                                                                           default ~/.cache/d4r-rocm-runtime
-#   D4R_GPU_ARCHS    GPU targets to build kernels for                       default gfx1101
+#   D4R_MAX_GLIBC     highest GLIBC ABI the Linux libraries may require     default 2.41
+#   D4R_GPU_ARCHS    GPU targets to build kernels for          default gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201
+#                    (RDNA4 targets also get a <target>-fp8 folder: the kernels for d4r.ini NativeFp8 = on)
 #   D4R_OPTISCALER_LICENSE  OptiScaler's LICENSE (GPL-3.0) text; default: the system's SPDX copy
 #   D4R_VKD3D_SRC    vkd3d-proton source checkout, for its license files     default ~/.cache/d4r-vkd3d-proton
 #   D4R_ZLUDA_SRC    ZLUDA source checkout, for its license files            default: D4R_ZLUDA_DIR's ../d4r-zluda-upstream
 #   SOURCE_DATE_EPOCH  timestamp given to every packaged file                default: the last commit's
+#   D4R_BUILD_INFO    optional build-toolchain/provenance text to include in d4r/source/
+#   D4R_PACKAGE_KERNEL_JOBS  independent target builds at once (default 1)
 # The zip also contains NVIDIA's files and the texture kernels built from NVIDIA's PTX; redistributing
 # those is up to whoever publishes it (they are not covered by d4r's license):
 #   D4R_BUNDLE_DLSS  nvngx_dlss.dll to include      D4R_BUNDLE_NGX  _nvngx.dll to include
-#   D4R_BUNDLE_TEX   directory with the texture-kernel code objects (kernels/build.sh tex), for gfx1101
+#   D4R_BUNDLE_TEX   directory with texture-kernel code objects (kernels/build.sh tex), one subdirectory
+#                    per target folder (gfx1101, gfx1201, gfx1201-fp8, ...), or a flat gfx1101 directory for
+#                    older builds
+#                    Accuracy texture sets go in accuracy/<target>/, built with D4R_PREFER_ACCURACY=1.
+#   D4R_ZLUDA_EMIT  d4r_emit from the patched ZLUDA build; required in full builds when an accuracy
+#                    texture set or L's unfolded texture variants are not supplied in D4R_BUNDLE_TEX.
 # D4R_BUNDLE_NVIDIA=0 leaves them out (d4r-VERSION-nonvidia.zip; users then add the two DLLs themselves).
 set -euo pipefail
 
@@ -37,7 +46,7 @@ ZLUDA="${D4R_ZLUDA_DIR:-$HOME/.cache/d4r-zluda-current}"
 VKD3D="${D4R_VKD3D_DIR:-$HOME/.cache/d4r-vkd3d-d4r}"
 ROCM="${D4R_ROCM_DIR:-/opt/rocm}"
 ROCM_RUNTIME="${D4R_ROCM_RUNTIME:-$HOME/.cache/d4r-rocm-runtime}"
-ARCHS="${D4R_GPU_ARCHS:-gfx1101}"
+ARCHS="${D4R_GPU_ARCHS:-gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201}"
 : "${D4R_OPTISCALER:?set D4R_OPTISCALER to the OptiScaler release archive or folder}"
 : "${D4R_DLSS_DLLS:?set D4R_DLSS_DLLS to the nvngx_dlss.dll files the kernel manifest accepts}"
 for f in "$ZLUDA/libnvcuda.so" "$VKD3D/d3d12.dll" "$VKD3D/d3d12core.dll" "$ROCM_RUNTIME/lib/libamdhip64.so.7"; do
@@ -98,6 +107,8 @@ cp "$ROOT/build/d4r_nvngx.dll" "$STAGE/d4r/nvngx.dll"
 cp "$ROOT/build/wine-nvcuda/x86_64-unix/nvcuda.dll.so" "$STAGE/d4r/nvcuda.dll"
 cp "$ZLUDA/libnvcuda.so" "$STAGE/d4r/zluda/libcuda.so"
 cp -r "$ROCM_RUNTIME/lib" "$STAGE/d4r/rocm/lib"
+"$ROOT/scripts/check_glibc_compat.sh" "${D4R_MAX_GLIBC:-2.41}" \
+  "$STAGE/d4r/nvcuda.dll" "$STAGE/d4r/zluda/libcuda.so" "$STAGE/d4r/rocm/lib/"*
 cp "$ROOT/packaging/d4r.ini" "$STAGE/d4r/d4r.ini"
 cp "$ROOT/packaging/d4r-check.sh" "$STAGE/d4r/d4r-check.sh"
 if [[ "$VARIANT" == full ]]; then
@@ -108,17 +119,99 @@ else
   printf 'Put NVIDIA'"'"'s NGX runtime, _nvngx.dll, in this folder (see D4R_README.txt).\r\n' > "$STAGE/d4r/ngx/README.txt"
 fi
 IFS=: read -r -a DLLS <<< "$D4R_DLSS_DLLS"
+# one folder per target; RDNA4 targets also get <target>-fp8 (native FP8 WMMA, d4r.ini NativeFp8), which the
+# bridge serves instead when that setting is on
+folders=()
 for arch in $ARCHS; do
-  D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" "$ROOT/kernels/build.sh" k "$STAGE/d4r/kernels/$arch" >/dev/null
-  D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" "$ROOT/kernels/build.sh" m "$STAGE/d4r/kernels/$arch" >/dev/null
-  rm -f "$STAGE/d4r/kernels/$arch"/*.resolution.txt  # empty LTO notes from clang's -save-temps
-  if [[ "$VARIANT" == full && "$arch" == gfx1101 ]]; then
-    for f in "$D4R_BUNDLE_TEX"/*.hsaco; do  # texture kernels only; the layers above are built from source
-      [[ -e "$STAGE/d4r/kernels/$arch/$(basename "$f")" ]] || cp "$f" "$STAGE/d4r/kernels/$arch/"
+  folders+=("$arch")
+  [[ "$arch" == gfx12* ]] && folders+=("$arch-fp8")
+done
+# Older texture bundles predate L. Complete them from the user's DLSS DLL rather
+# than silently shipping only M's folded variants under the new L support claim.
+ensure_l_textures() {
+  local dir="$1" accuracy="$2" missing=0 mv range v name
+  local names=(rrlite_dec0_4x4)
+  for mv in mvhi mvlo; do
+    for range in hdr ldr; do
+      names+=("rrlite_enc0_4x4_${mv}_${range}")
+      for v in 3_1 3_2; do names+=("rrlite_post_${v}_${mv}_${range}"); done
+    done
+  done
+  for name in "${names[@]}"; do [[ -f "$dir/$name.hsaco" ]] || missing=1; done
+  if [[ "$missing" == 1 ]]; then
+    : "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT or supply the unfolded L texture variants in D4R_BUNDLE_TEX}"
+    D4R_PREFER_ACCURACY="$accuracy" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" \
+      D4R_DLSS_DLL="$D4R_BUNDLE_DLSS" "$ROOT/kernels/build.sh" l "$dir" >/dev/null
+  fi
+}
+build_target() {
+  local folder="$1" arch fp8 tex_dir accurate accurate_tex f
+  arch="${folder%-fp8}"
+  fp8=0
+  [[ "$folder" == *-fp8 ]] && fp8=1
+  D4R_PREFER_ACCURACY=0 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" k "$STAGE/d4r/kernels/$folder" >/dev/null
+  D4R_PREFER_ACCURACY=0 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" m "$STAGE/d4r/kernels/$folder" >/dev/null
+  rm -f "$STAGE/d4r/kernels/$folder"/*.resolution.txt  # empty LTO notes from clang's -save-temps
+  if [[ "$VARIANT" == full ]]; then
+    tex_dir=""
+    if [[ -d "$D4R_BUNDLE_TEX/$folder" ]]; then
+      tex_dir="$D4R_BUNDLE_TEX/$folder"
+    elif [[ "$folder" == gfx1101 ]]; then
+      tex_dir="$D4R_BUNDLE_TEX"  # older, flat gfx1101 texture-kernel builds
+    fi
+    if [[ -n "$tex_dir" ]]; then
+      for f in "$tex_dir"/*.hsaco; do  # texture kernels only; the layers above are built from source
+        [[ -f "$f" ]] || continue
+        [[ -e "$STAGE/d4r/kernels/$folder/$(basename "$f")" ]] || cp "$f" "$STAGE/d4r/kernels/$folder/"
+      done
+    fi
+  fi
+  [[ "$VARIANT" != full ]] || ensure_l_textures "$STAGE/d4r/kernels/$folder" 0
+  python3 "$ROOT/kernels/tools/kernel_manifest.py" "$STAGE/d4r/kernels/$folder" "${DLLS[@]}"
+
+  # Every release target also gets conservative network and texture variants. Do not copy fast
+  # texture objects into this set: their compiler policy is fixed inside the binary.
+  accurate="$STAGE/d4r/kernels/accuracy/$folder"
+  D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" k "$accurate" >/dev/null
+  D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" m "$accurate" >/dev/null
+  if [[ "$VARIANT" == full ]]; then
+    accurate_tex="$D4R_BUNDLE_TEX/accuracy/$folder"
+    if [[ -f "$accurate_tex/d4r-accuracy.txt" && "$(cat "$accurate_tex/d4r-accuracy.txt")" == 1 ]]; then
+      for f in "$accurate_tex"/*.hsaco; do
+        [[ -f "$f" ]] || continue
+        [[ -e "$accurate/$(basename "$f")" ]] || cp "$f" "$accurate/"
+      done
+    else
+      : "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT or supply D4R_BUNDLE_TEX/accuracy/$folder}"
+      D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" \
+        D4R_DLSS_DLL="$D4R_BUNDLE_DLSS" "$ROOT/kernels/build.sh" tex "$accurate" >/dev/null
+    fi
+  fi
+  if [[ "$VARIANT" == full ]]; then
+    ensure_l_textures "$accurate" 1
+    for f in "$STAGE/d4r/kernels/$folder"/*.hsaco; do
+      [[ -f "$accurate/$(basename "$f")" ]] || {
+        echo "missing accuracy variant of $(basename "$f") for $folder" >&2; exit 2;
+      }
     done
   fi
-  python3 "$ROOT/kernels/tools/kernel_manifest.py" "$STAGE/d4r/kernels/$arch" "${DLLS[@]}"
+  rm -f "$accurate"/*.resolution.txt
+  python3 "$ROOT/kernels/tools/kernel_manifest.py" "$accurate" "${DLLS[@]}"
+}
+kernel_jobs="${D4R_PACKAGE_KERNEL_JOBS:-1}"
+[[ "$kernel_jobs" =~ ^[1-9][0-9]*$ ]] || { echo "D4R_PACKAGE_KERNEL_JOBS must be positive" >&2; exit 2; }
+pending=()
+kernel_failed=0
+for folder in "${folders[@]}"; do
+  build_target "$folder" &
+  pending+=("$!")
+  if (( ${#pending[@]} >= kernel_jobs )); then
+    wait "${pending[0]}" || kernel_failed=1
+    pending=("${pending[@]:1}")
+  fi
 done
+for pid in "${pending[@]}"; do wait "$pid" || kernel_failed=1; done
+[[ "$kernel_failed" == 0 ]] || { echo "native target build failed; no release ZIP created" >&2; exit 1; }
 
 # licenses and sources
 ZLUDA_SRC="${D4R_ZLUDA_SRC:-$(dirname "$ZLUDA")/d4r-zluda-upstream}"
@@ -143,6 +236,9 @@ file_version() { [[ -f "$1" ]] && strings -el "$1" | grep -A1 '^FileVersion$' |
 sed -e "s/@VERSION@/$VERSION/g" -e "s/@ZLUDA_COMMIT@/$ZLUDA_COMMIT/g" -e "s/@VKD3D_COMMIT@/$VKD3D_COMMIT/g" \
   -e "s/@DLSS_VERSION@/$(file_version "$STAGE/d4r/nvngx_dlss.dll")/g" -e "s/@NGX_VERSION@/$(file_version "$STAGE/d4r/ngx/_nvngx.dll")/g" \
   "$ROOT/packaging/SOURCES.txt" | variant > "$STAGE/d4r/source/SOURCES.txt"
+if [[ -n "${D4R_BUILD_INFO:-}" ]]; then
+  cp "$D4R_BUILD_INFO" "$STAGE/d4r/source/BUILD_INFO.txt"
+fi
 sed -e "s/@VERSION@/$VERSION/g" "$ROOT/packaging/D4R_README.txt" | variant | sed 's/$/\r/' > "$STAGE/D4R_README.txt"
 
 # One timestamp for every file (SOURCE_DATE_EPOCH, default the last commit): ZLUDA's kernel cache is keyed on

scripts/windows/benchmark-native-replay.ps1

@@ -0,0 +1,17 @@
+[CmdletBinding()]
+param([Parameter(Mandatory=$true)][string]$Module,
+    [Parameter(Mandatory=$true)][string]$Candidate,
+    [Parameter(Mandatory=$true)][string]$FixtureDirectory,
+    [string]$HipRoot, [string]$Probe, [string]$OutputDirectory,
+    [ValidateRange(1,10000)][int]$Pairs=32,
+    [ValidateRange(1,256)][int[]]$Batches=@(1,64),
+    [ValidateSet('events','dispatch')][string[]]$Timings=@('events','dispatch'))
+$ErrorActionPreference='Stop'
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+if (!$HipRoot) { $HipRoot=Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' }
+if (!$Probe) { $Probe=Join-Path $repo 'build/windows-rdna4-therock/d4r_native_replay_probe.exe' }
+if (!$OutputDirectory) { $OutputDirectory=Join-Path $repo ('test-results/native-benchmark-'+(Get-Date -Format 'yyyyMMdd-HHmmss-fff')) }
+& python.exe (Join-Path $repo 'tools/windows/benchmark_native_replay.py') --probe $Probe --hip-root $HipRoot `
+    --module $Module --candidate $Candidate --fixture-dir $FixtureDirectory --output-dir $OutputDirectory `
+    --pairs $Pairs --batches @Batches --timings @Timings
+if ($LASTEXITCODE) { throw "Native benchmark failed. Logs: $OutputDirectory" }

scripts/windows/build-gpu-telemetry.ps1

@@ -0,0 +1,28 @@
+[CmdletBinding()]
+param([string]$AdlxRoot, [string]$BuildDirectory, [string]$Compiler)
+$ErrorActionPreference='Stop'
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+$commit='32b5a740d42295c5dfe9026b9f52683da0f3af91'
+if (!$AdlxRoot) { $AdlxRoot=Join-Path $repo 'external/ADLX' }
+if (!$BuildDirectory) { $BuildDirectory=Join-Path $repo 'build/gpu-telemetry' }
+if (!$Compiler) { $Compiler=Join-Path $repo '.tools/llvm-mingw-20260922-ucrt-x86_64/bin/x86_64-w64-mingw32-clang.exe' }
+if (!(Test-Path -LiteralPath $AdlxRoot)) {
+    & git clone --no-checkout https://github.com/GPUOpen-LibrariesAndSDKs/ADLX.git $AdlxRoot
+    if ($LASTEXITCODE) { throw 'ADLX SDK download failed.' }
+    & git -C $AdlxRoot checkout --detach $commit
+    if ($LASTEXITCODE) { throw 'ADLX SDK checkout failed.' }
+}
+if ((& git -C $AdlxRoot rev-parse HEAD).Trim() -ne $commit -or (& git -C $AdlxRoot status --porcelain)) {
+    throw "Expected clean ADLX SDK $commit; use a separate AdlxRoot."
+}
+if (!(Test-Path -LiteralPath $Compiler)) { throw 'Set Compiler to an x64 LLVM-MinGW clang.exe.' }
+New-Item -ItemType Directory -Force $BuildDirectory | Out-Null
+$exe=Join-Path $BuildDirectory 'd4r_gpu_telemetry.exe'
+& $Compiler -std=c11 -O2 -Wall -Wextra -Wno-unknown-pragmas -DWIN32_LEAN_AND_MEAN -D_M_AMD64 `
+    -isystem "$AdlxRoot/SDK/Include" (Join-Path $repo 'tools/windows/gpu_telemetry.c') -o $exe
+if ($LASTEXITCODE) { throw "GPU telemetry compile failed: $LASTEXITCODE" }
+@{sdkCommit=$commit; sdkLicense=(Join-Path $AdlxRoot 'ADLX SDK License Agreement.pdf');
+    executable=$exe; compiler=$Compiler; sha256=(Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash} |
+    ConvertTo-Json | Set-Content -LiteralPath (Join-Path $BuildDirectory 'build-info.json') -Encoding UTF8
+Write-Host "Built local read-only telemetry: $exe"
+# The SDK, driver DLL and SDK-dependent binary are not added to public packages.

scripts/windows/build-native-k-experiment.ps1

@@ -0,0 +1,68 @@
+[CmdletBinding()]
+param([ValidateSet('enc0','enc1','enc2','enc3','enc4','dec5','dec4','dec3','dec2','dec1','dec0')]
+    [string]$Layer='enc1', [ValidateSet('source','cu','wgp')][string]$Mode='source',
+    [ValidateSet('baseline','load','store','both')][string]$NonTemporal='baseline',
+    [ValidateSet(0,1,2,4)][int]$PosTiles=0, [switch]$ShareValues, [switch]$PrivateValues,
+    [switch]$PackedAccumulator,
+    [string]$HipRoot, [string]$OutputDirectory)
+$ErrorActionPreference='Stop'
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+if (!$HipRoot) { $HipRoot=Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' }
+if ($ShareValues -and $PrivateValues) { throw 'Choose shared or private V storage.' }
+if (!$OutputDirectory) { $OutputDirectory=Join-Path $repo "build/native-k-experiment-$Layer-$Mode-$NonTemporal-pos$PosTiles-share$([int][bool]$ShareValues)-private$([int][bool]$PrivateValues)-packed$([int][bool]$PackedAccumulator)" }
+$OutputDirectory=[IO.Path]::GetFullPath($OutputDirectory)
+foreach ($baseline in @('build/native-k-gfx1201','build/windows-rdna4-therock','dist/windows-rdna4-command-list')) {
+    if ($OutputDirectory.StartsWith([IO.Path]::GetFullPath((Join-Path $repo $baseline)), [StringComparison]::OrdinalIgnoreCase)) {
+        throw 'Keep experimental objects outside the validated baseline/package.'
+    }
+}
+$compiler=Join-Path $HipRoot 'lib/llvm/bin/clang++.exe'
+if (!(Test-Path -LiteralPath $compiler)) { $compiler=Join-Path $HipRoot 'bin/clang++.exe' }
+$name="dltss_pwin_${Layer}_layer"
+$source=Join-Path $repo "kernels/k/$name.hip"
+$flags=@()
+if ((Get-Content -LiteralPath $source -Raw) -match '// d4r-build-flags: ([^\r\n]+)') { $flags=$Matches[1].Trim().Split(' ') }
+if ($Mode -ne 'source') {
+    $flags=@($flags | Where-Object { $_ -ne '-mcumode' -and $_ -ne '-mno-cumode' })
+    $flags+=if ($Mode -eq 'cu') { '-mcumode' } else { '-mno-cumode' }
+}
+if ($NonTemporal -in @('load','both')) { $flags+='-DPWIN_NT_LOAD' }
+if ($NonTemporal -in @('store','both')) { $flags+='-DPWIN_NT_STORE' }
+if ($PosTiles) {
+    if ($Layer -notin @('enc0','enc1','dec0')) { throw 'Token tiling is audited for position-only layers only.' }
+    $flags+=if ($Layer -eq 'enc1') { "-DD4R_K_ENC1_POS_MT=$PosTiles" } else { "-DD4R_K_POS_MT=$PosTiles" }
+}
+if ($ShareValues) {
+    if ($Layer -notin @('enc0','dec0') -and !($Layer -eq 'enc1' -and $PosTiles)) {
+        throw 'ShareValues requires a position-only template.'
+    }
+    $flags+='-DPOS_VSHARE'
+}
+if ($PackedAccumulator) { $flags+='-DD4R_K_PACKED_ACC' }
+if ($PrivateValues) { $flags+='-DD4R_K_PRIVATE_V' }
+New-Item -ItemType Directory -Force $OutputDirectory | Out-Null
+$object=Join-Path $OutputDirectory "$name.hsaco"
+$arguments=@('-x','hip','-std=c++17','--offload-arch=gfx1201','--offload-device-only',
+    '--no-gpu-bundle-output','-mno-wavefrontsize64','-nogpuinc','-nogpulib','-O3',
+    '-DD4R_DEVICE_ONLY_MINIMAL','-DD4R_K_FP16_BASELINE')+$flags+@($source,'-o',$object)
+$ErrorActionPreference='Continue'
+& $compiler @arguments > (Join-Path $OutputDirectory "$name.build.stdout.log") 2> (Join-Path $OutputDirectory "$name.build.stderr.log")
+$ErrorActionPreference='Stop'
+if ($LASTEXITCODE) { throw "Experimental K compile failed: $Layer" }
+& (Join-Path (Split-Path $compiler) 'llvm-objdump.exe') --disassemble --mcpu=gfx1201 $object `
+    > (Join-Path $OutputDirectory "$name.isa.txt")
+if ($LASTEXITCODE) { throw 'Native object ISA extraction failed.' }
+& (Join-Path (Split-Path $compiler) 'llvm-readobj.exe') --notes $object `
+    > (Join-Path $OutputDirectory "$name.metadata.txt")
+if ($LASTEXITCODE) { throw 'Native object metadata extraction failed.' }
+@{layer=$Layer; architecture='gfx1201'; experimental=$true; numericalValidation='pending';
+    mode=$Mode; nonTemporal=$NonTemporal; tokenTilesPerWave=$PosTiles; shareValues=[bool]$ShareValues;
+    privateValues=[bool]$PrivateValues;
+    packedAccumulator=[bool]$PackedAccumulator;
+    compiler=$compiler; arguments=$arguments;
+    sourceCommit=(git -C $repo rev-parse HEAD); workingTreeDirty=[bool](& git -C $repo status --porcelain);
+    compilerVersion=(& $compiler --version | Out-String).Trim();
+    sha256=(Get-FileHash -LiteralPath $object -Algorithm SHA256).Hash} |
+    ConvertTo-Json -Depth 4 | Set-Content -LiteralPath (Join-Path $OutputDirectory "$name.build-info.json") -Encoding UTF8
+Write-Host "Compiled gfx1201 K experiment: $Layer mode=$Mode nonTemporal=$NonTemporal posTiles=$PosTiles shareValues=$([bool]$ShareValues). Validate before installing."
+# No override manifest or game installation is produced by this script.

scripts/windows/build-native-k-f16-wmma.ps1

@@ -0,0 +1,40 @@
+[CmdletBinding()]
+param([ValidateSet('enc0','enc1','enc2','enc3','enc4','dec5','dec4','dec3','dec2','dec1','dec0')]
+    [string]$Layer='enc1', [string]$HipRoot, [string]$OutputDirectory)
+$ErrorActionPreference='Stop'
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+if (!$HipRoot) { $HipRoot=Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' }
+if (!$OutputDirectory) { $OutputDirectory=Join-Path $repo 'build/native-k-f16-wmma-experimental' }
+$OutputDirectory=[IO.Path]::GetFullPath($OutputDirectory)
+if ($OutputDirectory -eq [IO.Path]::GetFullPath((Join-Path $repo 'build/native-k-gfx1201'))) {
+    throw 'Keep the validated baseline separate from this experimental build.'
+}
+$compiler=Join-Path $HipRoot 'lib/llvm/bin/clang++.exe'
+if (!(Test-Path -LiteralPath $compiler)) { $compiler=Join-Path $HipRoot 'bin/clang++.exe' }
+if (!(Test-Path -LiteralPath $compiler)) { throw "HIP compiler missing under $HipRoot" }
+$name="dltss_pwin_${Layer}_layer"
+$source=Join-Path $repo "kernels/k/$name.hip"
+$flags=@()
+if ((Get-Content -LiteralPath $source -Raw) -match '// d4r-build-flags: ([^\r\n]+)') { $flags=$Matches[1].Trim().Split(' ') }
+New-Item -ItemType Directory -Force $OutputDirectory | Out-Null
+$object=Join-Path $OutputDirectory "$name.hsaco"
+$ErrorActionPreference='Continue'
+& $compiler -x hip -std=c++17 --offload-arch=gfx1201 --offload-device-only --no-gpu-bundle-output `
+    -mno-wavefrontsize64 -nogpuinc -nogpulib -O3 -DD4R_DEVICE_ONLY_MINIMAL `
+    -DD4R_K_FP16_BASELINE -DD4R_K_F16_WMMA @flags $source -o $object `
+    > (Join-Path $OutputDirectory "$name.build.stdout.log") 2> (Join-Path $OutputDirectory "$name.build.stderr.log")
+$ErrorActionPreference='Stop'
+if ($LASTEXITCODE) { throw "Experimental K FP16 WMMA compile failed: $Layer" }
+$isa=Join-Path $OutputDirectory "$name.isa.txt"
+& (Join-Path (Split-Path $compiler) 'llvm-objdump.exe') --disassemble --mcpu=gfx1201 $object > $isa
+if ($LASTEXITCODE -or !(Select-String -LiteralPath $isa -Pattern 'v_wmma_f16_16x16x16_f16')) {
+    throw 'Packed FP16 WMMA is absent from ISA; no alternative instruction is accepted.'
+}
+& (Join-Path (Split-Path $compiler) 'llvm-readobj.exe') --notes $object `
+    > (Join-Path $OutputDirectory "$name.metadata.txt")
+if ($LASTEXITCODE) { throw 'Native object metadata extraction failed.' }
+@{layer=$Layer; architecture='gfx1201'; experimental=$true; numericalValidation='pending';
+    compiler=$compiler; sha256=(Get-FileHash -LiteralPath $object -Algorithm SHA256).Hash} |
+    ConvertTo-Json | Set-Content -LiteralPath (Join-Path $OutputDirectory "$name.build-info.json") -Encoding UTF8
+Write-Host "Compiled experimental gfx1201 packed FP16 WMMA: $Layer. Validate before selecting another layer."
+# This script does not generate an override manifest or install into the game.

scripts/windows/build-native-m-fp8.ps1

@@ -0,0 +1,59 @@
+[CmdletBinding()]
+param([Parameter(Mandatory=$true)][string]$DlssDll, [string]$HipRoot, [string]$OutputDirectory, [switch]$HardwarePacking)
+$ErrorActionPreference='Stop'
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+if (!$HipRoot) { $HipRoot=Join-Path $repo '.tools/therock-10.2.0a20260929/_rocm_sdk_core' }
+if (!$OutputDirectory) { $OutputDirectory=Join-Path $repo $(if ($HardwarePacking) { 'build/native-m-fp8-hardware-pack' } else { 'build/native-m-fp8-experimental' }) }
+$OutputDirectory=[IO.Path]::GetFullPath($OutputDirectory)
+if ($OutputDirectory -eq [IO.Path]::GetFullPath((Join-Path $repo 'build/native-m-gfx1201'))) {
+    throw 'Keep the validated FP16 baseline separate from the experimental FP8 build.'
+}
+$DlssDll=(Get-Item -LiteralPath $DlssDll -ErrorAction Stop).FullName
+$compiler=Join-Path $HipRoot 'lib/llvm/bin/clang++.exe'
+if (!(Test-Path -LiteralPath $compiler)) { $compiler=Join-Path $HipRoot 'bin/clang++.exe' }
+if (!(Test-Path -LiteralPath $compiler)) { throw "HIP compiler missing under $HipRoot" }
+New-Item -ItemType Directory -Force $OutputDirectory | Out-Null
+$disassembler=Join-Path (Split-Path $compiler) 'llvm-objdump.exe'
+if (!(Test-Path -LiteralPath $disassembler)) { throw 'HIP llvm-objdump is required to confirm native FP8 instructions.' }
+foreach ($source in Get-ChildItem -LiteralPath (Join-Path $repo 'kernels/m') -Filter '*.hip' | Sort-Object Name) {
+    $flags=@()
+    if ((Get-Content -LiteralPath $source.FullName -Raw) -match '// d4r-build-flags: ([^\r\n]+)') { $flags=$Matches[1].Trim().Split(' ') }
+    if ($HardwarePacking) { $flags+='-DD4R_FP8_HW_PACK' }
+    $object=Join-Path $OutputDirectory ($source.BaseName+'.hsaco')
+    $ErrorActionPreference='Continue'
+    & $compiler -x hip -std=c++17 --offload-arch=gfx1201 --offload-device-only --no-gpu-bundle-output `
+        -mno-wavefrontsize64 -nogpuinc -nogpulib -O3 -DD4R_DEVICE_ONLY_MINIMAL `
+        -DSWIN_EXACT -DSWIN_EXACT_PV -DD4R_FP8_WMMA @flags $source.FullName `
+        -o $object > (Join-Path $OutputDirectory ($source.BaseName+'.build.stdout.log')) `
+        2> (Join-Path $OutputDirectory ($source.BaseName+'.build.stderr.log'))
+    $ErrorActionPreference='Stop'
+    if ($LASTEXITCODE) { throw "Experimental FP8 compile failed: $($source.Name)" }
+    $isa=Join-Path $OutputDirectory ($source.BaseName+'.isa.txt')
+    & $disassembler --disassemble --mcpu=gfx1201 $object > $isa
+    if ($LASTEXITCODE -or !(Select-String -LiteralPath $isa -Pattern 'v_wmma_f32_16x16x16_fp8_fp8')) {
+        throw "Native FP8 WMMA is absent from $($source.Name); no FP16 fallback is accepted."
+    }
+    if ($HardwarePacking -and !(Select-String -LiteralPath $isa -Pattern 'v_cvt_pk_fp8_f32')) { throw 'Native FP8 operand packing is absent from ISA.' }
+    Write-Host "Compiled gfx1201 FP8 WMMA: $($source.BaseName)"
+}
+if ($HardwarePacking) {
+    $probeDirectory=Join-Path $OutputDirectory 'packing-probe'
+    New-Item -ItemType Directory -Force $probeDirectory | Out-Null
+    & $compiler -x hip -std=c++17 --offload-arch=gfx1201 --offload-device-only --no-gpu-bundle-output `
+        -mno-wavefrontsize64 -nogpuinc -nogpulib -O3 -DD4R_DEVICE_ONLY_MINIMAL `
+        -DD4R_FP8_WMMA -DD4R_FP8_HW_PACK -DSWIN_EXACT -DSWIN_EXACT_PV `
+        (Join-Path $repo 'tools/windows/fp8_pack_probe.hip') -o (Join-Path $probeDirectory 'fp8_pack_probe.hsaco')
+    if ($LASTEXITCODE) { throw 'Native hardware packing diagnostic compile failed.' }
+}
+& python (Join-Path $repo 'kernels/tools/kernel_manifest.py') $OutputDirectory $DlssDll
+if ($LASTEXITCODE) { throw 'Experimental FP8 manifest generation failed.' }
+@{architecture='gfx1201'; family='M'; experimental=$true; nativeFp8=$true; accuracy=$true;
+    numerics='SWIN_EXACT; SWIN_EXACT_PV; FP8 WMMA for e4m3 GEMMs; exact FP16 attention';
+    validation='not validated; do not install before layer/full-network checks';
+    compiler=$compiler; compilerVersion=(& $compiler --version | Select-Object -First 1);
+    sourceCommit=(& git -C $repo rev-parse HEAD).Trim(); nativeFp8IsaVerified=$true; hardwarePacking=[bool]$HardwarePacking;
+    sourceWorkingTreeDirty=[bool](& git -C $repo status --porcelain);
+    dlssSha256=(Get-FileHash -LiteralPath $DlssDll -Algorithm SHA256).Hash;
+    modules=@(Get-ChildItem -LiteralPath $OutputDirectory -Filter '*.hsaco' | Get-FileHash -Algorithm SHA256)} |
+    ConvertTo-Json -Depth 5 | Set-Content -LiteralPath (Join-Path $OutputDirectory 'build-info.json') -Encoding UTF8
+Write-Host "Experimental native FP8 M objects: $OutputDirectory (baseline unchanged)"

scripts/windows/build-native-texture.ps1

@@ -0,0 +1,19 @@
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory=$true)][string]$DlssDll,
+    [string]$Kernel = 'hiluma_engine_output_depthinv_mvlo_hdr_max_v2_rel',
+    [string]$HipRoot = 'C:\Program Files\AMD\ROCm\7.2',
+    [string]$ZludaRoot,
+    [ValidateSet('cu','wgp')][string]$ShaderMode='cu',
+    [string]$GpuArch='gfx1201',
+    [string]$OutputDirectory
+)
+$ErrorActionPreference='Stop'
+. (Join-Path $PSScriptRoot 'gpu-target.ps1')
+[void](Get-D4RGpuTargetInfo $GpuArch)
+$repo=[IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../..'))
+if (!$ZludaRoot) { $ZludaRoot=Join-Path $repo 'dist/zluda-windows-final' }
+if (!$OutputDirectory) { $OutputDirectory=Join-Path $repo "build/private-textures-$GpuArch" }
+& python (Join-Path $repo 'tools/windows/build_native_texture.py') --dlss-dll $DlssDll --hip-root $HipRoot `
+    --zluda-root $ZludaRoot --kernel $Kernel --output-directory $OutputDirectory --shader-mode $ShaderMode --gpu-arch $GpuArch
+if ($LASTEXITCODE) { throw "Native texture build failed ($LASTEXITCODE); inspect the output work directory." }