wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
7117c38674c612105d8ce5f3fa6cac8a6792f354
d4r/tools/d4r_nvngx_shim.cpp
T
woutermans 7117c38674 Add CUDA NGX Ray Reconstruction support 
Run the signed CUDA-capable denoiser through feature 13 with independent D/E presets, pre-Init discovery, per-frame guide and camera snapshots, normalized integer guide staging, paired alpha output, and recorded-resource retirement.

Add the required ZLUDA/LLVM patch, behavioral configuration tests, GPU harness coverage, optional denoiser installation and packaging, and tested support limitations. Preserve upstream attribution and distinguish LLVM licensing from the ZLUDA portions of the patch.
2026-10-12 12:47:53 +02:00

8499 lines
402 KiB
C++
Raw Blame History

// Experimental native Vulkan NGX frontend: d4r_vulkan_frontend.inl.
// Modified in this fork for CUDA Ray Reconstruction support and validation (2026).
// d4r_nvngx.dll: an NGX core replacement that implements the D3D12 DLSS
// entry points on top of the official NGX core's CUDA path (which runs on AMD
// through the Wine nvcuda bridge and ZLUDA). OptiScaler loads it through its
// NvngxPath setting.
//
// A D3D12 EvaluateFeature call only records into the game's still-open command
// list, while DLSS runs on a separate CUDA context. Each evaluation therefore:
// 1. records copies of the inputs into readback buffers plus a GPU-written
// frame marker (WriteBufferImmediate) on the game's command list,
// 2. records a copy of the most recent finished DLSS result from an upload
// buffer into the output texture,
// 3. queues a CUDA job that waits for the marker, uploads the inputs into
// CUDA arrays, runs the official DLSS evaluation, and stores the result
// into an upload buffer for a later frame.
// The output is therefore one or more frames behind the game's other passes.
//
// Environment:
// D4R_NGX_CORE Windows path of the official _nvngx.dll (required)
// D4R_NGX_FEATURE_DIR directory holding nvngx_dlss.dll (optional, added to
// the feature search paths)
// D4R_SHIM_LOG log file path (default: d4r_nvngx.log next to the DLL)
// D4R_PROFILE=1 log per-frame CPU stages and default-stream GPU event time
// D4R_SHIM_OUTPUT_HASH=1 log an FNV-1a hash of every DLSS output as produced
// D4R_SHIM_OUTPUT_DUMP_DIR=<Windows path> save selected raw RGBA16F DLSS frames
// D4R_SHIM_OUTPUT_DUMP_START / _COUNT / _EVERY select frames (defaults 1/1/1)
// D4R_SHIM_INPUT_DUMP_DIR=<Windows path> save selected canonical input planes
// (use D4R_SHIM_VRAM_VERIFY=1 with VRAM interop)
// D4R_SHIM_CAPTURE_TRIGGER=<Windows path> select the next frame when this file appears
// (D4R_SHIM_CAPTURE_COUNT=N: that frame and the N-1 after it;
// D4R_SHIM_CAPTURE_MEMORY=1 buffers them in memory, written afterwards)
// D4R_SHIM_INPUT_STATE / D4R_SHIM_DEPTH_STATE / D4R_SHIM_OUTPUT_STATE
// D3D12 resource states the inputs/output are in when
// EvaluateFeature is called (defaults 0x40, 0x40, 0x8)
// D4R_SHIM_VRAM_INTEROP=1 keep inputs and output in VRAM (see "VRAM interop")
// D4R_SHIM_SPLIT_FRAME=1 with VRAM interop and the d4r vkd3d-proton patch,
// present each frame's own DLSS result (see "Split frames")
// D4R_SHIM_INPLACE=1 experimental, with VKD3D_D4R_LINEAR_TEXTURES=1 in the patched vkd3d-proton: the output
// kernel writes the game's output texture itself (no game-side output copy).
// D4R_SHIM_OUTPUT_PACKED=1 lets M's output kernel store R10G10B10A2 texels itself.
// (the DLSS-Denoiser library nvngx_dlssd.dll, which serves Ray Reconstruction, must sit
// next to this DLL: that is where the NGX core looks for it. It needs
// a build exporting NVSDK_NGX_CUDA_* (3.10.7); without it Ray
// Reconstruction reports itself unavailable and Super Sampling is
// unaffected.)
// D4R_RR_ENABLE=1/0 Ask for Ray Reconstruction / forbid it. Unset (auto) offers it
// exactly when the denoiser loaded and initialised.
// D4R_RR_PRESET Force the Ray Reconstruction render preset
// (NVSDK_NGX_RayReconstruction_Hint_Render_Preset: 4 = D, 5 = E).
// Independent of D4R_DLSS_PRESET, which only affects Super Sampling.
#define WIDL_EXPLICIT_AGGREGATE_RETURNS
#include <windows.h>
#include <d3d12.h>
#include <vulkan/vulkan_core.h>
#include "d4r_motion_dilation.h"
#include <algorithm>
#include <bit>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <cstdarg>
#include <cstdint>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <deque>
#include <functional>
#include <list>
#include <future>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include "d4r_event_wait.h"
#include "d4r_frame_completion.h"
#include "d4r_win32_wait.h"
#include "d4r_vkd3d_interop.h"
#include "../engine/game.h"
#include "../engine/game_m.h"
#include "../engine/vk_dispatch.h"
// --- NGX types (declared locally: the core's exports differ from the SDK) --
using NgxResult = unsigned int;
constexpr NgxResult NGX_SUCCESS = 0x1;
constexpr NgxResult NGX_FAIL = 0xBAD00000;
constexpr NgxResult NGX_FAIL_FEATURE_NOT_SUPPORTED = NGX_FAIL | 1;
constexpr NgxResult NGX_FAIL_PLATFORM_ERROR = NGX_FAIL | 2;
constexpr NgxResult NGX_FAIL_INVALID_PARAMETER = NGX_FAIL | 5;
constexpr NgxResult NGX_FAIL_NOT_INITIALIZED = NGX_FAIL | 7;
constexpr NgxResult NGX_FAIL_UNSUPPORTED_FORMAT = NGX_FAIL | 14;
constexpr unsigned int NGX_FEATURE_SUPER_SAMPLING = 1;
// nvngx_dlssd.dll (DLSS-Denoiser / Ray Reconstruction), NVSDK_NGX_Feature.
constexpr unsigned int NGX_FEATURE_RAY_RECONSTRUCTION = 13;
struct NgxHandle
{
unsigned int Id;
};
struct NgxPathListInfo
{
const wchar_t* const* Path;
unsigned int Length;
};
struct NgxLoggingInfo
{
void* LoggingCallback;
int MinimumLoggingLevel;
bool DisableOtherLoggingSinks;
};
struct NgxFeatureCommonInfo
{
NgxPathListInfo PathListInfo;
void* InternalData;
NgxLoggingInfo LoggingInfo;
};
struct NgxFeatureRequirement
{
unsigned int FeatureSupported;
unsigned int MinHWArchitecture;
char MinOSVersion[255];
};
// MSVC-ABI parameter accessors (d4r_ngx_param_msvc.cpp, built with clang-cl).
extern "C"
{
void d4r_ngx_set_ull(void* parameters, const char* name, unsigned long long value);
void d4r_ngx_set_float(void* parameters, const char* name, float value);
void d4r_ngx_set_uint(void* parameters, const char* name, unsigned int value);
void d4r_ngx_set_int(void* parameters, const char* name, int value);
void d4r_ngx_set_void(void* parameters, const char* name, void* value);
NgxResult d4r_ngx_get_ull(void* parameters, const char* name, unsigned long long* value);
NgxResult d4r_ngx_get_float(void* parameters, const char* name, float* value);
NgxResult d4r_ngx_get_uint(void* parameters, const char* name, unsigned int* value);
NgxResult d4r_ngx_get_int(void* parameters, const char* name, int* value);
NgxResult d4r_ngx_get_void(void* parameters, const char* name, void** value);
NgxResult d4r_ngx_get_d3d12_resource(void* parameters, const char* name, ID3D12Resource** value);
void* d4r_ngx_parameters_create();
void d4r_ngx_parameters_destroy(void* parameters);
}
// --- CUDA driver API (through nvcuda.dll, the Wine bridge to ZLUDA) --------
using CudaDevicePtr = unsigned long long;
using CudaArray = void*;
using CudaObject = unsigned long long;
using CudaEvent = void*;
struct CudaArrayDescriptor
{
size_t Width;
size_t Height;
uint32_t Format;
uint32_t NumChannels;
};
struct CudaArray3DDescriptor
{
size_t Width;
size_t Height;
size_t Depth;
uint32_t Format;
uint32_t NumChannels;
uint32_t Flags;
};
struct CudaMemcpy2D
{
size_t srcXInBytes;
size_t srcY;
uint32_t srcMemoryType;
uint32_t srcAlignment;
const void* srcHost;
CudaDevicePtr srcDevice;
CudaArray srcArray;
size_t srcPitch;
size_t dstXInBytes;
size_t dstY;
uint32_t dstMemoryType;
uint32_t dstAlignment;
void* dstHost;
CudaDevicePtr dstDevice;
CudaArray dstArray;
size_t dstPitch;
size_t WidthInBytes;
size_t Height;
};
static_assert(sizeof(CudaMemcpy2D) == 128);
struct CudaResourceDesc
{
uint32_t resType;
uint32_t alignment;
union
{
struct
{
CudaArray hArray;
} array;
int reserved[32];
} res;
uint32_t flags;
uint32_t reserved;
};
static_assert(sizeof(CudaResourceDesc) == 144);
struct CudaTextureDesc
{
uint32_t addressMode[3];
uint32_t filterMode;
uint32_t flags;
uint32_t maxAnisotropy;
uint32_t mipmapFilterMode;
float mipmapLevelBias;
float minMipmapLevelClamp;
float maxMipmapLevelClamp;
float borderColor[4];
int32_t reserved[12];
};
static_assert(sizeof(CudaTextureDesc) == 104);
constexpr uint32_t CUDA_FORMAT_HALF = 0x10;
constexpr uint32_t CUDA_FORMAT_FLOAT = 0x20;
constexpr uint32_t CUDA_MEMORY_HOST = 1;
constexpr uint32_t CUDA_MEMORY_DEVICE = 2;
constexpr uint32_t CUDA_MEMORY_ARRAY = 3;
struct CudaApi
{
int(WINAPI* memAlloc)(CudaDevicePtr*, size_t) = nullptr;
int(WINAPI* memFree)(CudaDevicePtr) = nullptr;
int(WINAPI* arrayCreate)(CudaArray*, const CudaArrayDescriptor*) = nullptr;
int(WINAPI* array3DCreate)(CudaArray*, const CudaArray3DDescriptor*) = nullptr;
int(WINAPI* arrayDestroy)(CudaArray) = nullptr;
int(WINAPI* memcpy2D)(const CudaMemcpy2D*) = nullptr;
int(WINAPI* texObjectCreate)(CudaObject*, const CudaResourceDesc*, const CudaTextureDesc*, const void*) = nullptr;
int(WINAPI* texObjectDestroy)(CudaObject) = nullptr;
int(WINAPI* surfObjectCreate)(CudaObject*, const CudaResourceDesc*) = nullptr;
int(WINAPI* surfObjectDestroy)(CudaObject) = nullptr;
int(WINAPI* ctxSynchronize)() = nullptr;
int(WINAPI* ctxGetDevice)(int*) = nullptr;
int(WINAPI* eventCreate)(CudaEvent*, unsigned int) = nullptr;
int(WINAPI* eventRecord)(CudaEvent, void*) = nullptr;
int(WINAPI* eventQuery)(CudaEvent) = nullptr;
int(WINAPI* eventSynchronize)(CudaEvent) = nullptr;
int(WINAPI* eventElapsedTime)(float*, CudaEvent, CudaEvent) = nullptr;
int(WINAPI* eventDestroy)(CudaEvent) = nullptr;
// Optional: overlapped staging (see async_copies()).
int(WINAPI* memAllocHost)(void**, size_t) = nullptr;
int(WINAPI* memFreeHost)(void*) = nullptr;
int(WINAPI* memcpyHtoDAsync)(CudaDevicePtr, const void*, size_t, void*) = nullptr;
int(WINAPI* memcpyDtoHAsync)(void*, CudaDevicePtr, size_t, void*) = nullptr;
int(WINAPI* streamCreate)(void**, unsigned int) = nullptr;
int(WINAPI* streamSynchronize)(void*) = nullptr;
// Optional (d4r nvcuda bridge): device <-> array copies queued on the null stream.
int(WINAPI* memcpy2DAsync)(const CudaMemcpy2D*, void*) = nullptr;
// Optional (d4r nvcuda bridge): null-stream wait for a device u32, and a write from another stream.
int(WINAPI* streamWaitValue32)(CudaDevicePtr, uint32_t) = nullptr;
int(WINAPI* writeValue32)(CudaDevicePtr, uint32_t) = nullptr;
// Optional (d4r nvcuda bridge): surfaces of an array store to linear memory instead (d4r native kernels).
int(WINAPI* setArrayRedirect)(CudaArray, CudaDevicePtr, uint32_t) = nullptr;
int(WINAPI* outputKernelNative)() = nullptr;
// Optional (d4r nvcuda bridge): report a pitch-linear texture to NGX as an array of that size/format.
int(WINAPI* registerLinearTexture)(CudaObject, size_t, size_t, uint32_t, uint32_t) = nullptr;
int(WINAPI* dilateMotion)(const D4rMotionDilationParams*) = nullptr;
};
// --- official NGX core CUDA API --------------------------------------------
struct CoreApi
{
NgxResult (*init)(unsigned long long, const wchar_t*, const NgxFeatureCommonInfo*, unsigned int) = nullptr;
NgxResult (*initProjectId)(const char*, int, const char*, const wchar_t*, unsigned int,
const NgxFeatureCommonInfo*) = nullptr;
NgxResult (*shutdown)() = nullptr;
NgxResult (*getParameters)(void**) = nullptr;
NgxResult (*allocateParameters)(void**) = nullptr;
NgxResult (*getCapabilityParameters)(void**) = nullptr;
NgxResult (*destroyParameters)(void*) = nullptr;
NgxResult (*createFeature)(unsigned int, void*, NgxHandle**) = nullptr;
NgxResult (*releaseFeature)(NgxHandle*) = nullptr;
NgxResult (*evaluateFeature)(const NgxHandle*, void*, void*) = nullptr;
// The core's CUDA entry point takes a CUdevice ordinal, not an IDXGIAdapter* (its
// disassembly rejects a negative value in the first argument, which is the ordinal
// check). The D3D12 export this file implements takes an adapter and must translate.
NgxResult (*getFeatureRequirements)(int, const void*, NgxFeatureRequirement*) = nullptr;
};
// --- DLSS-Denoiser (Ray Reconstruction) ----------------------------------------
//
// nvngx_dlssd.dll carries the Ray Reconstruction models, and the NGX core loads and
// dispatches it itself: the core is the only thing that knows the denoiser's module name,
// its kernel map and its lifetime. So both features are created, evaluated and released
// through g.ngx, with feature id 13 for Ray Reconstruction, and this file's job is only to
// answer honestly whether that route exists.
//
// Existence is decided by loading the denoiser and looking for its CUDA entry points, the
// only reliable external evidence - builds genuinely differ. The 3.10.7 build exports the
// whole NVSDK_NGX_CUDA_* set; the DLAA-only build exports D3D11/D3D12/Vulkan/DirectSR and
// no CUDA entry points, and cannot serve Ray Reconstruction however it is loaded.
//
// The pointers here are resolved only as that evidence and are never called. In particular
// NVSDK_NGX_CUDA_Init in the 310.7 build is `mov eax, 0xBAD00001; ret` (RVA 0x22b80), a stub
// that only ever reports "not supported"; the core's own init is what brings the denoiser up.
// A single yes/no, not a table of entry points: nothing here is ever called. The denoiser is
// loaded by the NGX core, which is the only thing that may drive it.
struct DenoiserProbe
{
bool cudaCapable = false;
};
// --- logging -----------------------------------------------------------------
static std::mutex g_logMutex;
static FILE* g_log = nullptr;
static HMODULE g_selfModule = nullptr;
// --- portable install ----------------------------------------------------------
// The drag-in release keeps this DLL (as nvngx.dll, explicitly selected by OptiScaler's NvngxPath),
// the nvcuda bridge, ZLUDA and the native kernels in an d4r folder next to the game's executable,
// with that game's settings in d4r\d4r.ini. The file's settings become the environment variables the
// developer launcher sets (config/d4r.ini.default documents them), and a variable that is already set,
// for example in the game's launch options, wins. Settings for the Linux side (ZLUDA, the bridge) are
// handed to the bridge right after it is loaded, before its first CUDA call. Without an d4r.ini next
// to the DLL nothing changes.
struct PortableInstall
{
bool active = false;
std::wstring dir; // the d4r folder
std::string unixDir; // the same folder as a Linux path
std::vector<std::pair<std::string, std::string>> unixEnv; // for the bridge's d4rSetEnv
std::vector<std::string> notes; // logged once the log is open
};
static PortableInstall g_portable;
static std::once_flag g_portableOnce;
static bool g_logFresh = false; // a portable install starts a new log for every launch
struct IniEntry
{
std::string section, key, value;
};
static std::string ascii_lower(std::string text)
{
for (char& c : text)
c = static_cast<char>(tolower(static_cast<unsigned char>(c)));
return text;
}
static std::string trim(const std::string& text)
{
const size_t first = text.find_first_not_of(" \t\r\n");
if (first == std::string::npos)
return std::string();
return text.substr(first, text.find_last_not_of(" \t\r\n") - first + 1);
}
// INI in the configparser dialect d4r_config.py reads: [Section], key = value, full-line comments
// with ; or #, inline comments after whitespace.
static std::vector<IniEntry> parse_ini(const std::string& text)
{
std::vector<IniEntry> entries;
std::string section;
size_t start = text.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0;
while (start < text.size())
{
size_t end = text.find('\n', start);
if (end == std::string::npos)
end = text.size();
const std::string line = trim(text.substr(start, end - start));
start = end + 1;
if (line.empty() || line[0] == ';' || line[0] == '#')
continue;
if (line[0] == '[')
{
section = ascii_lower(trim(line.substr(1, line.find(']') - 1)));
continue;
}
const size_t equals = line.find('=');
if (equals == std::string::npos)
continue;
std::string value = line.substr(equals + 1);
for (size_t i = 1; i < value.size(); ++i)
if ((value[i] == ';' || value[i] == '#') && (value[i - 1] == ' ' || value[i - 1] == '\t'))
{
value.resize(i);
break;
}
entries.push_back({section, trim(line.substr(0, equals)), trim(value)});
}
return entries;
}
// The last value for [section] key; "" when it is missing, empty or "auto".
static std::string ini_value(const std::vector<IniEntry>& ini, const char* section, const char* key)
{
std::string result;
for (const IniEntry& entry : ini)
if (entry.section == section && ascii_lower(entry.key) == ascii_lower(key))
result = entry.value;
return ascii_lower(result) == "auto" ? std::string() : result;
}
// 1 or 0 for a boolean setting, fallback when it is unset or not a boolean.
static int ini_flag(const std::vector<IniEntry>& ini, const char* section, const char* key, int fallback)
{
const std::string value = ascii_lower(ini_value(ini, section, key));
if (value.empty())
return fallback;
if (value == "1" || value == "true" || value == "yes" || value == "on")
return 1;
if (value == "0" || value == "false" || value == "no" || value == "off")
return 0;
g_portable.notes.push_back("d4r.ini: [" + std::string(section) + "] " + key + " must be true or false, not '" +
value + "'; using " + (fallback ? "true" : "false"));
return fallback;
}
static bool read_whole_file(const std::wstring& path, std::string& text)
{
HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (file == INVALID_HANDLE_VALUE)
return false;
LARGE_INTEGER size = {};
bool ok = GetFileSizeEx(file, &size) && size.QuadPart < (1 << 20);
if (ok)
{
text.resize(static_cast<size_t>(size.QuadPart));
DWORD read = 0;
ok = text.empty() || (ReadFile(file, text.data(), static_cast<DWORD>(text.size()), &read, nullptr) &&
read == text.size());
}
CloseHandle(file);
return ok;
}
static bool file_exists(const std::wstring& path)
{
const DWORD attributes = GetFileAttributesW(path.c_str());
return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
}
// Wine's kernel32 maps a Windows path to the Linux one.
static std::string unix_path(const std::wstring& path)
{
using UnixFileNameFn = char*(CDECL*)(const wchar_t*);
const auto function = reinterpret_cast<UnixFileNameFn>(
reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "wine_get_unix_file_name")));
char* result = function != nullptr ? function(path.c_str()) : nullptr;
if (result == nullptr)
return std::string();
std::string text(result);
HeapFree(GetProcessHeap(), 0, result);
return text;
}
static std::wstring widen(const std::string& text)
{
std::wstring wide(text.size() + 1, L'\0');
const int length = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), -1, wide.data(), static_cast<int>(wide.size()));
wide.resize(length > 0 ? static_cast<size_t>(length - 1) : 0);
return wide;
}
// A Windows path from d4r.ini: absolute (C:\..., \\...) or relative to the d4r folder.
static std::wstring portable_path(const std::string& value)
{
const std::wstring path = widen(value);
if ((path.size() > 2 && path[1] == L':') || (path.size() > 1 && path[0] == L'\\' && path[1] == L'\\'))
return path;
return g_portable.dir + L"\\" + path;
}
static void portable_set(const char* name, const std::string& value)
{
if (GetEnvironmentVariableA(name, nullptr, 0) == 0)
SetEnvironmentVariableA(name, value.c_str());
}
static void portable_set(const wchar_t* name, const std::wstring& value)
{
if (GetEnvironmentVariableW(name, nullptr, 0) == 0)
SetEnvironmentVariableW(name, value.c_str());
}
static void portable_set_unix(const char* name, const std::string& value)
{
g_portable.unixEnv.emplace_back(name, value);
}
// [DLSS] Model: a preset letter, a friendly name or a render-preset number; "" leaves the choice to the game.
static std::string preset_number(const std::string& model)
{
std::string name;
for (char c : model)
name += static_cast<char>(toupper(static_cast<unsigned char>(c)));
if (name.empty() || name == "GAME" || name == "DEFAULT")
return std::string();
if (name.find_first_not_of("0123456789") == std::string::npos)
return name;
static const std::pair<const char*, const char*> aliases[] = {
{"CNN", "E"}, {"DLSS3", "E"}, {"DLSS4", "K"}, {"DLSS4.5", "M"}, {"TRANSFORMER", "K"}};
for (const auto& [alias, letter] : aliases)
if (name == alias)
name = letter;
static const std::pair<const char*, int> letters[] = {{"A", 1}, {"B", 2}, {"C", 3}, {"D", 4}, {"E", 5}, {"F", 6},
{"G", 7}, {"J", 10}, {"K", 11}, {"L", 12}, {"M", 13}};
for (const auto& [letter, number] : letters)
if (name == letter)
return std::to_string(number);
g_portable.notes.push_back("d4r.ini: [DLSS] Model '" + model + "' is not a preset (use K, E, L, M or a number); "
"leaving the preset to the game");
return std::string();
}
static void load_portable_config()
{
wchar_t module[MAX_PATH] = {};
const DWORD length = GetModuleFileNameW(g_selfModule, module, MAX_PATH);
if (length == 0 || length >= MAX_PATH)
return;
std::wstring dir(module, length);
dir.resize(dir.find_last_of(L"\\/"));
g_portable.dir = dir; // also without d4r.ini: the default log goes here
std::string text;
if (!read_whole_file(dir + L"\\d4r.ini", text))
return; // developer layout: the launcher's environment has everything
const std::vector<IniEntry> ini = parse_ini(text);
g_portable.active = true;
g_portable.unixDir = unix_path(dir);
if (g_portable.unixDir.empty())
g_portable.notes.push_back("cannot map the d4r folder to a Linux path (not running under Wine?)");
// Files of the install; NGX looks for nvngx_dlss.dll next to this DLL.
portable_set(L"D4R_NVCUDA_BRIDGE", dir + L"\\nvcuda.dll");
const std::string core = ini_value(ini, "paths", "NgxCore");
portable_set(L"D4R_NGX_CORE", core.empty() ? dir + L"\\ngx\\_nvngx.dll" : portable_path(core));
portable_set(L"D4R_NGX_FEATURE_DIR", dir);
const std::string log = ini_value(ini, "debug", "Log");
g_logFresh = log.empty();
portable_set(L"D4R_SHIM_LOG", log.empty() ? dir + L"\\d4r_nvngx.log" : portable_path(log));
portable_set("D4R_CUDA_CAPTURE", "0"); // the bridge would otherwise save every DLSS module it loads
const std::string zluda = ini_value(ini, "paths", "ZludaDir");
portable_set_unix("D4R_ZLUDA_LIBCUDA", (zluda.empty() ? g_portable.unixDir + "/zluda" : zluda) + "/libcuda.so");
// The release's bundled ROCm runtime unless d4r.ini names another
const std::string rocm = ini_value(ini, "paths", "RocmDir");
if (!rocm.empty())
portable_set_unix("D4R_ROCM_DIR", rocm);
else if (!g_portable.unixDir.empty() && GetFileAttributesW((dir + L"\\rocm\\lib").c_str()) != INVALID_FILE_ATTRIBUTES)
portable_set_unix("D4R_ROCM_DIR", g_portable.unixDir + "/rocm");
const std::string cache = ini_value(ini, "paths", "CacheDir");
portable_set_unix("D4R_ZLUDA_CACHE_HOME", cache.empty() ? "~/.cache/d4r" : cache);
std::string model = ini_value(ini, "dlss", "Model");
if (ascii_lower(model) != "game")
model = preset_number(model.empty() ? "K" : model);
else
model.clear();
if (!model.empty())
portable_set("D4R_DLSS_PRESET", model);
// Ray Reconstruction is configured on its own section and drives its own feature, so
// nothing here reaches [DLSS] Model and nothing there reaches this.
//
// Enable: true -> D4R_RR_ENABLE=1, false -> 0, auto/unset -> the variable is left alone,
// which is the same thing as auto here: Ray Reconstruction is offered exactly when a
// CUDA-capable denoiser really loaded and initialised.
const std::string rrEnable = ascii_lower(ini_value(ini, "rayreconstruction", "Enable"));
if (rrEnable == "true" || rrEnable == "on" || rrEnable == "1")
portable_set("D4R_RR_ENABLE", "1");
else if (rrEnable == "false" || rrEnable == "off" || rrEnable == "0")
portable_set("D4R_RR_ENABLE", "0");
else if (!rrEnable.empty() && rrEnable != "auto")
g_portable.notes.push_back("d4r.ini: [RayReconstruction] Enable must be true, false or auto, not '" +
rrEnable + "'; treating it as auto");
// Model: auto/unset leaves the choice to the game; "default" is the SDK's Default preset
// (0); D (4) and E (5) are the two the SDK gives a meaning to. F and above are
// documented as reverting to default behaviour, so they are not offered. E is the model
// to pick when the depth-of-field guide is in use. These are the same values
// scripts/d4r_config.py accepts, so both ways of reading d4r.ini agree.
const std::string rrModel = ascii_lower(ini_value(ini, "rayreconstruction", "Model"));
if (!rrModel.empty() && rrModel != "auto" && rrModel != "game")
{
std::string upper = rrModel;
for (char& c : upper)
c = static_cast<char>(toupper(static_cast<unsigned char>(c)));
if (upper == "DEFAULT")
portable_set("D4R_RR_PRESET", "0");
else if (upper == "D")
portable_set("D4R_RR_PRESET", "4");
else if (upper == "E")
portable_set("D4R_RR_PRESET", "5");
else if (upper == "0" || upper == "4" || upper == "5")
portable_set("D4R_RR_PRESET", upper);
else
g_portable.notes.push_back("d4r.ini: [RayReconstruction] Model must be auto, default, D or E, not '" +
rrModel + "'; leaving the model to the game");
}
const std::string age = ini_value(ini, "latency", "FrameAge");
if (age.empty() || age == "0")
{
portable_set("D4R_SHIM_SPLIT_FRAME", "1");
portable_set("D4R_SHIM_MAX_IN_FLIGHT", "3");
}
else if (age == "1" || age == "2" || age == "3")
{
portable_set("D4R_SHIM_SPLIT_FRAME", "0");
portable_set("D4R_SHIM_MAX_IN_FLIGHT", age);
}
else
g_portable.notes.push_back("d4r.ini: [Latency] FrameAge must be 0-3, not '" + age + "'");
// Native kernels: d4r\kernels, each used only when DLSS's PTX for it matches (see the bridge).
const std::string native = ascii_lower(ini_value(ini, "kernels", "NativeKernels"));
// The bridge selects accuracy variants and enforces their compiler/sync policy before cuInit.
portable_set_unix("D4R_PREFER_ACCURACY", ini_flag(ini, "kernels", "PreferAccuracy", 0) ? "1" : "0");
portable_set_unix("D4R_NATIVE_SWIN_ENCODERS", ini_flag(ini, "kernels", "NativeSwinEncoders", 1) ? "1" : "0");
const bool nativeOn = native.empty() || native == "on" || native == "true" || native == "1" || native == "fast";
if (!nativeOn && native != "off" && native != "false" && native != "0")
g_portable.notes.push_back("d4r.ini: [Kernels] NativeKernels must be on or off, not '" + native + "'");
if (nativeOn && !g_portable.unixDir.empty())
portable_set_unix("D4R_ZLUDA_NATIVE_DIR", g_portable.unixDir + "/kernels");
portable_set_unix("D4R_ZLUDA_WMMA", ini_flag(ini, "kernels", "Wmma", 1) ? "1" : "0");
portable_set_unix("D4R_ZLUDA_WMMA_FP8", ini_flag(ini, "kernels", "Fp8Wmma", 1) ? "1" : "0");
// RDNA4's native FP8 WMMA; ZLUDA and the bridge ignore it on other GPUs
portable_set_unix("D4R_ZLUDA_WMMA_FP8_NATIVE", ini_flag(ini, "kernels", "NativeFp8", 1) ? "1" : "0");
portable_set_unix("D4R_ZLUDA_IGNORE_DENORMAL", ini_flag(ini, "kernels", "IgnoreDenormals", 1) ? "1" : "0");
const std::string maxBlock = ini_value(ini, "kernels", "ImplicitMaxBlock");
portable_set_unix("D4R_ZLUDA_IMPLICIT_MAX_BLOCK", maxBlock.empty() ? "256" : maxBlock);
static const struct
{
const char* key;
const char* variable;
int fallback;
} interop[] = {{"VramInterop", "D4R_SHIM_VRAM_INTEROP", 1}, {"InputSync", "D4R_SHIM_INPUT_SYNC", 1},
{"LinearInputs", "D4R_SHIM_LINEAR_INPUTS", 1},
{"EvalSync", "D4R_SHIM_EVAL_SYNC", 0}};
for (const auto& setting : interop)
portable_set(setting.variable, ini_flag(ini, "interop", setting.key, setting.fallback) ? "1" : "0");
portable_set_unix("D4R_ELIDE_NGX_SYNC", ini_flag(ini, "interop", "ElideNgxSync", 1) ? "1" : "0");
const std::string poll = ini_value(ini, "interop", "MarkerPollUs");
portable_set("D4R_SHIM_MARKER_POLL_US", poll.empty() ? "200" : poll);
// Direct output takes effect only while NGX runs the native output kernel (the bridge reports it); the
// release cannot ship that kernel, which is built from NVIDIA's PTX.
portable_set("D4R_SHIM_OUTPUT_DIRECT", ini_flag(ini, "interop", "DirectOutput", nativeOn) ? "1" : "0");
// [Engine]: d4r's own Vulkan inference for DLSS 4 (K); the model is looked up in d4r\engine\k
portable_set("D4R_ENGINE", ini_flag(ini, "engine", "Enabled", 0) ? "1" : "0");
if (const std::string engineModel = ini_value(ini, "engine", "ModelDir"); !engineModel.empty())
portable_set(L"D4R_ENGINE_MODEL_DIR", portable_path(engineModel));
// Network: preset M's network on the native HIP layers ("hip", the default when the model has them: faster) or
// on the engine's Vulkan shader ("vulkan")
if (const std::string network = ini_value(ini, "engine", "Network"); network == "hip" || network == "vulkan")
portable_set("D4R_ENGINE_NETWORK", network);
else if (!network.empty() && network != "auto")
g_portable.notes.push_back("[Engine] Network must be auto, hip or vulkan; using auto");
portable_set("D4R_SHIM_WATERMARK", ini_flag(ini, "dlss", "ShowWatermark", 0) ? "1" : "0");
const std::string dilation = ini_value(ini, "interop", "MotionVectorDilation");
if (dilation.empty() || dilation == "0" || dilation == "1" || dilation == "2")
portable_set("D4R_MOTION_DILATION", dilation.empty() ? "0" : dilation);
else
g_portable.notes.push_back("MotionVectorDilation must be 0, 1 or 2; using original motion vectors");
if (ini_flag(ini, "debug", "Profile", 0))
portable_set("D4R_PROFILE", "1");
if (const std::string level = ini_value(ini, "debug", "NgxLogLevel"); !level.empty())
portable_set("D4R_NGX_LOG_LEVEL", level);
// [Env]: any other variable, for both sides
for (const IniEntry& entry : ini)
if (entry.section == "env" && !entry.key.empty())
{
portable_set(entry.key.c_str(), entry.value);
portable_set_unix(entry.key.c_str(), entry.value);
}
if (!file_exists(dir + L"\\nvngx_dlss.dll"))
g_portable.notes.push_back("nvngx_dlss.dll is missing: copy NVIDIA's DLSS library (310.7 or 310.9 recommended) "
"into the d4r folder");
// Only a note: without the denoiser Super Sampling is unaffected and Ray Reconstruction
// simply reports itself unavailable, which is the same thing the capability answer says.
if (ini_flag(ini, "rayreconstruction", "Enable", 1) && !file_exists(dir + L"\\nvngx_dlssd.dll"))
g_portable.notes.push_back("nvngx_dlssd.dll is missing: copy NVIDIA's DLSS-Denoiser library (3.10.7, the "
"build that exports NVSDK_NGX_CUDA_*) into the d4r folder to get Ray "
"Reconstruction");
wchar_t corePath[MAX_PATH] = {};
const DWORD coreLength = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
if (coreLength > 0 && coreLength < MAX_PATH && !file_exists(corePath))
g_portable.notes.push_back("the NGX core is missing: copy NVIDIA's _nvngx.dll into d4r\\ngx");
}
static void ensure_portable_config()
{
std::call_once(g_portableOnce, load_portable_config);
}
static void log_open()
{
ensure_portable_config();
wchar_t path[MAX_PATH] = {};
DWORD length = GetEnvironmentVariableW(L"D4R_SHIM_LOG", path, MAX_PATH);
if (length > 0 && length < MAX_PATH)
g_log = _wfopen(path, g_logFresh ? L"w" : L"a");
else if (!g_portable.dir.empty()) // no log when the DLL's folder is unknown
g_log = _wfopen((g_portable.dir + L"\\d4r_nvngx.log").c_str(), g_logFresh ? L"w" : L"a");
}
static void logf(const char* format, ...)
{
std::lock_guard<std::mutex> lock(g_logMutex);
if (g_log == nullptr)
log_open();
if (g_log == nullptr)
return;
std::fprintf(g_log, "[%10.3f] ", GetTickCount64() / 1000.0);
va_list arguments;
va_start(arguments, format);
std::vfprintf(g_log, format, arguments);
va_end(arguments);
std::fputc('\n', g_log);
std::fflush(g_log);
}
// NGX core and feature diagnostics, routed into the shim log.
static void ngx_log_callback(const char* message, int level, unsigned int component)
{
if (message == nullptr)
return;
std::string line(message);
while (!line.empty() && (line.back() == '\n' || line.back() == '\r'))
line.pop_back();
logf("NGX[%d/%u] %s", level, component, line.c_str());
}
static unsigned int env_uint(const char* name, unsigned int fallback)
{
char value[64] = {};
DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
if (length == 0 || length >= sizeof(value) || value[0] < '0' || value[0] > '9')
return fallback; // also signs and text: strtoul turns "-1" into 4294967295
return static_cast<unsigned int>(std::strtoul(value, nullptr, 0));
}
static std::string env_string(const char* name)
{
char value[1024] = {};
const DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
return length > 0 && length < sizeof(value) ? std::string(value, length) : std::string();
}
static bool profile_enabled()
{
static const bool enabled = env_uint("D4R_PROFILE", 0) != 0;
return enabled;
}
using ProfileClock = std::chrono::steady_clock;
static double profile_ms(ProfileClock::time_point start, ProfileClock::time_point end)
{
return std::chrono::duration<double, std::milli>(end - start).count();
}
static double profile_thread_cpu_ms()
{
FILETIME created = {}, exited = {}, kernel = {}, user = {};
if (!GetThreadTimes(GetCurrentThread(), &created, &exited, &kernel, &user))
return -1.0;
const uint64_t kernelTicks = (static_cast<uint64_t>(kernel.dwHighDateTime) << 32) | kernel.dwLowDateTime;
const uint64_t userTicks = (static_cast<uint64_t>(user.dwHighDateTime) << 32) | user.dwLowDateTime;
return static_cast<double>(kernelTicks + userTicks) / 10000.0;
}
// --- CUDA worker thread --------------------------------------------------------
// Every NGX-core and CUDA call runs on this thread so the ZLUDA context that
// the nvcuda bridge creates on first use stays current.
class Worker
{
public:
void start()
{
// Detached: the process may exit without shutting NGX down.
std::thread([this] { run(); }).detach();
}
void post(std::function<void()> job)
{
{
std::lock_guard<std::mutex> lock(mutex_);
jobs_.push_back(std::move(job));
}
condition_.notify_one();
}
size_t pending()
{
std::lock_guard<std::mutex> lock(mutex_);
return jobs_.size();
}
template <typename Function> auto call(Function function) -> decltype(function())
{
using Result = decltype(function());
auto task = std::make_shared<std::packaged_task<Result()>>(std::move(function));
auto future = task->get_future();
post([task] { (*task)(); });
return future.get();
}
private:
void run()
{
for (;;)
{
std::function<void()> job;
{
std::unique_lock<std::mutex> lock(mutex_);
condition_.wait(lock, [this] { return !jobs_.empty(); });
job = std::move(jobs_.front());
jobs_.pop_front();
}
job();
}
}
std::mutex mutex_;
std::condition_variable condition_;
std::deque<std::function<void()>> jobs_;
};
// --- half/float and packed format conversions --------------------------------
static uint16_t float_to_half(float value)
{
uint32_t bits;
std::memcpy(&bits, &value, sizeof(bits));
const uint32_t sign = (bits >> 16) & 0x8000u;
const uint32_t exponent = (bits >> 23) & 0xffu;
uint32_t mantissa = bits & 0x7fffffu;
if (exponent == 0xff)
return static_cast<uint16_t>(sign | 0x7c00u | (mantissa ? 0x200u : 0u));
int halfExponent = static_cast<int>(exponent) - 112;
if (halfExponent >= 31)
return static_cast<uint16_t>(sign | 0x7c00u);
uint32_t shift = 13;
if (halfExponent <= 0)
{
if (halfExponent < -10)
return static_cast<uint16_t>(sign);
mantissa |= 0x800000u;
shift = static_cast<uint32_t>(14 - halfExponent);
halfExponent = 0;
}
const uint32_t halfMantissa = mantissa >> shift;
const uint32_t remainder = mantissa & ((1u << shift) - 1u);
const uint32_t halfway = 1u << (shift - 1u);
uint32_t result = sign | (static_cast<uint32_t>(halfExponent) << 10);
result += halfMantissa & (halfExponent == 0 ? 0x7ffu : 0x3ffu);
if (remainder > halfway || (remainder == halfway && (halfMantissa & 1u)))
++result;
return static_cast<uint16_t>(result);
}
static float half_to_float(uint16_t bits)
{
const uint32_t sign = static_cast<uint32_t>(bits & 0x8000u) << 16;
const uint32_t exponent = (bits >> 10) & 0x1fu;
const uint32_t mantissa = bits & 0x3ffu;
uint32_t result;
if (exponent == 0)
{
float magnitude = static_cast<float>(mantissa) * (1.0f / 16777216.0f);
std::memcpy(&result, &magnitude, sizeof(result));
result |= sign;
}
else if (exponent == 0x1f)
result = sign | 0x7f800000u | (mantissa << 13);
else
result = sign | ((exponent + 112u) << 23) | (mantissa << 13);
float value;
std::memcpy(&value, &result, sizeof(value));
return value;
}
// Unsigned small floats (R11G11B10): 5-bit exponent, 6 or 5 bit mantissa.
static float small_float_to_float(uint32_t bits, int mantissaBits)
{
const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
if (exponent == 0)
return std::ldexp(static_cast<float>(mantissa), -14 - mantissaBits);
if (exponent == 0x1f)
return mantissa ? NAN : INFINITY;
return std::ldexp(1.0f + static_cast<float>(mantissa) / static_cast<float>(1u << mantissaBits),
static_cast<int>(exponent) - 15);
}
static uint32_t float_to_small_float(float value, int mantissaBits)
{
if (!(value > 0.0f))
return 0; // negatives and NaN clamp to zero
const float maximum = std::ldexp(2.0f - std::ldexp(1.0f, -mantissaBits), 15);
if (value >= maximum)
value = maximum;
int exponent;
const float fraction = std::frexp(value, &exponent); // value = fraction * 2^exponent
int biased = exponent - 1 + 15;
if (biased <= 0)
{
const uint32_t mantissa = static_cast<uint32_t>(std::lround(std::ldexp(value, 14 + mantissaBits)));
return mantissa;
}
uint32_t mantissa = static_cast<uint32_t>(std::lround((fraction * 2.0f - 1.0f) * static_cast<float>(1u << mantissaBits)));
if (mantissa >= (1u << mantissaBits))
{
mantissa = 0;
++biased;
}
if (biased >= 31)
return (30u << mantissaBits) | ((1u << mantissaBits) - 1u);
return (static_cast<uint32_t>(biased) << mantissaBits) | mantissa;
}
// Canonical CUDA-side layouts: color RGBA16F, depth R32F, motion RG16F,
// exposure R32F, output RGBA16F.
enum class Plane
{
Color,
Depth,
Motion,
Exposure
};
static bool supported_input(Plane plane, DXGI_FORMAT format)
{
switch (plane)
{
case Plane::Color:
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB ||
format == DXGI_FORMAT_R8G8B8A8_TYPELESS || format == DXGI_FORMAT_B8G8R8A8_UNORM ||
format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB || format == DXGI_FORMAT_B8G8R8A8_TYPELESS ||
format == DXGI_FORMAT_R9G9B9E5_SHAREDEXP;
case Plane::Depth:
return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS ||
format == DXGI_FORMAT_D24_UNORM_S8_UINT || format == DXGI_FORMAT_R24G8_TYPELESS ||
format == DXGI_FORMAT_R24_UNORM_X8_TYPELESS || format == DXGI_FORMAT_D16_UNORM ||
format == DXGI_FORMAT_R16_UNORM || format == DXGI_FORMAT_R16_TYPELESS ||
format == DXGI_FORMAT_R16_FLOAT;
case Plane::Motion:
return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS ||
format == DXGI_FORMAT_R32G32_FLOAT || format == DXGI_FORMAT_R32G32_TYPELESS ||
format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R8G8B8A8_UNORM;
case Plane::Exposure:
return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS ||
format == DXGI_FORMAT_R16_FLOAT || format == DXGI_FORMAT_R16_TYPELESS ||
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R16G16B16A16_FLOAT;
}
return false;
}
static bool supported_output(DXGI_FORMAT format)
{
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_TYPELESS ||
format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_TYPELESS ||
format == DXGI_FORMAT_R9G9B9E5_SHAREDEXP;
}
static float unorm(uint32_t value, int bits)
{
return static_cast<float>(value) / static_cast<float>((1u << bits) - 1u);
}
static uint32_t to_unorm(float value, int bits)
{
const float clamped = value < 0.0f ? 0.0f : (value > 1.0f ? 1.0f : value);
return static_cast<uint32_t>(std::lround(clamped * static_cast<float>((1u << bits) - 1u)));
}
// Converts one row of a D3D12 texel format into the canonical layout.
static void convert_row_in(Plane plane, DXGI_FORMAT format, const uint8_t* source, void* destination, UINT width)
{
uint16_t* half = static_cast<uint16_t*>(destination);
float* single = static_cast<float*>(destination);
for (UINT x = 0; x < width; ++x)
{
switch (plane)
{
case Plane::Color:
{
float rgba[4] = {0.0f, 0.0f, 0.0f, 1.0f};
switch (format)
{
case DXGI_FORMAT_R16G16B16A16_FLOAT:
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
std::memcpy(half + x * 4, source + x * 8, 8);
continue;
case DXGI_FORMAT_R32G32B32A32_FLOAT:
std::memcpy(rgba, source + x * 16, 16);
break;
case DXGI_FORMAT_R11G11B10_FLOAT:
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, 4);
rgba[0] = small_float_to_float(packed & 0x7ffu, 6);
rgba[1] = small_float_to_float((packed >> 11) & 0x7ffu, 6);
rgba[2] = small_float_to_float((packed >> 22) & 0x3ffu, 5);
break;
}
case DXGI_FORMAT_R10G10B10A2_UNORM:
case DXGI_FORMAT_R10G10B10A2_TYPELESS:
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, 4);
rgba[0] = unorm(packed & 0x3ffu, 10);
rgba[1] = unorm((packed >> 10) & 0x3ffu, 10);
rgba[2] = unorm((packed >> 20) & 0x3ffu, 10);
rgba[3] = unorm(packed >> 30, 2);
break;
}
case DXGI_FORMAT_B8G8R8A8_UNORM:
case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB:
case DXGI_FORMAT_B8G8R8A8_TYPELESS:
rgba[0] = unorm(source[x * 4 + 2], 8);
rgba[1] = unorm(source[x * 4 + 1], 8);
rgba[2] = unorm(source[x * 4 + 0], 8);
rgba[3] = unorm(source[x * 4 + 3], 8);
break;
case DXGI_FORMAT_R9G9B9E5_SHAREDEXP:
{
// Microsoft spec: c = (p >> offset) & 0x1FF, E = (p >> 27) & 0x1F,
// v = c * 2^(E - 24) for every E: there is no implicit leading bit.
uint32_t packed;
std::memcpy(&packed, source + x * 4, 4);
uint32_t e = (packed >> 27) & 0x1Fu;
float scale = std::ldexp(1.0f, (int)e - 24);
rgba[0] = static_cast<float>((packed >> 0) & 0x1FFu) * scale;
rgba[1] = static_cast<float>((packed >> 9) & 0x1FFu) * scale;
rgba[2] = static_cast<float>((packed >> 18) & 0x1FFu) * scale;
rgba[3] = 1.0f;
break;
}
default: // RGBA8 variants
for (int c = 0; c < 4; ++c)
rgba[c] = unorm(source[x * 4 + c], 8);
break;
}
for (int c = 0; c < 4; ++c)
half[x * 4 + c] = float_to_half(rgba[c]);
break;
}
case Plane::Depth:
{
switch (format)
{
case DXGI_FORMAT_D24_UNORM_S8_UINT:
case DXGI_FORMAT_R24G8_TYPELESS:
case DXGI_FORMAT_R24_UNORM_X8_TYPELESS:
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, 4);
single[x] = unorm(packed & 0xffffffu, 24);
break;
}
case DXGI_FORMAT_D16_UNORM:
case DXGI_FORMAT_R16_UNORM:
case DXGI_FORMAT_R16_TYPELESS:
{
uint16_t value;
std::memcpy(&value, source + x * 2, 2);
single[x] = unorm(value, 16);
break;
}
case DXGI_FORMAT_R16_FLOAT:
{
uint16_t value;
std::memcpy(&value, source + x * 2, 2);
single[x] = half_to_float(value);
break;
}
default: // 32-bit float depth (the depth plane of D32S8 copies as 4 bytes)
std::memcpy(single + x, source + x * 4, 4);
break;
}
break;
}
case Plane::Motion:
switch (format)
{
case DXGI_FORMAT_R16G16_FLOAT:
case DXGI_FORMAT_R16G16_TYPELESS:
std::memcpy(half + x * 2, source + x * 4, 4);
break;
case DXGI_FORMAT_R16G16B16A16_FLOAT:
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
std::memcpy(half + x * 2, source + x * 8, 4);
break;
case DXGI_FORMAT_R32G32B32A32_FLOAT:
case DXGI_FORMAT_R32G32_FLOAT:
case DXGI_FORMAT_R32G32_TYPELESS:
{
const size_t stride = format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 8;
float mv[2];
std::memcpy(mv, source + x * stride, 8);
half[x * 2] = float_to_half(mv[0]);
half[x * 2 + 1] = float_to_half(mv[1]);
break;
}
case DXGI_FORMAT_R8G8B8A8_UNORM:
{
const uint8_t* rgba = source + x * 4;
half[x * 2] = float_to_half(static_cast<float>(rgba[0]) / 255.0f);
half[x * 2 + 1] = float_to_half(static_cast<float>(rgba[1]) / 255.0f);
break;
}
default:
break;
}
break;
case Plane::Exposure:
switch (format)
{
case DXGI_FORMAT_R16_FLOAT:
case DXGI_FORMAT_R16_TYPELESS:
case DXGI_FORMAT_R16G16B16A16_FLOAT:
{
uint16_t value;
std::memcpy(&value, source + x * (format == DXGI_FORMAT_R16G16B16A16_FLOAT ? 8 : 2), 2);
single[x] = half_to_float(value);
break;
}
default:
std::memcpy(single + x, source + x * (format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 4), 4);
break;
}
break;
}
}
}
// Formats whose texels are bit-identical to the canonical CUDA layout (the
// cases convert_row_in handles as a plain copy), so the mapped staging rows can
// be copied straight into the CUDA array with their D3D12 row pitch.
static bool canonical_input(Plane plane, DXGI_FORMAT format)
{
switch (plane)
{
case Plane::Color:
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS;
case Plane::Depth:
return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS;
case Plane::Motion:
return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS;
case Plane::Exposure:
return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS;
}
return false;
}
// R11G11B10 floats share binary16's 5-bit exponent and bias, so each channel
// widens to half exactly by shifting its mantissa; NaN becomes the quiet NaN
// float_to_half produces. Matches convert_row_in bit for bit, without libm.
static uint16_t small_float_to_half(uint32_t bits, int mantissaBits)
{
const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
if (exponent == 0x1f && mantissa != 0)
return 0x7e00u;
return static_cast<uint16_t>((exponent << 10) | (mantissa << (10 - mantissaBits)));
}
static void convert_r11g11b10_row(const uint8_t* source, uint16_t* half, UINT width)
{
for (UINT x = 0; x < width; ++x)
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, 4);
half[x * 4 + 0] = small_float_to_half(packed & 0x7ffu, 6);
half[x * 4 + 1] = small_float_to_half((packed >> 11) & 0x7ffu, 6);
half[x * 4 + 2] = small_float_to_half((packed >> 22) & 0x3ffu, 5);
half[x * 4 + 3] = 0x3c00u; // 1.0
}
}
// Packs one RGBA16F DLSS texel into the 9-9-9-5 shared-exponent layout.
// Microsoft spec: v = c * 2^(E - 24), c in [0, 511]. The exponent is chosen as
// the smallest E such that all channels fit in 9 bits, then each channel is
// rounded to the nearest representable value (round-half-even, matching
// D3D's conversion semantics). Values above the format's max (65408.0) are
// clamped, like D3D12 does.
static void rgbe_pack(const float rgba[4], uint8_t* destination)
{
const float MAX_VALUE = 65408.0f; // 511 * 2^(31 - 24)
float r = rgba[0] < 0.0f ? 0.0f : (rgba[0] > MAX_VALUE ? MAX_VALUE : rgba[0]);
float g = rgba[1] < 0.0f ? 0.0f : (rgba[1] > MAX_VALUE ? MAX_VALUE : rgba[1]);
float b = rgba[2] < 0.0f ? 0.0f : (rgba[2] > MAX_VALUE ? MAX_VALUE : rgba[2]);
float mx = fmaxf(fmaxf(r, g), b);
// Smallest scale (i.e. smallest e) such that mx / scale rounds to at most 511. This puts
// the largest channel in the 9-bit range [256, 511] (lower only at e == 0,
// which also encodes zero), the maximum precision the format allows.
int e = 31;
for (int et = 0; et <= 31; ++et)
{
if (mx / std::ldexp(1.0f, et - 24) < 511.5f) { e = et; break; }
}
const float scale = std::ldexp(1.0f, e - 24);
const float inv = 1.0f / scale;
// Round to nearest, ties to even (D3D round-to-nearest-even semantics).
auto to_c = [](float v, float inv) -> uint32_t
{
double d = std::nearbyint(v * inv); // round-half-even
if (d < 0.0) d = 0.0;
if (d > 511.0) d = 511.0;
return static_cast<uint32_t>(d);
};
uint32_t packed = 0;
packed |= to_c(r, inv) << 0;
packed |= to_c(g, inv) << 9;
packed |= to_c(b, inv) << 18;
packed |= (e & 0x1Fu) << 27;
std::memcpy(destination, &packed, 4);
}
// Converts one row of RGBA16F DLSS output into the game's output format.
static void convert_row_out(DXGI_FORMAT format, const uint16_t* source, uint8_t* destination, UINT width)
{
for (UINT x = 0; x < width; ++x)
{
const uint16_t* texel = source + x * 4;
switch (format)
{
case DXGI_FORMAT_R16G16B16A16_FLOAT:
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
std::memcpy(destination + x * 8, texel, 8);
break;
case DXGI_FORMAT_R32G32B32A32_FLOAT:
for (int c = 0; c < 4; ++c)
{
const float value = half_to_float(texel[c]);
std::memcpy(destination + x * 16 + c * 4, &value, 4);
}
break;
case DXGI_FORMAT_R11G11B10_FLOAT:
{
const uint32_t packed = float_to_small_float(half_to_float(texel[0]), 6) |
(float_to_small_float(half_to_float(texel[1]), 6) << 11) |
(float_to_small_float(half_to_float(texel[2]), 5) << 22);
std::memcpy(destination + x * 4, &packed, 4);
break;
}
case DXGI_FORMAT_R10G10B10A2_UNORM:
case DXGI_FORMAT_R10G10B10A2_TYPELESS:
{
const uint32_t packed = to_unorm(half_to_float(texel[0]), 10) |
(to_unorm(half_to_float(texel[1]), 10) << 10) |
(to_unorm(half_to_float(texel[2]), 10) << 20) |
(to_unorm(half_to_float(texel[3]), 2) << 30);
std::memcpy(destination + x * 4, &packed, 4);
break;
}
case DXGI_FORMAT_B8G8R8A8_UNORM:
case DXGI_FORMAT_B8G8R8A8_TYPELESS:
destination[x * 4 + 0] = static_cast<uint8_t>(to_unorm(half_to_float(texel[2]), 8));
destination[x * 4 + 1] = static_cast<uint8_t>(to_unorm(half_to_float(texel[1]), 8));
destination[x * 4 + 2] = static_cast<uint8_t>(to_unorm(half_to_float(texel[0]), 8));
destination[x * 4 + 3] = static_cast<uint8_t>(to_unorm(half_to_float(texel[3]), 8));
break;
case DXGI_FORMAT_R9G9B9E5_SHAREDEXP:
{
float rgba[4] = {half_to_float(texel[0]), half_to_float(texel[1]), half_to_float(texel[2]), 1.0f};
rgbe_pack(rgba, destination + x * 4);
break;
}
default: // RGBA8
for (int c = 0; c < 4; ++c)
destination[x * 4 + c] = static_cast<uint8_t>(to_unorm(half_to_float(texel[c]), 8));
break;
}
}
}
// --- global state ----------------------------------------------------------------
struct Global
{
std::mutex mutex;
bool started = false;
Worker worker; // every NGX-core and CUDA call
std::thread::id workerThread;
Worker prep, finish; // CPU-only pipeline stages around it
Worker cleanup; // deferred GPU resource destruction, never under allocator locks
std::atomic<unsigned int> resourceOwners{0};
bool shutdownPending = false; // under mutex
HMODULE core = nullptr;
CoreApi ngx;
HMODULE cuda = nullptr;
CudaApi cu;
bool ngxInitialized = false;
ID3D12Device* device = nullptr;
std::vector<std::wstring> paths;
std::vector<const wchar_t*> pathPointers;
NgxFeatureCommonInfo featureInfo = {};
std::wstring dataPath;
std::atomic<unsigned int> nextHandleId{0x7200};
// Ray Reconstruction: nvngx_dlssd.dll, loaded and initialised only when asked for.
HMODULE denoiserModule = nullptr;
DenoiserProbe denoiser;
bool denoiserAttempted = false;
std::wstring denoiserPath;
};
static Global g;
static std::mutex g_featureApiMutex;
template <typename Function> static bool load_export(HMODULE module, const char* name, Function& function)
{
function = reinterpret_cast<Function>(reinterpret_cast<void*>(GetProcAddress(module, name)));
if (function == nullptr)
logf("missing export %s", name);
return function != nullptr;
}
// OptiScaler's DLSS-input hook answers kernel32 LoadLibrary* calls for any
// *nvngx*.dll outside the game directory with OptiScaler itself, which would
// hide the official core's CUDA exports. ntdll's LdrLoadDll sits below those
// hooks (OptiScaler's own proxies load libraries the same way).
static HMODULE load_library_below_hooks(const wchar_t* path)
{
struct UnicodeString
{
USHORT Length;
USHORT MaximumLength;
PWSTR Buffer;
};
using LdrLoadDllFn = LONG(NTAPI*)(PWSTR, PULONG, UnicodeString*, PHANDLE);
auto ldrLoadDll = reinterpret_cast<LdrLoadDllFn>(
reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "LdrLoadDll")));
if (ldrLoadDll == nullptr)
return LoadLibraryW(path);
std::wstring copy(path);
UnicodeString name;
name.Length = static_cast<USHORT>(copy.size() * sizeof(wchar_t));
name.MaximumLength = static_cast<USHORT>(name.Length + sizeof(wchar_t));
name.Buffer = copy.data();
HANDLE module = nullptr;
const LONG status = ldrLoadDll(nullptr, nullptr, &name, &module);
if (status < 0)
{
logf("LdrLoadDll(%ls) failed: NTSTATUS 0x%08lx", path, static_cast<unsigned long>(status));
return nullptr;
}
return static_cast<HMODULE>(module);
}
// With DLSS inputs enabled, OptiScaler also hooks ntdll-level loads and
// answers every *nvngx*.dll request with itself unless the requested path
// contains the game EXE's directory in lowercase (its test for "the game's own
// NGX core"). The official core must keep its _nvngx.dll name, so request it
// as <lowercase exe dir>\..\..\<core path>: the hook sees the game
// directory, and Wine collapses the ".." segments to the staged core without
// anything being written into the game folder.
static std::wstring core_path_inside_exe_dir(const wchar_t* corePath)
{
std::wstring core(corePath);
wchar_t exe[MAX_PATH] = {};
const DWORD length = GetModuleFileNameW(nullptr, exe, MAX_PATH);
if (length == 0 || length >= MAX_PATH || core.size() < 3 || core[1] != L':' || exe[1] != L':' ||
towlower(core[0]) != towlower(exe[0]))
return core;
std::wstring directory(exe);
directory.resize(directory.find_last_of(L"\\/"));
for (wchar_t& c : directory)
c = static_cast<wchar_t>(towlower(c));
std::wstring path = directory;
size_t depth = 0;
for (size_t index = 3; index <= directory.size(); ++index)
if (index == directory.size() || directory[index] == L'\\' || directory[index] == L'/')
++depth;
for (size_t level = 0; level < depth; ++level)
path += L"\\..";
path += L"\\";
path += core.substr(3);
logf("requesting NGX core as %ls", path.c_str());
return path;
}
static bool load_libraries()
{
DWORD length = 0;
// Under Proton, nvcuda.dll resolves to Proton's own builtin, which skips the
// bridge's fixups. Loading the bridge by path first makes every later
// LoadLibrary("nvcuda.dll"), including the NGX core's, reuse it.
wchar_t bridgePath[MAX_PATH] = {};
length = GetEnvironmentVariableW(L"D4R_NVCUDA_BRIDGE", bridgePath, MAX_PATH);
if (length > 0 && length < MAX_PATH)
{
HMODULE bridge = LoadLibraryW(bridgePath);
logf("preloaded nvcuda bridge %ls: %p (error %lu)", bridgePath, static_cast<void*>(bridge),
bridge != nullptr ? 0ul : GetLastError());
// d4r.ini's Linux-side settings, before the bridge's first CUDA call loads ZLUDA
if (bridge != nullptr && g_portable.active)
{
using SetEnvFn = int(WINAPI*)(const char*, const char*, int);
const auto setEnv = reinterpret_cast<SetEnvFn>(reinterpret_cast<void*>(GetProcAddress(bridge, "d4rSetEnv")));
if (setEnv == nullptr)
logf("the nvcuda bridge has no d4rSetEnv; d4r.ini's ZLUDA settings are not applied");
for (const auto& [name, value] : g_portable.unixEnv)
if (setEnv != nullptr && setEnv(name.c_str(), value.c_str(), 0))
logf(" %s=%s%s", name.c_str(), value.c_str(),
getenv(name.c_str()) != nullptr && value != getenv(name.c_str()) ? " (set by the environment)" : "");
}
}
wchar_t corePath[MAX_PATH] = {};
length = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
if (length == 0 || length >= MAX_PATH)
{
logf("D4R_NGX_CORE is not set; cannot locate the official NGX core");
return false;
}
g.core = load_library_below_hooks(core_path_inside_exe_dir(corePath).c_str());
if (g.core == nullptr)
{
logf("loading D4R_NGX_CORE=%ls failed: %lu", corePath, GetLastError());
return false;
}
// A successful LoadLibrary can still be OptiScaler's interception proxy.
// Diagnose that before trying to bind the rest of the CUDA NGX API.
if (GetProcAddress(g.core, "NVSDK_NGX_CUDA_Init") == nullptr)
{
wchar_t loadedPath[MAX_PATH] = {};
GetModuleFileNameW(g.core, loadedPath, MAX_PATH);
logf("NGX core load returned an incompatible/intercepted module: %ls; "
"expected %ls. Set [Hooks] HookOriginalNvngxOnly=true in OptiScaler.ini",
loadedPath, corePath);
g.core = nullptr;
return false;
}
bool ok = true;
ok &= load_export(g.core, "NVSDK_NGX_CUDA_Init", g.ngx.init);
if (!load_export(g.core, "NVSDK_NGX_CUDA_Init_ProjectID", g.ngx.initProjectId))
g.ngx.initProjectId = nullptr;
ok &= load_export(g.core, "NVSDK_NGX_CUDA_Shutdown", g.ngx.shutdown);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetParameters", g.ngx.getParameters);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_AllocateParameters", g.ngx.allocateParameters);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetCapabilityParameters", g.ngx.getCapabilityParameters);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_DestroyParameters", g.ngx.destroyParameters);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_CreateFeature", g.ngx.createFeature);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_ReleaseFeature", g.ngx.releaseFeature);
ok &= load_export(g.core, "NVSDK_NGX_CUDA_EvaluateFeature", g.ngx.evaluateFeature);
// Optional: the core answers feature 13 by consulting the denoiser it loaded.
if (!load_export(g.core, "NVSDK_NGX_CUDA_GetFeatureRequirements", g.ngx.getFeatureRequirements))
g.ngx.getFeatureRequirements = nullptr;
g.cuda = LoadLibraryA("nvcuda.dll");
if (g.cuda == nullptr)
{
logf("LoadLibraryA(nvcuda.dll) failed: %lu", GetLastError());
return false;
}
ok &= load_export(g.cuda, "cuMemAlloc", g.cu.memAlloc);
ok &= load_export(g.cuda, "cuMemFree", g.cu.memFree);
ok &= load_export(g.cuda, "cuArrayCreate", g.cu.arrayCreate);
ok &= load_export(g.cuda, "cuArray3DCreate", g.cu.array3DCreate);
ok &= load_export(g.cuda, "cuArrayDestroy", g.cu.arrayDestroy);
ok &= load_export(g.cuda, "cuMemcpy2D", g.cu.memcpy2D);
ok &= load_export(g.cuda, "cuTexObjectCreate", g.cu.texObjectCreate);
ok &= load_export(g.cuda, "cuTexObjectDestroy", g.cu.texObjectDestroy);
ok &= load_export(g.cuda, "cuSurfObjectCreate", g.cu.surfObjectCreate);
ok &= load_export(g.cuda, "cuSurfObjectDestroy", g.cu.surfObjectDestroy);
ok &= load_export(g.cuda, "cuCtxSynchronize", g.cu.ctxSynchronize);
if (!load_export(g.cuda, "cuCtxGetDevice", g.cu.ctxGetDevice))
g.cu.ctxGetDevice = nullptr;
// The d4r bridge's own export keeps waiting when the application's syncs are elided
// (D4R_ELIDE_NGX_SYNC); cuCtxSynchronize above stays the fallback.
{
decltype(g.cu.ctxSynchronize) ownSync = nullptr;
if (load_export(g.cuda, "d4rCtxSynchronize", ownSync))
g.cu.ctxSynchronize = ownSync;
}
if (!load_export(g.cuda, "cuMemAllocHost", g.cu.memAllocHost) ||
!load_export(g.cuda, "cuMemFreeHost", g.cu.memFreeHost) ||
!load_export(g.cuda, "cuMemcpyHtoDAsync", g.cu.memcpyHtoDAsync) ||
!load_export(g.cuda, "cuMemcpyDtoHAsync", g.cu.memcpyDtoHAsync) ||
!load_export(g.cuda, "cuStreamCreate", g.cu.streamCreate) ||
!load_export(g.cuda, "cuStreamSynchronize", g.cu.streamSynchronize))
{
g.cu.memAllocHost = nullptr;
logf("overlapped staging unavailable; using synchronous copies");
}
if (!load_export(g.cuda, "d4rMemcpy2DAsync", g.cu.memcpy2DAsync))
g.cu.memcpy2DAsync = nullptr;
if (!load_export(g.cuda, "d4rStreamWaitValue32", g.cu.streamWaitValue32) ||
!load_export(g.cuda, "d4rWriteValue32", g.cu.writeValue32))
g.cu.streamWaitValue32 = nullptr, g.cu.writeValue32 = nullptr;
if (!load_export(g.cuda, "d4rSetArrayRedirect", g.cu.setArrayRedirect))
g.cu.setArrayRedirect = nullptr;
if (!load_export(g.cuda, "d4rOutputKernelNative", g.cu.outputKernelNative))
g.cu.outputKernelNative = nullptr;
if (!load_export(g.cuda, "d4rRegisterLinearTexture", g.cu.registerLinearTexture))
g.cu.registerLinearTexture = nullptr;
g.cu.dilateMotion = reinterpret_cast<decltype(g.cu.dilateMotion)>(
reinterpret_cast<void*>(GetProcAddress(g.cuda, "d4rDilateMotionVectors")));
if (profile_enabled() || env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0)
{
const bool create = load_export(g.cuda, "cuEventCreate", g.cu.eventCreate);
const bool record = load_export(g.cuda, "cuEventRecord", g.cu.eventRecord);
const bool wait = env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0
? load_export(g.cuda, "d4rEventSynchronize", g.cu.eventSynchronize) : true;
const bool query = load_export(g.cuda, "cuEventQuery", g.cu.eventQuery);
const bool destroy = load_export(g.cuda, "cuEventDestroy", g.cu.eventDestroy);
const bool elapsed = profile_enabled() ? load_export(g.cuda, "cuEventElapsedTime", g.cu.eventElapsedTime) : true;
if (profile_enabled() && !(create && record && destroy && elapsed))
logf("D4R_PROFILE: CUDA event timing unavailable; CPU stage timings remain enabled");
if (env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0 && !(create && record && wait && query && destroy))
logf("blocking output wait unavailable; using context sync");
}
logf("loaded NGX core %ls and nvcuda.dll: %s", corePath, ok ? "all exports present" : "exports missing");
return ok;
}
// Loads nvngx_dlssd.dll and looks for the CUDA entry points that mark a build able to serve
// Ray Reconstruction. Nothing here calls into the denoiser: the NGX core loads and
// dispatches it, so this is a presence-and-capability check, not a second lifecycle.
// Without it Super Sampling is unaffected and Ray Reconstruction reports itself unavailable.
//
// The 3.10.7 build of this library carries the whole snippet stack and exports
// NVSDK_NGX_CUDA_*. Other builds - notably the DLAA-only "rel_tot" one, which ships D3D11,
// D3D12, Vulkan and DirectSR and nothing else - do not, and are rejected here rather than
// half-supported: a missing entry point means there is no CUDA path to bridge to.
static bool load_denoiser_library()
{
if (g.denoiserAttempted)
return g.denoiser.cudaCapable;
g.denoiserAttempted = true;
ensure_portable_config();
// The core finds nvngx_dlssd.dll beside the DLL that called into it and ignores the
// feature search paths, so the only place worth looking is here: offering a path that
// could not redirect the core's own load would be claiming a selection it does not have.
wchar_t self[MAX_PATH] = {};
const DWORD selfLength = GetModuleFileNameW(g_selfModule, self, MAX_PATH);
if (selfLength == 0 || selfLength >= MAX_PATH)
{
logf("Ray Reconstruction: cannot locate this DLL, so the denoiser cannot be found");
return false;
}
std::wstring directory(self, selfLength);
directory.resize(directory.find_last_of(L"\\/"));
g.denoiserPath = directory + L"\\nvngx_dlssd.dll";
if (!file_exists(g.denoiserPath))
{
logf("Ray Reconstruction: no denoiser at %ls; the NGX core loads it from beside this DLL, so copy "
"NVIDIA's nvngx_dlssd.dll (3.10.7) there",
g.denoiserPath.c_str());
return false;
}
g.denoiserModule = load_library_below_hooks(g.denoiserPath.c_str());
if (g.denoiserModule == nullptr)
{
logf("Ray Reconstruction: loading %ls failed: %lu", g.denoiserPath.c_str(), GetLastError());
return false;
}
// Which entry points exist is only ever evidence - none of them is called from here.
// Create/Evaluate/Release is what "can serve Ray Reconstruction" means. NVSDK_NGX_CUDA_Init
// is deliberately not required and is never called: in the 310.7 build it is a stub that
// always reports 0xBAD00001.
void* create = nullptr, *evaluate = nullptr, *release = nullptr;
load_export(g.denoiserModule, "NVSDK_NGX_CUDA_CreateFeature", create);
load_export(g.denoiserModule, "NVSDK_NGX_CUDA_EvaluateFeature", evaluate);
load_export(g.denoiserModule, "NVSDK_NGX_CUDA_ReleaseFeature", release);
g.denoiser.cudaCapable = create != nullptr && evaluate != nullptr && release != nullptr;
wchar_t mapped[MAX_PATH] = {};
GetModuleFileNameW(g.denoiserModule, mapped, MAX_PATH);
logf("found the DLSS-Denoiser library %ls (mapped as %ls): %s", g.denoiserPath.c_str(), mapped,
g.denoiser.cudaCapable ? "CUDA entry points present"
: "no CUDA entry points - not a CUDA-capable denoiser build");
return g.denoiser.cudaCapable;
}
// D4R_RR_ENABLE: 1 asks for Ray Reconstruction, 0 forbids it, and unset (auto) offers it
// exactly when the denoiser really loaded. Auto is the default because the only honest answer
// to a game asking whether Ray Reconstruction is available is what this process can actually
// run - never a fixed yes.
static bool rr_enabled()
{
return env_uint("D4R_RR_ENABLE", 1) != 0;
}
// What the NGX core last said about the denoiser it loaded. The core brings the denoiser up
// during its own init and reports the outcome here; a file merely existing on disk proves
// nothing, because the core can come up perfectly well for Super Sampling with a denoiser
// that failed to initialise.
struct DenoiserCapability
{
bool known = false;
int available = 0;
int initResult = NGX_FAIL_NOT_INITIALIZED;
};
static DenoiserCapability g_denoiserCapability;
static std::mutex g_denoiserCapabilityMutex;
static DenoiserCapability denoiser_capability()
{
std::lock_guard<std::mutex> lock(g_denoiserCapabilityMutex);
return g_denoiserCapability;
}
// Whether a Ray Reconstruction feature can be created right now: the switch is on, a
// CUDA-capable denoiser build is present, and the core itself reported that denoiser
// initialised. Anything less is reported as unavailable rather than discovered at
// CreateFeature, where a game has already committed to the mode.
// Asks the core what it made of the denoiser. Must run on the worker, like every other core
// call, and must not be called from inside one.
static void refresh_denoiser_capability();
static bool ray_reconstruction_available()
{
const DenoiserCapability capability = denoiser_capability();
return rr_enabled() && g.ngxInitialized && g.denoiser.cudaCapable &&
capability.known && capability.available != 0;
}
// Why the route is not on offer, for the log and for the capability answer.
static const char* rr_denial_reason()
{
if (!rr_enabled())
return "disabled by D4R_RR_ENABLE=0";
if (!g.ngxInitialized)
return "the NGX core is not initialised";
if (!g.denoiser.cudaCapable)
return g.denoiserAttempted ? "nvngx_dlssd.dll has no NVSDK_NGX_CUDA_* entry points (needs a 3.10.7 build)"
: "no nvngx_dlssd.dll was found next to this DLL";
const DenoiserCapability capability = denoiser_capability();
if (!capability.known)
return "the NGX core has not reported the denoiser's state yet";
if (capability.available == 0)
{
thread_local char text[160];
snprintf(text, sizeof(text), "the NGX core reports the denoiser unavailable (init 0x%08x)",
static_cast<unsigned int>(capability.initResult));
return text;
}
return "available";
}
// A game that initialises NGX with its project ID (engine integrations such as Unreal's) is identified
// to the NGX core the same way.
struct ProjectIdentity
{
std::string id;
int engineType = 0;
std::string engineVersion;
};
// Discovery is legal before NGX Init; use the same pinned worker for both entry points.
// The caller holds g.mutex.
static NgxResult ensure_workers_started()
{
if (!g.started)
{
// Allocators can retain COM callbacks and detached workers past NGX shutdown.
HMODULE pinned = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_PIN,
reinterpret_cast<LPCWSTR>(g_selfModule), &pinned))
return NGX_FAIL_PLATFORM_ERROR;
g.worker.start();
g.workerThread = g.worker.call([] { return std::this_thread::get_id(); });
g.prep.start();
g.finish.start();
g.cleanup.start();
g.started = true;
}
return NGX_SUCCESS;
}
static NgxResult initialize(unsigned long long applicationId, const wchar_t* dataPath, ID3D12Device* device,
const NgxFeatureCommonInfo* featureInfo, unsigned int sdkVersion,
const ProjectIdentity* project = nullptr)
{
std::lock_guard<std::mutex> lock(g.mutex);
g.shutdownPending = false;
if (g.ngxInitialized)
return NGX_SUCCESS;
if (ensure_workers_started() != NGX_SUCCESS)
return NGX_FAIL_PLATFORM_ERROR;
ensure_portable_config();
if (g_portable.active && !g.core)
{
logf("portable install in %ls (settings from d4r.ini, Linux path %s)", g_portable.dir.c_str(),
g_portable.unixDir.c_str());
for (const std::string& note : g_portable.notes)
logf("d4r: %s", note.c_str());
}
if (g.core == nullptr && !load_libraries())
{
g.core = nullptr;
return NGX_FAIL_PLATFORM_ERROR;
}
if (device != nullptr)
{
device->AddRef();
g.device = device;
// NGX only accepts a CUDA device whose LUID matches the D3D12 adapter;
// the nvcuda bridge reports this LUID from cuDeviceGetLuid.
LUID luid;
device->GetAdapterLuid(&luid);
char low[24], high[24];
std::snprintf(low, sizeof(low), "0x%08lx", static_cast<unsigned long>(luid.LowPart));
std::snprintf(high, sizeof(high), "0x%08lx", static_cast<unsigned long>(luid.HighPart));
SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", low);
SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", high);
SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", "1");
logf("CUDA device LUID set to the D3D12 adapter LUID %s:%s", high, low);
}
g.paths.clear();
wchar_t featureDirectory[MAX_PATH] = {};
DWORD length = GetEnvironmentVariableW(L"D4R_NGX_FEATURE_DIR", featureDirectory, MAX_PATH);
if (length > 0 && length < MAX_PATH)
g.paths.emplace_back(featureDirectory);
if (featureInfo != nullptr && featureInfo->PathListInfo.Path != nullptr)
for (unsigned int index = 0; index < featureInfo->PathListInfo.Length; ++index)
if (featureInfo->PathListInfo.Path[index] != nullptr)
g.paths.emplace_back(featureInfo->PathListInfo.Path[index]);
g.pathPointers.clear();
for (const std::wstring& path : g.paths)
{
g.pathPointers.push_back(path.c_str());
logf("feature search path: %ls", path.c_str());
}
g.featureInfo = {};
g.featureInfo.PathListInfo.Path = g.pathPointers.data();
g.featureInfo.PathListInfo.Length = static_cast<unsigned int>(g.pathPointers.size());
g.featureInfo.LoggingInfo.LoggingCallback = reinterpret_cast<void*>(&ngx_log_callback);
g.featureInfo.LoggingInfo.MinimumLoggingLevel = static_cast<int>(env_uint("D4R_NGX_LOG_LEVEL", 1)); // 1=on, 2=verbose
g.featureInfo.LoggingInfo.DisableOtherLoggingSinks = false;
g.dataPath = dataPath != nullptr ? dataPath : L".";
// The CUDA path takes the same application identity as the game's D3D12 init. (The SDK sample's id
// is only a last resort: NGX stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output for it.)
const unsigned long long id = applicationId != 0 ? applicationId : 241534723ULL;
const unsigned int version = sdkVersion != 0 ? sdkVersion : 0x15;
// Logging info is only read for API versions 0x14 and later.
const unsigned int apiVersion = version < 0x14 ? 0x14 : version;
NgxResult result;
if (project != nullptr && !project->id.empty() && g.ngx.initProjectId != nullptr)
{
result = g.worker.call([&] {
return g.ngx.initProjectId(project->id.c_str(), project->engineType, project->engineVersion.c_str(),
g.dataPath.c_str(), apiVersion, &g.featureInfo);
});
logf("NVSDK_NGX_CUDA_Init_ProjectID(project=%s, engine=%d %s, sdk=0x%x) -> 0x%08x", project->id.c_str(),
project->engineType, project->engineVersion.c_str(), version, result);
}
else
{
result = g.worker.call([&] { return g.ngx.init(id, g.dataPath.c_str(), &g.featureInfo, apiVersion); });
logf("NVSDK_NGX_CUDA_Init(app=%llu, sdk=0x%x) -> 0x%08x", id, version, result);
}
using LoadErrorFn = const char*(WINAPI*)();
const auto loadError = reinterpret_cast<LoadErrorFn>(reinterpret_cast<void*>(GetProcAddress(g.cuda, "d4rLoadError")));
if (loadError != nullptr && loadError()[0] != '\0')
logf("nvcuda bridge: %s", loadError());
g.ngxInitialized = result == NGX_SUCCESS;
// The denoiser is not initialised here. The NGX core loads and dispatches
// nvngx_dlssd.dll itself, so a second init would duplicate that state rather than add
// anything; this only looks for the library and checks it is a build that can serve
// Ray Reconstruction. Its own NVSDK_NGX_CUDA_Init is a stub that always fails, so
// calling it would report "unavailable" for a denoiser that is in fact working.
if (rr_enabled())
{
load_denoiser_library();
logf("Ray Reconstruction: %s", rr_denial_reason());
}
else
logf("Ray Reconstruction: disabled by D4R_RR_ENABLE=0");
return result;
}
// --- per-feature state ---------------------------------------------------------
constexpr int kSlots = 3;
// One more result slot than input slots: a slot is reused only once the GPU
// is past every command list that copies from it (see claim_output_slot).
constexpr int kOutputSlots = kSlots + 1;
// A VkBuffer in device-local memory on vkd3d-proton's VkDevice, exported to
// the CUDA side (see "VRAM interop").
struct VramBuffer
{
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
CudaDevicePtr device = 0;
void* external = nullptr;
size_t bytes = 0;
void* mapped = nullptr; // engine readiness word: CPU-coherent, kept until HIP and Vulkan retire
// What a pooled buffer can be reused as: 0 the interop transfer buffers (create_vram_buffer), 1 and 2 the
// engine's device-local and host-coherent storage buffers (create_engine_shared_buffer).
uint8_t kind = 0;
};
// A temporary image for colour or exposure format conversion.
struct VramImage
{
VkImage image = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
UINT width = 0, height = 0;
VkFormat format = VK_FORMAT_UNDEFINED;
};
struct Staging
{
ID3D12Resource* buffer = nullptr;
uint8_t* mapped = nullptr;
D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout = {};
UINT rows = 0;
UINT64 total = 0;
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
UINT width = 0;
UINT height = 0;
std::vector<ID3D12Resource*> retired;
void retire()
{
if (buffer != nullptr)
retired.push_back(buffer);
buffer = nullptr;
mapped = nullptr;
width = height = 0;
format = DXGI_FORMAT_UNKNOWN;
}
void release()
{
if (buffer != nullptr)
buffer->Release();
for (ID3D12Resource* old : retired)
old->Release();
*this = Staging{};
}
};
struct CudaImage
{
CudaArray array = nullptr;
CudaObject object = 0; // texture or surface object
uint32_t format = 0, channels = 0; // the layout the array carries, so a format change
// recreates it rather than only reacting to a resize
UINT width = 0;
UINT height = 0;
};
// --- Ray Reconstruction auxiliary inputs --------------------------------------
//
// Super Sampling needs four surfaces (colour, depth, motion vectors, exposure) and the
// file above already moves those. Ray Reconstruction additionally runs a denoiser network
// over the ray-traced G-buffer, so a game that enables it registers a further set of
// surfaces: normals, roughness, diffuse and specular albedo, emissive, the ray hit
// distances and directions, and the effect guides for particles, transparency, fog and
// depth of field.
//
// Those surfaces are read in whatever format the game renders them in, and the denoiser
// samples them through texture objects minted by this file. Rather than force them
// through the canonical planes above - which would re-encode a UNORM normal as half and
// silently change the values the network sees - each is copied with its own format and
// channel count, so the texture object describes exactly what the game drew.
// CUarray_format values beyond the two above (nvsdk: CU_AD_FORMAT_*).
constexpr uint32_t CUDA_FORMAT_UNSIGNED_INT8 = 0x01;
constexpr uint32_t CUDA_FORMAT_SIGNED_INT8 = 0x08;
constexpr uint32_t CUDA_FORMAT_UNSIGNED_INT16 = 0x2;
constexpr uint32_t CUDA_FORMAT_SIGNED_INT16 = 0x09;
constexpr uint32_t CUDA_FORMAT_UNSIGNED_INT32 = 0x03;
constexpr uint32_t CUDA_FORMAT_SIGNED_INT32 = 0x0a;
// How one denoiser input reaches the GPU.
enum class AuxLayout
{
Verbatim, // the bytes are already exactly what the CUDA array must hold
Converted // the bytes must be unpacked into floats first (see convert_aux_row)
};
// The CUDA array a denoiser input uses.
//
// Only formats that are byte-for-byte correct go down the verbatim path, and that is a
// shorter list than it looks. A CUDA array stores element_size * channels bytes per texel
// and only ever has 1, 2 or 4 channels, so a packed format has no verbatim layout at all:
// R10G10B10A2_UNORM is 4 bytes per texel, but a 4-channel uint array is 16, and copying
// "4 channels' worth" would read four times past the end of every row. R11G11B10_FLOAT has
// three channels, which no CUDA array has.
//
// Nor is memory layout the only question. Sampling an integer-typed CUDA array normalizes it,
// which is right for a UNORM or SNORM surface and wrong for a UINT one, where a D3D12 shader
// reads the raw integer. And B8G8R8A8_UNORM has RGBA bytes in BGRA order, so a verbatim copy
// would hand the denoiser a texture whose red and blue channels are swapped.
//
// Everything that is not in this table is converted to a float array instead, which is
// correct for all of those cases at the cost of a per-row unpack.
static bool aux_array_layout(DXGI_FORMAT format, uint32_t& arrayFormat, uint32_t& channels, AuxLayout& how)
{
struct Entry
{
DXGI_FORMAT format;
uint32_t arrayFormat;
uint32_t channels;
};
// UNORM and SNORM map onto signed/unsigned integer arrays of the same width: sampling
// normalizes them, which is exactly how a D3D12 shader reads the resource.
static const Entry entries[] = {
{DXGI_FORMAT_R8_UNORM, CUDA_FORMAT_UNSIGNED_INT8, 1},
{DXGI_FORMAT_R8_SNORM, CUDA_FORMAT_SIGNED_INT8, 1},
{DXGI_FORMAT_R8G8_UNORM, CUDA_FORMAT_UNSIGNED_INT8, 2},
{DXGI_FORMAT_R8G8_SNORM, CUDA_FORMAT_SIGNED_INT8, 2},
{DXGI_FORMAT_R8G8B8A8_UNORM, CUDA_FORMAT_UNSIGNED_INT8, 4},
{DXGI_FORMAT_R8G8B8A8_SNORM, CUDA_FORMAT_SIGNED_INT8, 4},
{DXGI_FORMAT_R16_UNORM, CUDA_FORMAT_UNSIGNED_INT16, 1},
{DXGI_FORMAT_R16_SNORM, CUDA_FORMAT_SIGNED_INT16, 1},
{DXGI_FORMAT_R16_FLOAT, CUDA_FORMAT_HALF, 1},
{DXGI_FORMAT_R16G16_UNORM, CUDA_FORMAT_UNSIGNED_INT16, 2},
{DXGI_FORMAT_R16G16_SNORM, CUDA_FORMAT_SIGNED_INT16, 2},
{DXGI_FORMAT_R16G16_FLOAT, CUDA_FORMAT_HALF, 2},
{DXGI_FORMAT_R16G16B16A16_UNORM, CUDA_FORMAT_UNSIGNED_INT16, 4},
{DXGI_FORMAT_R16G16B16A16_SNORM, CUDA_FORMAT_SIGNED_INT16, 4},
{DXGI_FORMAT_R16G16B16A16_FLOAT, CUDA_FORMAT_HALF, 4},
{DXGI_FORMAT_R32_FLOAT, CUDA_FORMAT_FLOAT, 1},
{DXGI_FORMAT_R32G32_FLOAT, CUDA_FORMAT_FLOAT, 2},
{DXGI_FORMAT_R32G32B32A32_FLOAT, CUDA_FORMAT_FLOAT, 4},
};
for (const Entry& entry : entries)
if (entry.format == format)
{
arrayFormat = entry.arrayFormat;
channels = entry.channels;
how = AuxLayout::Verbatim;
return true;
}
// Converted formats all end up as float arrays, so a shader read of the denoiser's
// texture returns the same numbers a D3D12 read of the caller's resource would.
struct Converted
{
DXGI_FORMAT format;
uint32_t channels;
};
static const Converted converted[] = {
{DXGI_FORMAT_R8_UINT, 1},
{DXGI_FORMAT_R8G8_UINT, 2},
{DXGI_FORMAT_R8G8B8A8_UINT, 4},
{DXGI_FORMAT_R16_UINT, 1},
{DXGI_FORMAT_R32_UINT, 1},
{DXGI_FORMAT_R32_SINT, 1},
{DXGI_FORMAT_R32G32_UINT, 2},
{DXGI_FORMAT_R32G32_SINT, 2},
{DXGI_FORMAT_R32G32B32A32_UINT, 4},
{DXGI_FORMAT_B8G8R8A8_UNORM, 4},
{DXGI_FORMAT_R10G10B10A2_UNORM, 4},
{DXGI_FORMAT_R11G11B10_FLOAT, 3},
};
for (const Converted& entry : converted)
if (entry.format == format)
{
arrayFormat = CUDA_FORMAT_FLOAT;
channels = entry.channels;
how = AuxLayout::Converted;
return true;
}
return false;
}
// Unpacks one row of a converted denoiser input into `channels` floats per texel. Every
// branch reads exactly the bytes of one texel and no more.
static void convert_aux_row(DXGI_FORMAT format, const uint8_t* source, float* destination, UINT width,
uint32_t channels)
{
switch (format)
{
case DXGI_FORMAT_R8_UINT:
for (UINT x = 0; x < width; ++x)
destination[x] = static_cast<float>(source[x]);
return;
case DXGI_FORMAT_R8G8_UINT:
for (UINT x = 0; x < width; ++x)
{
destination[x * 2] = static_cast<float>(source[x * 2]);
destination[x * 2 + 1] = static_cast<float>(source[x * 2 + 1]);
}
return;
case DXGI_FORMAT_R8G8B8A8_UINT:
for (UINT x = 0; x < width; ++x)
for (uint32_t c = 0; c < 4; ++c)
destination[x * 4 + c] = static_cast<float>(source[x * 4 + c]);
return;
case DXGI_FORMAT_R16_UINT:
for (UINT x = 0; x < width; ++x)
{
uint16_t value;
std::memcpy(&value, source + x * 2, sizeof(value));
destination[x] = static_cast<float>(value);
}
return;
case DXGI_FORMAT_R32_UINT:
for (UINT x = 0; x < width; ++x)
{
uint32_t value;
std::memcpy(&value, source + x * 4, sizeof(value));
destination[x] = static_cast<float>(value);
}
return;
case DXGI_FORMAT_R32_SINT:
for (UINT x = 0; x < width; ++x)
{
int32_t value;
std::memcpy(&value, source + x * 4, sizeof(value));
destination[x] = static_cast<float>(value);
}
return;
case DXGI_FORMAT_R32G32_UINT:
for (UINT x = 0; x < width; ++x)
{
uint32_t value[2];
std::memcpy(value, source + x * 8, sizeof(value));
destination[x * 2] = static_cast<float>(value[0]);
destination[x * 2 + 1] = static_cast<float>(value[1]);
}
return;
case DXGI_FORMAT_R32G32_SINT:
for (UINT x = 0; x < width; ++x)
{
int32_t value[2];
std::memcpy(value, source + x * 8, sizeof(value));
destination[x * 2] = static_cast<float>(value[0]);
destination[x * 2 + 1] = static_cast<float>(value[1]);
}
return;
case DXGI_FORMAT_R32G32B32A32_UINT:
for (UINT x = 0; x < width; ++x)
for (uint32_t c = 0; c < 4; ++c)
{
uint32_t value;
std::memcpy(&value, source + x * 16 + c * 4, sizeof(value));
destination[x * 4 + c] = static_cast<float>(value);
}
return;
case DXGI_FORMAT_B8G8R8A8_UNORM:
// Stored B, G, R, A; the denoiser's texture is read as R, G, B, A.
for (UINT x = 0; x < width; ++x)
{
const uint8_t* texel = source + x * 4;
destination[x * 4] = unorm(texel[2], 8);
destination[x * 4 + 1] = unorm(texel[1], 8);
destination[x * 4 + 2] = unorm(texel[0], 8);
destination[x * 4 + 3] = unorm(texel[3], 8);
}
return;
case DXGI_FORMAT_R10G10B10A2_UNORM:
for (UINT x = 0; x < width; ++x)
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, sizeof(packed));
destination[x * 4] = unorm(packed & 0x3FF, 10);
destination[x * 4 + 1] = unorm((packed >> 10) & 0x3FF, 10);
destination[x * 4 + 2] = unorm((packed >> 20) & 0x3FF, 10);
destination[x * 4 + 3] = unorm((packed >> 30) & 0x3, 2);
}
return;
case DXGI_FORMAT_R11G11B10_FLOAT:
for (UINT x = 0; x < width; ++x)
{
uint32_t packed;
std::memcpy(&packed, source + x * 4, sizeof(packed));
destination[x * 3] = small_float_to_float(packed & 0x7FF, 6);
destination[x * 3 + 1] = small_float_to_float((packed >> 11) & 0x7FF, 6);
destination[x * 3 + 2] = small_float_to_float((packed >> 22) & 0x3FF, 5);
}
return;
default:
break;
}
// Unreachable: aux_array_layout only marks a format Converted when this switch handles it.
for (UINT x = 0; x < width * channels; ++x)
destination[x] = 0.0f;
}
// The texel size `format` occupies, for the pitch of an interop buffer holding it verbatim.
static size_t aux_texel_bytes(uint32_t arrayFormat, uint32_t channels)
{
const size_t element = arrayFormat == CUDA_FORMAT_FLOAT || arrayFormat == CUDA_FORMAT_UNSIGNED_INT32 ||
arrayFormat == CUDA_FORMAT_SIGNED_INT32
? 4
: (arrayFormat == CUDA_FORMAT_HALF || arrayFormat == CUDA_FORMAT_UNSIGNED_INT16 ||
arrayFormat == CUDA_FORMAT_SIGNED_INT16)
? 2
: 1;
return element * channels;
}
// One auxiliary surface: the name to ask the caller for, and the name the denoiser reads
// it under. They differ for the two albedos, which the SDK and the shipped library spell
// differently, in which case `source` names the caller's parameter and `publish` the
// denoiser's.
struct AuxInput
{
const char* source;
const char* publish;
};
// Every auxiliary read name nvngx_dlssd.dll looks up, verified against the names in the
// shipped library and the SDK's nvsdk_ngx_defs_dlssd.h. A caller may bind several of these
// to one texture, so surfaces are keyed on the resource and carry every name bound to it.
//
// The publish column is the name the denoiser reads, taken from the shipped library's own
// lookup table (its .rdata strings, each referenced exactly once from
// NGXDLSSD::EvaluateFeature). It is not always the SDK's spelling: nvngx_dlssd.dll declares
// DLSS.Input.Normals.Subrect.Base.X/Y and DLSS.Input.Roughness.Subrect.Base.X/Y, but it reads
// the normals and roughness textures themselves as GBuffer.Normals and GBuffer.Roughness. It
// does read the two albedos as DLSS.Input.DiffuseAlbedo and DLSS.Input.SpecularAlbedo, which a
// caller may instead supply as GBuffer.DiffuseAlbedo / GBuffer.SpecularAlbedo - the SDK defines
// both spellings, so both are accepted and both publish the name the denoiser asks for. The
// subrect origins in kAuxSubrectNames are a separate matter and keep the SDK's own names.
static const AuxInput kAuxInputs[] = {
// Ray-traced G-buffer
{"GBuffer.Normals", "GBuffer.Normals"},
{"GBuffer.Roughness", "GBuffer.Roughness"},
{"DLSS.Input.DiffuseAlbedo", "DLSS.Input.DiffuseAlbedo"},
{"GBuffer.DiffuseAlbedo", "DLSS.Input.DiffuseAlbedo"},
{"GBuffer.IndirectAlbedo", "GBuffer.IndirectAlbedo"},
{"DLSS.Input.SpecularAlbedo", "DLSS.Input.SpecularAlbedo"},
{"GBuffer.SpecularAlbedo", "DLSS.Input.SpecularAlbedo"},
{"GBuffer.Emissive", "GBuffer.Emissive"},
{"GBuffer.SpecularMvec", "GBuffer.SpecularMvec"},
// Ray-tracing noise hints
{"RayTracingHitDistance", "RayTracingHitDistance"},
{"DLSSD.DiffuseHitDistance", "DLSSD.DiffuseHitDistance"},
{"DLSSD.SpecularHitDistance", "DLSSD.SpecularHitDistance"},
{"DLSSD.DiffuseRayDirection", "DLSSD.DiffuseRayDirection"},
{"DLSSD.SpecularRayDirection", "DLSSD.SpecularRayDirection"},
{"DLSSD.DiffuseRayDirectionHitDistance", "DLSSD.DiffuseRayDirectionHitDistance"},
{"DLSSD.SpecularRayDirectionHitDistance", "DLSSD.SpecularRayDirectionHitDistance"},
{"DLSSD.ReflectedAlbedo", "DLSSD.ReflectedAlbedo"},
// Per-effect guides, read only when the game preserves that effect
{"DLSSD.ColorBeforeParticles", "DLSSD.ColorBeforeParticles"},
{"DLSSD.ColorAfterParticles", "DLSSD.ColorAfterParticles"},
{"DLSSD.ColorBeforeTransparency", "DLSSD.ColorBeforeTransparency"},
{"DLSSD.ColorAfterTransparency", "DLSSD.ColorAfterTransparency"},
{"DLSSD.ColorBeforeFog", "DLSSD.ColorBeforeFog"},
{"DLSSD.ColorAfterFog", "DLSSD.ColorAfterFog"},
{"DLSSD.DepthOfFieldGuide", "DLSSD.DepthOfFieldGuide"},
{"DLSSD.ColorBeforeDepthOfField", "DLSSD.ColorBeforeDepthOfField"},
{"DLSSD.ColorAfterDepthOfField", "DLSSD.ColorAfterDepthOfField"},
{"DLSSD.ScreenSpaceRefractionGuide", "DLSSD.ScreenSpaceRefractionGuide"},
{"DLSSD.ColorBeforeScreenSpaceRefraction", "DLSSD.ColorBeforeScreenSpaceRefraction"},
{"DLSSD.ColorAfterScreenSpaceRefraction", "DLSSD.ColorAfterScreenSpaceRefraction"},
{"DLSSD.ScreenSpaceSubsurfaceScatteringGuide", "DLSSD.ScreenSpaceSubsurfaceScatteringGuide"},
{"DLSSD.ColorBeforeScreenSpaceSubsurfaceScattering", "DLSSD.ColorBeforeScreenSpaceSubsurfaceScattering"},
{"DLSSD.ColorAfterScreenSpaceSubsurfaceScattering", "DLSSD.ColorAfterScreenSpaceSubsurfaceScattering"},
{"DLSSD.Alpha", "DLSSD.Alpha"},
// Masks and secondary motion
{"DLSS.DisocclusionMask", "DLSS.DisocclusionMask"},
{"DLSS.Input.Reduce.Ghost.Mask", "DLSS.Input.Reduce.Ghost.Mask"},
{"DLSSD.ResponsivityMask", "DLSSD.ResponsivityMask"},
{"MotionVectorsReflection", "MotionVectorsReflection"},
{"TransparencyMask", "TransparencyMask"},
{"DepthHighRes", "DepthHighRes"},
};
// Scalars the denoiser reads per frame, forwarded from the caller's own parameters.
struct AuxScalar
{
const char* name;
};
static const AuxScalar kAuxScalars[] = {
{"DLSS.Denoise.Mode"},
{"DLSS.Roughness.Mode"},
{"DLSS.Use.HW.Depth"},
{"DLSS.Use.Folded.Network"},
{"DLSSD.IndicatorLevel"},
{"DLSS.Translucency.Type"},
{"DLSS.Subsurface.Type"},
{"DLSS.Specular.Type"},
};
// The camera matrices the denoiser reprojects with, forwarded as the float[16] the caller
// registered (the SDK passes matrices by pointer, not as sixteen scalars).
static const char* const kAuxMatrices[] = {"WorldToViewMatrix", "ViewToClipMatrix"};
static unsigned int get_uint_or(void* parameters, const char* name, unsigned int fallback);
// The Subrect origins nvngx_dlssd.dll declares for its auxiliary inputs
// (nvsdk_ngx_defs_dlssd.h:60-67 and 99-146). A name here has both a .Subrect.Base.X and a
// .Subrect.Base.Y parameter; the parallel FrameParams array holds one origin per entry, and
// the strings are static so no per-frame allocation is needed to forward them.
static const char* const kAuxSubrectNames[] = {
"DLSS.Input.DiffuseAlbedo", "DLSS.Input.SpecularAlbedo", "DLSS.Input.Normals",
"DLSS.Input.Roughness", "DLSSD.Alpha", "DLSSD.ReflectedAlbedo",
"DLSSD.ColorBeforeParticles", "DLSSD.ColorAfterParticles", "DLSSD.ColorBeforeTransparency",
"DLSSD.ColorAfterTransparency", "DLSSD.ColorBeforeFog", "DLSSD.ColorAfterFog",
"DLSSD.ScreenSpaceRefractionGuide", "DLSSD.ColorBeforeScreenSpaceRefraction",
"DLSSD.ColorAfterScreenSpaceRefraction", "DLSSD.ScreenSpaceSubsurfaceScatteringGuide",
"DLSSD.ColorBeforeScreenSpaceSubsurfaceScattering",
"DLSSD.ColorAfterScreenSpaceSubsurfaceScattering", "DLSSD.DepthOfFieldGuide",
"DLSSD.ColorBeforeDepthOfField", "DLSSD.ColorAfterDepthOfField", "DLSSD.DiffuseHitDistance",
"DLSSD.SpecularHitDistance", "DLSSD.DiffuseRayDirection", "DLSSD.SpecularRayDirection",
"DLSSD.DiffuseRayDirectionHitDistance", "DLSSD.SpecularRayDirectionHitDistance",
};
struct FrameParams
{
float jitterX = 0, jitterY = 0, mvScaleX = 1, mvScaleY = 1, sharpness = 0;
float preExposure = 1, exposureScale = 1, frameTime = 16.6f;
int reset = 0, invertX = 0, invertY = 0;
unsigned int renderWidth = 0, renderHeight = 0;
unsigned int colorBaseX = 0, colorBaseY = 0, depthBaseX = 0, depthBaseY = 0;
unsigned int mvBaseX = 0, mvBaseY = 0, outputBaseX = 0, outputBaseY = 0;
bool hasExposure = false;
bool vram = false; // inputs and output stay in VRAM (VRAM interop)
bool split = false; // this frame presents its own result (split frames)
unsigned motionDilation = 0; // actually applied, worker only
// D4R_SHIM_INPLACE: the game's linear output texture, written without a game-side copy (0 = none)
CudaDevicePtr outputInPlace = 0;
uint32_t outputInPlacePitch = 0;
// D4R_SHIM_OUTPUT_PACKED: the output kernel stores the game's R10G10B10A2 texels itself, so the game-side
// copy (or the in-place texture) takes them without the RGBA16F conversion image and blit
bool packedOutput = false;
// Ray Reconstruction: the denoiser's own per-frame settings and camera matrices, copied
// off the caller's parameter object on the game thread so the worker never reads an
// object the game may have destroyed by then. A scalar the caller did not set this frame
// carries the creation-time value, not an invented one and not last frame's override; a
// matrix the caller did not set is published as absent, because the pointer names this
// frame's storage and nothing outlives it.
int auxScalar[sizeof(kAuxScalars) / sizeof(kAuxScalars[0])] = {};
bool auxScalarSet[sizeof(kAuxScalars) / sizeof(kAuxScalars[0])] = {};
float auxMatrix[sizeof(kAuxMatrices) / sizeof(kAuxMatrices[0])][16] = {};
bool auxMatrixSet[sizeof(kAuxMatrices) / sizeof(kAuxMatrices[0])] = {};
// One Subrect origin per entry of kAuxSubrectNames, snapshotted per frame. Zero means the
// caller did not register one, and zero is what is forwarded - never the previous frame's.
unsigned int auxSubrect[sizeof(kAuxSubrectNames) / sizeof(kAuxSubrectNames[0])][2] = {};
};
struct FrameTiming
{
bool enabled = false;
ProfileClock::time_point gameStart, queuedAt;
double slotWait = 0, inputRecord = 0, outputRecord = 0, gameCall = 0, gameInterval = 0;
double convert[4] = {}, upload[4] = {};
double download = 0, outputConvert = 0;
double prep = 0, worker = 0, workerWait = 0, finishWait = 0, markerWait = 0;
double ngxHost = 0, ctxSync = 0, gpuEval = -1;
double motionDilationWall = 0;
bool markerDeferred = false; // the worker waits for the input marker (D4R_SHIM_DEFER_MARKER)
bool gpuEventsRecorded = false;
double outputSyncWall = 0, outputSyncCpu = -1;
bool outputSyncBlocking = false;
double h2dTotal = 0, d2hIssue = 0, d2hWait = 0;
ProfileClock::time_point prepStart, workerQueued, finishQueued;
size_t queueDepth = 0;
uint32_t presentedFrame = 0;
};
// A plane in the canonical CUDA layout, tightly packed, in ordinary host
// memory: filled from the D3D12 staging by one pipeline stage and consumed by
// the next (see prepare_inputs / run_evaluation / publish_output).
struct HostPlane
{
uint8_t* bytes = nullptr; // page-locked when overlapped staging is on
size_t capacity = 0;
bool pinned = false;
std::vector<uint8_t> fallback;
UINT width = 0, height = 0;
size_t rowBytes = 0;
size_t size() const { return rowBytes * height; }
};
// Ray Reconstruction auxiliary surface: one per distinct D3D12Resource the caller registers,
// keyed on the resource because several names routinely resolve to one texture. The surface
// carries the caller's own DXGI format - it is never converted to a canonical plane - so the
// texture object the denoiser samples describes exactly what was rendered.
//
// The surface outlives the frames that name it: `holds` counts queued frames still referring
// to it, and the resource is AddRef'd so a caller that releases it while a frame is in flight
// cannot pull it out from under the copy recorded on its command list.
struct AuxSurface
{
ID3D12Resource* resource = nullptr;
std::vector<const char*> publish;
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
UINT width = 0, height = 0;
uint32_t arrayFormat = 0, channels = 0;
AuxLayout how = AuxLayout::Verbatim; // whether the bytes reach the array as they are
uint32_t lastSeen = 0; // frame the caller last registered this resource
unsigned int holds = 0; // queued frames still naming this surface
bool dropping = false; // no longer registered; freed once holds reaches zero
CudaImage image; // worker only
CudaObject handle = 0; // what the parameters point at; its address must stay put
Staging staging[kSlots];
HostPlane host[kSlots]; // unpacked floats, only for a converted surface (worker only)
};
// What one frame's slot bound: which surface answered which denoiser name, for that frame
// only. Publish reads this, never the live surface list, so a surface the caller registered
// last frame is never handed to the denoiser for a frame that did not register it.
struct AuxBinding
{
AuxSurface* surface = nullptr;
std::vector<const char*> names;
uint32_t frame = 0;
};
struct InputSlot
{
Staging color, depth, motion, exposure;
VramBuffer vram[4]; // VRAM interop: the planes in canonical layout
// D4R_SHIM_LINEAR_INPUTS: pitch-linear texture objects on vram[] handed to NGX (worker only)
CudaObject linearTexture[4] = {};
CudaDevicePtr linearPointer[4] = {};
UINT linearWidth[4] = {}, linearHeight[4] = {};
size_t linearPitch[4] = {};
CudaDevicePtr dilatedMotion = 0;
size_t dilatedBytes = 0;
CudaObject dilatedTexture = 0;
UINT dilatedWidth = 0, dilatedHeight = 0;
size_t dilatedPitch = 0;
std::vector<AuxBinding> auxBindings; // this slot's denoiser inputs, for one frame
// The DLSSD.OutputAlpha texture the caller asked for on this frame, held until the frame
// retires. Present only on a frame that actually registered it.
ID3D12Resource* alphaResource = nullptr;
UINT alphaWidth = 0, alphaHeight = 0, alphaBaseX = 0, alphaBaseY = 0;
UINT alphaExtentW = 0, alphaExtentH = 0;
DXGI_FORMAT alphaFormat = DXGI_FORMAT_UNKNOWN;
uint32_t alphaWriteback = 0;
bool alphaRequested = false;
std::atomic<bool> busy{false}; // readback staging owned by the pipeline
HostPlane host[4];
std::atomic<bool> hostBusy{false}; // host planes not yet uploaded by the worker
};
// DLSSD.OutputAlpha is an OUTPUT the denoiser writes (nvsdk_ngx_defs_dlssd.h:98, with its own
// Subrect base at 101/102). It is delivered back through its own ring of slots, modelled on
// OutputSlot above, because it has exactly the same lifetime problem as the colour output: a
// result staged for one frame is read by a later frame's command list, so it must not be
// rewritten until every reader has passed.
struct AlphaSlot
{
Staging staging; // upload buffer holding one result
std::atomic<uint32_t> lastReadFrame{0}; // reserved/last-read frame, as OutputSlot's
std::atomic<uint32_t> producedFrame{0}; // frame that filled it
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
UINT baseX = 0, baseY = 0; // where this result sits in the staging buffer
UINT width = 0, height = 0; // the region it covers
uint32_t writeback = 0; // destination texel kind (see alpha_output_layout)
};
struct OutputSlot
{
Staging staging;
VramBuffer vram; // VRAM interop: the result in RGBA16F
std::atomic<uint32_t> lastReadFrame{0}; // last frame whose command list copies from it
std::atomic<uint32_t> producedFrame{0};
};
// Views and their D3D12 image owners survive every command list recorded by this feature.
struct EngineBorrowedView
{
ID3D12Resource* resource = nullptr;
VkDevice device = VK_NULL_HANDLE;
VkImageView view = VK_NULL_HANDLE;
VkFormat format = VK_FORMAT_UNDEFINED;
VkImageAspectFlags aspect = 0;
VkImageUsageFlags usage = 0;
~EngineBorrowedView()
{
if (view) d4r::vkTable.DestroyImageView(device, view, nullptr);
if (resource) resource->Release();
}
};
struct Feature
{
NgxHandle handle = {}; // returned to the caller
bool nativeVulkan = false; // distinct resource/lifetime contract
std::atomic<bool> retiring{false};
IUnknown* resources = nullptr; // main reference plus command-allocator references
NgxHandle* cudaHandle = nullptr;
void* cudaParams = nullptr;
CudaDevicePtr scratch = 0;
unsigned int width = 0, height = 0, outWidth = 0, outHeight = 0;
unsigned int cudaWidth = 0, cudaHeight = 0; // render size the NGX feature was created with (worker only)
unsigned int refusedWidth = 0, refusedHeight = 0; // render size NGX would not create a feature for (worker only)
int quality = 0, flags = 0, subrects = 0;
std::array<unsigned int, 6> creationPresets{};
bool cudaCreationDeferred = false;
unsigned int preset = 0;
CudaEvent profileStart = nullptr, profileEnd = nullptr;
CudaEvent outputReadyEvent = nullptr;
// output wait pacing (D4R_SHIM_OUTPUT_POLL_US): smoothed time from the event record to its completion, kept
// separately for each wait site (they wait for very different amounts of GPU work)
double publishWaitEmaUs = 0.0, evalWaitEmaUs = 0.0;
// native Vulkan input wait: smoothed time from recording to the input event (waiters may overlap)
std::atomic<double> vkInputWaitEmaUs{0.0};
bool profileEventsReady = false;
CudaImage color, depth, motion, exposure, output;
CudaObject colorHandle = 0, depthHandle = 0, motionHandle = 0, exposureHandle = 0, outputHandle = 0;
InputSlot inputs[kSlots];
OutputSlot outputs[kOutputSlots];
HostPlane outputHost[2]; // downloaded results waiting for publish_output
std::atomic<bool> outputHostBusy[2] = {false, false};
unsigned int nextOutputHost = 0; // worker only
// Overlapped staging (worker only): linear device buffers the async copies
// target, moved to/from the CUDA arrays with device-to-device copies.
CudaDevicePtr planeLinear[4] = {}, outputLinear = 0;
size_t planeLinearBytes[4] = {}, outputLinearBytes = 0;
// Frames queued but not yet published (or dropped). evaluate() blocks the
// game at D4R_SHIM_MAX_IN_FLIGHT (default 3) so a backlog cannot build up
// between pipeline stages and age the presented result.
std::atomic<int> inFlight{0};
DXGI_FORMAT outputFormat = DXGI_FORMAT_UNKNOWN;
UINT outputWidth = 0, outputHeight = 0;
// VRAM interop: decided on the first evaluation, dropped for good if the
// formats later stop qualifying.
bool vram = false, vramDecided = false;
VramImage colorConversion;
VramImage depthConversion;
VramImage motionConversion;
VramImage exposureConversion;
VramImage outputConversion;
// Split frames: the rest of frame N's command list is submitted only once
// splitSemaphore reaches N (all admitted frames through N have retired).
std::atomic<bool> split{false};
VkSemaphore splitSemaphore = VK_NULL_HANDLE;
std::mutex splitMutex;
uint64_t splitSignalled = 0; // under splitMutex
D4rFrameCompletion splitCompletion; // under splitMutex; cancellation can finish out of order
// Inputs become ready on the CPU before CUDA work is queued. GPU-side waits
// for recorded-but-discarded command lists can otherwise wedge the device.
// D4R_SHIM_OUTPUT_DIRECT: this frame's result is stored by the native output kernel straight into its
// destination buffer (worker only), so the array -> buffer copy is skipped.
bool outputRedirected = false;
// The previous frame's output kernel was the native one and honoured the redirect (worker writes, game
// thread reads): only then is a frame recorded for packed output.
std::atomic<bool> packedReady{false};
bool outputDirectAllowed = false; // the forced preset's output kernel honours the redirect
// D4R_SHIM_LINEAR_INPUTS: NGX samples the interop buffers directly through pitch-linear
// texture objects (rows padded to 256 bytes), so the buffer -> array copies disappear.
bool linearInputs = false;
bool dilationReported = false;
std::vector<VramBuffer> retiredBuffers; // may still be read by queued command lists
std::vector<VramImage> retiredImages;
ID3D12Resource* marker = nullptr;
volatile uint32_t* markerValue = nullptr; // [0] latest input; [1 + slot] exact input frame
uint32_t frame = 0;
uint32_t rejectedFormatCount = 0; // game thread; rate-limits format diagnostics
std::atomic<int> latestOutput{-1};
std::atomic<uint32_t> completedFrames{0};
uint32_t evaluatedFrames = 0; // worker only
// D4R_ENGINE: d4r's own inference, recorded into the game's command lists (game thread only). Recorded
// lists retain `resources`, so this is destroyed only once the GPU has finished with them.
uint64_t engineViewBytes = 0;
bool engineViewBudgetReported = false;
std::vector<std::unique_ptr<EngineBorrowedView>> engineViews; // destroyed after engine descriptor pools
std::unique_ptr<d4r::GameUpscaler> engine;
std::unique_ptr<d4r::GameUpscalerM> engineM; // preset M/L, original network
std::unique_ptr<d4r::GameUpscalerM> engineMDynamic; // Vulkan at changing sizes; original HIP resources stay alive
unsigned int engineMLastWidth = 0, engineMLastHeight = 0;
unsigned int engineMRenderWidth = 0, engineMRenderHeight = 0; // first evaluation's effective extent
// Preset M and preset K with the network on the native HIP layers ([Engine] Network): the buffers shared
// with HIP and the bridge's network. The game's list is split around the network, as for the CUDA backend's
// split frames. `engineHead` is the network's result (M's expansion / K's final store reads it).
VramBuffer engineTokens, engineHead;
// Input readiness for the HIP network: engineSync holds the frame number the engine's front writes (a Vulkan
// fill) and the network's stream waits for it on the GPU (engineGpuWait), so the layers start without a
// CPU round trip. engineWaitFrame: the frame a queued stream wait is waiting for (0: none).
VramBuffer engineSync;
std::atomic<bool> engineGpuWait{false};
bool engineNetTimeline = false; // native HIP callback and dropped-frame signals share one timeline lock
std::atomic<uint32_t> engineWaitFrame{0};
double engineNetWaitEmaUs = 0.0; // worker only: smoothed launch-to-completion wait, for adaptive polling
void* engineNet = nullptr;
bool engineRefused = false;
float engineJitter[2] = {0, 0}; // of the previous engine frame
// Serializes picking the slot to present (game thread) against picking
// the slot to overwrite (worker).
std::mutex outputMutex;
// Ray Reconstruction: feature 13, which the NGX core dispatches to the denoiser it
// loaded. Super Sampling features leave this false and differ only in their feature id
// and their preset names.
bool rayReconstruction = false;
// The denoiser's settings as the caller stated them at creation time. A setting the
// caller varies per frame must fall back to this, not to the previous frame's value:
// the parameter object outlives the frame, so an untouched one would silently carry
// over an override that is no longer being asked for.
int auxScalarBaseline[sizeof(kAuxScalars) / sizeof(kAuxScalars[0])] = {};
// The denoiser's alpha output. Its CUDA surface is float32 single channel: the vendor's
// transformer output kernel stores it as a nullable scalar float32 pixel
// (sust.p.2d.b32.zero at param+248), so that is what the surface is created as.
CudaImage alphaImage;
HostPlane alphaHost; // worker: the read-back float32 result
AlphaSlot alpha[kOutputSlots];
std::atomic<int> latestAlpha{-1}; // the slot holding the newest unpresented result
// A list, not a vector or a deque: the parameter object holds &AuxSurface.handle for the
// duration of the denoiser's evaluate, so every element must keep its address. A vector
// moves elements on growth and a deque invalidates *all* references when an element is
// erased from the middle - which is exactly what dropping a surface does while other
// surfaces are still queued. A list's erase touches only the element removed.
std::list<AuxSurface> aux; // the caller's registered surfaces, under auxMutex
// The game thread rewrites this list from the caller's parameter object while the worker
// reads it, and the parameter object holds &aux.handle for the duration of the
// denoiser's evaluate, so every aux access is under this lock.
std::mutex auxMutex;
};
static std::mutex g_featuresMutex;
static std::vector<Feature*> g_features;
static bool create_buffer(D3D12_HEAP_TYPE heapType, UINT64 size, ID3D12Resource** buffer, uint8_t** mapped)
{
D3D12_HEAP_PROPERTIES heap = {};
heap.Type = heapType;
D3D12_RESOURCE_DESC desc = {};
desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
desc.Width = size;
desc.Height = 1;
desc.DepthOrArraySize = 1;
desc.MipLevels = 1;
desc.SampleDesc.Count = 1;
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
const D3D12_RESOURCE_STATES state =
heapType == D3D12_HEAP_TYPE_READBACK ? D3D12_RESOURCE_STATE_COPY_DEST : D3D12_RESOURCE_STATE_GENERIC_READ;
HRESULT hr = g.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, state, nullptr,
__uuidof(ID3D12Resource), reinterpret_cast<void**>(buffer));
if (FAILED(hr))
{
logf("CreateCommittedResource(heap=%d, %llu bytes) failed: 0x%08lx", heapType,
static_cast<unsigned long long>(size), hr);
return false;
}
D3D12_RANGE readRange = {0, heapType == D3D12_HEAP_TYPE_READBACK ? static_cast<SIZE_T>(size) : 0};
hr = (*buffer)->Map(0, &readRange, reinterpret_cast<void**>(mapped));
if (FAILED(hr))
{
logf("Map failed: 0x%08lx", hr);
(*buffer)->Release();
*buffer = nullptr;
return false;
}
return true;
}
// Ensures staging matches the resource's first subresource; heap selects
// readback (inputs) or upload (output).
static bool ensure_staging(Staging& staging, ID3D12Resource* resource, D3D12_HEAP_TYPE heapType)
{
D3D12_RESOURCE_DESC desc;
resource->GetDesc(&desc);
if (staging.buffer != nullptr && staging.format == desc.Format && staging.width == desc.Width &&
staging.height == desc.Height)
return true;
staging.retire();
UINT64 rowSize = 0;
g.device->GetCopyableFootprints(&desc, 0, 1, 0, &staging.layout, &staging.rows, &rowSize, &staging.total);
if (!create_buffer(heapType, staging.total, &staging.buffer, &staging.mapped))
return false;
staging.format = desc.Format;
staging.width = static_cast<UINT>(desc.Width);
staging.height = desc.Height;
logf("staging %s %ux%u format=%d rowPitch=%u total=%llu", heapType == D3D12_HEAP_TYPE_READBACK ? "readback" : "upload",
staging.width, staging.height, staging.format, staging.layout.Footprint.RowPitch,
static_cast<unsigned long long>(staging.total));
return true;
}
static void transition(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES before,
D3D12_RESOURCE_STATES after)
{
if (before == after)
return;
D3D12_RESOURCE_BARRIER barrier = {};
barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barrier.Transition.pResource = resource;
barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
barrier.Transition.StateBefore = before;
barrier.Transition.StateAfter = after;
list->ResourceBarrier(1, &barrier);
}
static void copy_to_staging(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, Staging& staging,
D3D12_RESOURCE_STATES state)
{
transition(list, resource, state, D3D12_RESOURCE_STATE_COPY_SOURCE);
D3D12_TEXTURE_COPY_LOCATION destination = {};
destination.pResource = staging.buffer;
destination.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
destination.PlacedFootprint = staging.layout;
D3D12_TEXTURE_COPY_LOCATION source = {};
source.pResource = resource;
source.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
source.SubresourceIndex = 0;
list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
transition(list, resource, D3D12_RESOURCE_STATE_COPY_SOURCE, state);
}
// --- CUDA-side resources (worker thread) -----------------------------------------
// Sampler of an input texture: clamp addressing, normalized coordinates (as NGX's own textures); bilinear
// for the colour input (D4R_SHIM_COLOR_LINEAR), optionally for motion / depth (experiments).
static CudaTextureDesc input_sampler(UINT width, uint32_t channels, bool linear)
{
CudaTextureDesc sampler = {};
sampler.addressMode[0] = 1;
sampler.addressMode[1] = 1;
static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
sampler.filterMode = linear && colorLinear ? 1 : 0;
static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
sampler.filterMode = 1;
sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
return sampler;
}
static bool ensure_cuda_image(CudaImage& image, UINT width, UINT height, uint32_t format, uint32_t channels,
bool surface, bool linear)
{
if (image.array != nullptr && image.width == width && image.height == height &&
image.format == format && image.channels == channels)
return true;
if (image.object != 0)
surface ? g.cu.surfObjectDestroy(image.object) : g.cu.texObjectDestroy(image.object);
if (image.array != nullptr)
g.cu.arrayDestroy(image.array);
image = CudaImage{};
int result;
if (surface)
{
const CudaArray3DDescriptor descriptor{width, height, 0, format, channels, 2u /* SURFACE_LDST */};
result = g.cu.array3DCreate(&image.array, &descriptor);
}
else
{
const CudaArrayDescriptor descriptor{width, height, format, channels};
result = g.cu.arrayCreate(&image.array, &descriptor);
}
if (result != 0)
{
logf("CUDA array create %ux%u fmt=0x%x ch=%u failed: %d", width, height, format, channels, result);
return false;
}
CudaResourceDesc resource = {};
resource.resType = 0; // CU_RESOURCE_TYPE_ARRAY
resource.res.array.hArray = image.array;
if (surface)
result = g.cu.surfObjectCreate(&image.object, &resource);
else
{
// Matches the texture objects NGX itself creates: clamp addressing,
// normalized coordinates.
CudaTextureDesc sampler = {};
sampler.addressMode[0] = 1;
sampler.addressMode[1] = 1;
// D4R_SHIM_COLOR_LINEAR=0 samples the colour input with point filtering (experiment).
static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
sampler.filterMode = linear && colorLinear ? 1 : 0;
// D4R_SHIM_MOTION_LINEAR / D4R_SHIM_DEPTH_LINEAR=1: bilinear filtering for
// those planes (identified by format; experiment).
static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
sampler.filterMode = 1;
sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
result = g.cu.texObjectCreate(&image.object, &resource, &sampler, nullptr);
}
if (result != 0)
{
logf("CUDA %s object create failed: %d", surface ? "surface" : "texture", result);
return false;
}
image.width = width;
image.height = height;
image.format = format;
image.channels = channels;
return true;
}
// Device <-> array copy. Queued on the null stream, where NGX evaluates, when the bridge provides
// it (D4R_SHIM_ASYNC_2D=0: ZLUDA's cuMemcpy2D); callers synchronize before the data is used.
static int copy_2d(const CudaMemcpy2D& copy)
{
static const bool async = env_uint("D4R_SHIM_ASYNC_2D", 1) != 0;
if (async && g.cu.memcpy2DAsync != nullptr && g.cu.memcpy2DAsync(&copy, nullptr) == 0)
return 0;
return g.cu.memcpy2D(&copy);
}
// Splits row-wise staging work (tens of MB per frame at 1440p, which one core
// moves at ~11 GB/s) across helper threads plus the calling worker thread.
// D4R_SHIM_COPY_THREADS sets the total thread count (default 4, 1 = serial).
class RowPool
{
public:
void run(UINT rows, size_t bytes, const std::function<void(UINT, UINT)>& task)
{
const unsigned int parts = threads();
if (parts <= 1 || bytes < (2u << 20) || rows < parts * 8)
{
task(0, rows);
return;
}
{
std::lock_guard<std::mutex> lock(mutex_);
task_ = &task;
rows_ = rows;
parts_ = parts;
remaining_ = parts - 1;
++generation_;
}
start_.notify_all();
task(0, rows / parts);
std::unique_lock<std::mutex> lock(mutex_);
done_.wait(lock, [this] { return remaining_ == 0; });
task_ = nullptr;
}
private:
unsigned int threads()
{
std::call_once(once_, [this] {
count_ = env_uint("D4R_SHIM_COPY_THREADS", 4);
count_ = count_ < 1 ? 1 : (count_ > 16 ? 16 : count_);
for (unsigned int index = 1; index < count_; ++index)
std::thread([this, index] { helper(index); }).detach();
});
return count_;
}
void helper(unsigned int index)
{
uint64_t seen = 0;
for (;;)
{
const std::function<void(UINT, UINT)>* task;
UINT begin, end;
{
std::unique_lock<std::mutex> lock(mutex_);
start_.wait(lock, [&] { return generation_ != seen; });
seen = generation_;
task = task_;
begin = static_cast<UINT>(static_cast<uint64_t>(rows_) * index / parts_);
end = static_cast<UINT>(static_cast<uint64_t>(rows_) * (index + 1) / parts_);
}
(*task)(begin, end);
{
std::lock_guard<std::mutex> lock(mutex_);
--remaining_;
}
done_.notify_one();
}
}
std::once_flag once_;
unsigned int count_ = 1;
std::mutex mutex_;
std::condition_variable start_, done_;
const std::function<void(UINT, UINT)>* task_ = nullptr;
UINT rows_ = 0;
unsigned int parts_ = 1, remaining_ = 0;
uint64_t generation_ = 0;
};
// One pool per CPU pipeline stage; a pool serves one caller at a time.
static RowPool g_inputRows, g_outputRows;
// Copies rows between pitched buffers.
static void copy_rows(RowPool& pool, uint8_t* destination, size_t destinationPitch, const uint8_t* source,
size_t sourcePitch, size_t rowBytes, UINT rows)
{
pool.run(rows, rowBytes * rows, [&](UINT begin, UINT end) {
if (destinationPitch == rowBytes && sourcePitch == rowBytes)
std::memcpy(destination + begin * rowBytes, source + begin * rowBytes, rowBytes * (end - begin));
else
for (UINT y = begin; y < end; ++y)
std::memcpy(destination + y * destinationPitch, source + y * sourcePitch, rowBytes);
});
}
// Overlapped staging: frame N's uploads run while frame N-1's result downloads
// (PCIe is full duplex), as async copies between page-locked host memory and
// linear device buffers on two non-blocking streams. D4R_SHIM_ASYNC_COPIES=0
// keeps the synchronous host<->array copies.
static bool async_copies()
{
static const bool enabled = env_uint("D4R_SHIM_ASYNC_COPIES", 1) != 0;
return enabled && g.cu.memAllocHost != nullptr;
}
struct CopyStreams
{
void* up = nullptr;
void* down = nullptr;
};
// Worker thread only.
static CopyStreams* copy_streams()
{
static CopyStreams streams;
static bool attempted = false;
if (!attempted)
{
attempted = true;
constexpr unsigned int kNonBlocking = 1; // CU_STREAM_NON_BLOCKING
if (g.cu.streamCreate(&streams.up, kNonBlocking) != 0 || g.cu.streamCreate(&streams.down, kNonBlocking) != 0)
{
logf("cuStreamCreate failed; using synchronous copies");
streams = {};
}
}
return streams.up != nullptr ? &streams : nullptr;
}
// Grows a host plane's buffer. Page-locked memory is allocated on the CUDA
// worker thread (the only one with a current context).
static void reserve_host(HostPlane& host, size_t bytes)
{
if (bytes <= host.capacity)
return;
auto allocate = [&host, bytes] {
if (host.pinned)
g.cu.memFreeHost(host.bytes);
host = HostPlane{};
void* pinned = nullptr;
if (async_copies() && g.cu.memAllocHost(&pinned, bytes) == 0)
{
host.bytes = static_cast<uint8_t*>(pinned);
host.pinned = true;
}
else
{
host.fallback.resize(bytes);
host.bytes = host.fallback.data();
}
host.capacity = bytes;
return 0;
};
if (async_copies() && std::this_thread::get_id() != g.workerThread)
g.worker.call(allocate);
else
allocate();
}
static void release_host(HostPlane& host)
{
if (host.pinned)
g.cu.memFreeHost(host.bytes);
host = HostPlane{};
}
// Worker thread only.
static bool ensure_linear(CudaDevicePtr& buffer, size_t& capacity, size_t bytes)
{
if (bytes <= capacity)
return true;
if (buffer != 0)
g.cu.memFree(buffer);
buffer = 0;
capacity = 0;
if (g.cu.memAlloc(&buffer, bytes) != 0)
{
buffer = 0;
logf("cuMemAlloc(%zu) for staging failed", bytes);
return false;
}
capacity = bytes;
return true;
}
static uint32_t plane_format(Plane plane)
{
return (plane == Plane::Color || plane == Plane::Motion) ? CUDA_FORMAT_HALF : CUDA_FORMAT_FLOAT;
}
static uint32_t plane_channels(Plane plane)
{
return plane == Plane::Color ? 4 : (plane == Plane::Motion ? 2 : 1);
}
// --- VRAM interop ------------------------------------------------------------------
//
// With D4R_SHIM_VRAM_INTEROP=1 the inputs and the result never leave VRAM.
// vkd3d-proton's interop interface hands out the game's VkDevice, the VkImage
// behind each D3D12 texture, and the raw VkCommandBuffer behind the game's
// D3D12 command list. evaluate() records Vulkan copies of the inputs into
// exportable device-local buffers (colour formats other than RGBA16F are
// converted by a blit first), and of the latest result out of one; the Wine
// nvcuda bridge (d4rImportVulkanMemory) maps those buffers into CUDA. The
// worker then only moves data between them and the CUDA arrays on the GPU.
// Frames whose formats do not qualify use the readback/upload path.
struct VulkanInterop
{
std::once_flag once;
bool ready = false;
ID3D12DXVKInteropDevice1* interop = nullptr;
ID3D12DXVKInteropDeviceD4R2* lifetime = nullptr;
ID3D12DXVKInteropDeviceD4R* split = nullptr; // only with the d4r vkd3d-proton patch
ID3D12DXVKInteropDeviceD4R3* linear = nullptr; // experimental: linear exportable game textures
VkDevice device = VK_NULL_HANDLE;
VkPhysicalDeviceMemoryProperties memory = {};
VkPhysicalDevice physical = VK_NULL_HANDLE;
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
PFN_vkCreateBuffer createBuffer = nullptr;
PFN_vkDestroyBuffer destroyBuffer = nullptr;
PFN_vkGetBufferMemoryRequirements bufferRequirements = nullptr;
PFN_vkBindBufferMemory bindBuffer = nullptr;
PFN_vkCreateImage createImage = nullptr;
PFN_vkDestroyImage destroyImage = nullptr;
PFN_vkGetImageMemoryRequirements imageRequirements = nullptr;
PFN_vkBindImageMemory bindImage = nullptr;
PFN_vkAllocateMemory allocate = nullptr;
PFN_vkFreeMemory free = nullptr;
PFN_vkMapMemory map = nullptr;
PFN_vkUnmapMemory unmap = nullptr;
PFN_vkCmdPipelineBarrier barrier = nullptr;
PFN_vkCmdCopyImageToBuffer copyImageToBuffer = nullptr;
PFN_vkCmdCopyBufferToImage copyBufferToImage = nullptr;
PFN_vkCmdBlitImage blit = nullptr;
PFN_vkCmdFillBuffer fill = nullptr;
PFN_vkCreateSemaphore createSemaphore = nullptr;
PFN_vkDestroySemaphore destroySemaphore = nullptr;
PFN_vkSignalSemaphore signalSemaphore = nullptr;
int(WINAPI* import)(VkDevice, uint64_t, uint64_t, CudaDevicePtr*, void**) = nullptr;
int(WINAPI* release)(void*) = nullptr;
};
static VulkanInterop g_vk;
static bool vram_interop_requested()
{
static const bool requested = env_uint("D4R_SHIM_VRAM_INTEROP", 0) != 0;
return requested;
}
// Game thread, once.
static void init_vram_interop()
{
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDevice1), reinterpret_cast<void**>(&g_vk.interop))))
{
logf("VRAM interop: ID3D12DXVKInteropDevice1 unavailable (not vkd3d-proton?)");
return;
}
VkInstance instance = VK_NULL_HANDLE;
VkPhysicalDevice physical = VK_NULL_HANDLE;
if (FAILED(g_vk.interop->GetVulkanHandles(&instance, &physical, &g_vk.device)))
{
logf("VRAM interop: GetVulkanHandles failed");
return;
}
UINT count = 0;
g_vk.interop->GetDeviceExtensions(&count, nullptr);
std::vector<const char*> extensions(count);
g_vk.interop->GetDeviceExtensions(&count, extensions.data());
bool exportable = false;
for (const char* name : extensions)
exportable |= std::strcmp(name, "VK_KHR_external_memory_win32") == 0;
HMODULE vulkan = GetModuleHandleA("vulkan-1.dll");
if (vulkan == nullptr)
vulkan = LoadLibraryA("vulkan-1.dll");
PFN_vkGetInstanceProcAddr instanceProc = nullptr;
if (vulkan != nullptr)
load_export(vulkan, "vkGetInstanceProcAddr", instanceProc);
load_export(g.cuda, "d4rImportVulkanMemory", g_vk.import);
load_export(g.cuda, "d4rReleaseVulkanMemory", g_vk.release);
if (!exportable || instanceProc == nullptr || g_vk.import == nullptr || g_vk.release == nullptr)
{
logf("VRAM interop: unavailable (external_memory_win32 %d, vulkan-1 %p, bridge exports %d)", exportable,
reinterpret_cast<void*>(instanceProc), g_vk.import != nullptr && g_vk.release != nullptr);
return;
}
auto deviceProc = reinterpret_cast<PFN_vkGetDeviceProcAddr>(instanceProc(instance, "vkGetDeviceProcAddr"));
auto memoryProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(
instanceProc(instance, "vkGetPhysicalDeviceMemoryProperties"));
if (deviceProc == nullptr || memoryProperties == nullptr)
return;
memoryProperties(physical, &g_vk.memory);
g_vk.physical = physical;
g_vk.formatProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceFormatProperties>(
instanceProc(instance, "vkGetPhysicalDeviceFormatProperties"));
bool ok = g_vk.formatProperties != nullptr;
auto load = [&](auto& function, const char* name) {
function = reinterpret_cast<std::remove_reference_t<decltype(function)>>(deviceProc(g_vk.device, name));
ok &= function != nullptr;
};
load(g_vk.createBuffer, "vkCreateBuffer");
load(g_vk.destroyBuffer, "vkDestroyBuffer");
load(g_vk.bufferRequirements, "vkGetBufferMemoryRequirements");
load(g_vk.bindBuffer, "vkBindBufferMemory");
load(g_vk.createImage, "vkCreateImage");
load(g_vk.destroyImage, "vkDestroyImage");
load(g_vk.imageRequirements, "vkGetImageMemoryRequirements");
load(g_vk.bindImage, "vkBindImageMemory");
load(g_vk.allocate, "vkAllocateMemory");
load(g_vk.free, "vkFreeMemory");
load(g_vk.map, "vkMapMemory");
load(g_vk.unmap, "vkUnmapMemory");
load(g_vk.barrier, "vkCmdPipelineBarrier");
load(g_vk.copyImageToBuffer, "vkCmdCopyImageToBuffer");
load(g_vk.copyBufferToImage, "vkCmdCopyBufferToImage");
load(g_vk.blit, "vkCmdBlitImage");
g_vk.fill = reinterpret_cast<PFN_vkCmdFillBuffer>(deviceProc(g_vk.device, "vkCmdFillBuffer"));
load(g_vk.createSemaphore, "vkCreateSemaphore");
load(g_vk.destroySemaphore, "vkDestroySemaphore");
load(g_vk.signalSemaphore, "vkSignalSemaphore");
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R), reinterpret_cast<void**>(&g_vk.split))))
g_vk.split = nullptr;
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R2), reinterpret_cast<void**>(&g_vk.lifetime))))
g_vk.lifetime = nullptr;
if (env_uint("D4R_SHIM_INPLACE", 0) == 0 ||
FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R3), reinterpret_cast<void**>(&g_vk.linear))))
g_vk.linear = nullptr;
g_vk.ready = ok;
logf("VRAM interop: %s (VkDevice %p), split frames %s", ok ? "ready" : "missing Vulkan entry points",
static_cast<void*>(g_vk.device), g_vk.split != nullptr ? "available" : "unavailable (stock vkd3d-proton)");
}
static bool vram_interop_available()
{
if (!vram_interop_requested())
return false;
std::call_once(g_vk.once, init_vram_interop);
return g_vk.ready;
}
// Device-local memory; prefers memory the CPU cannot map (no BAR window).
static int device_local_type(uint32_t typeBits)
{
for (int pass = 0; pass < 2; ++pass)
for (uint32_t index = 0; index < g_vk.memory.memoryTypeCount; ++index)
{
const VkMemoryPropertyFlags flags = g_vk.memory.memoryTypes[index].propertyFlags;
if ((typeBits & (1u << index)) && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
(pass == 1 || !(flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)))
return static_cast<int>(index);
}
return -1;
}
// Buffers of released features, reused by later ones. ROCm up to at least 7.2.4 never frees memory mapped with
// hipExternalMemoryGetMappedBuffer (the mapping's buffer view is retained twice but released once by hipFree), so
// destroying a buffer would leak its memory: each DLSS quality change (a new feature) would add ~100 MB of VRAM.
static std::mutex g_vramPoolMutex;
static std::vector<VramBuffer> g_vramPool;
constexpr size_t kVramPoolLimit = 64;
// The smallest pooled buffer of `kind` that holds `bytes`.
static bool take_pooled_vram_buffer(VramBuffer& target, size_t bytes, uint8_t kind = 0)
{
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
auto best = g_vramPool.end();
for (auto it = g_vramPool.begin(); it != g_vramPool.end(); ++it)
if (it->kind == kind && it->bytes >= bytes && (best == g_vramPool.end() || it->bytes < best->bytes))
best = it;
if (best == g_vramPool.end())
return false;
target = *best;
g_vramPool.erase(best);
return true;
}
// Game thread. The CUDA import runs on the worker, which owns the context.
static bool create_vram_buffer(VramBuffer& target, size_t bytes)
{
if (take_pooled_vram_buffer(target, bytes))
return true;
VramBuffer buffer;
VkExternalMemoryBufferCreateInfo external = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
info.size = bytes;
info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
VkResult result = g_vk.createBuffer(g_vk.device, &info, nullptr, &buffer.buffer);
VkMemoryRequirements requirements = {};
if (result == VK_SUCCESS)
g_vk.bufferRequirements(g_vk.device, buffer.buffer, &requirements);
const int type = device_local_type(requirements.memoryTypeBits);
VkMemoryDedicatedAllocateInfo dedicated = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
dedicated.buffer = buffer.buffer;
VkExportMemoryAllocateInfo exportInfo = {VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, &dedicated};
exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = static_cast<uint32_t>(type);
if (result == VK_SUCCESS && type < 0)
result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
if (result == VK_SUCCESS)
result = g_vk.allocate(g_vk.device, &allocation, nullptr, &buffer.memory);
if (result == VK_SUCCESS)
result = g_vk.bindBuffer(g_vk.device, buffer.buffer, buffer.memory, 0);
int imported = -1;
if (result == VK_SUCCESS)
imported = g.worker.call([&] {
return g_vk.import(g_vk.device, reinterpret_cast<uint64_t>(buffer.memory), requirements.size,
&buffer.device, &buffer.external);
});
if (result != VK_SUCCESS || imported != 0)
{
logf("VRAM interop: exportable buffer of %zu bytes failed (VkResult %d, import %d)", bytes, result, imported);
if (buffer.memory != VK_NULL_HANDLE)
g_vk.free(g_vk.device, buffer.memory, nullptr);
if (buffer.buffer != VK_NULL_HANDLE)
g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
return false;
}
buffer.bytes = bytes;
target = buffer;
return true;
}
// Only once no command list or CUDA work uses the buffer.
static void destroy_vram_buffer(VramBuffer& buffer)
{
if (buffer.external != nullptr)
g.worker.call([&buffer] {
const int freed = g.cu.memFree(buffer.device);
const int released = g_vk.release(buffer.external);
if (freed != 0 || released != 0)
logf("VRAM interop: freeing %zu-byte buffer: cuMemFree %d, release %d", buffer.bytes, freed, released);
return released;
});
if (buffer.mapped != nullptr)
g_vk.unmap(g_vk.device, buffer.memory);
if (buffer.memory != VK_NULL_HANDLE)
g_vk.free(g_vk.device, buffer.memory, nullptr);
if (buffer.buffer != VK_NULL_HANDLE)
g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
buffer = VramBuffer{};
}
// Only once no command list or CUDA work uses the buffer: keeps it for a later feature (see g_vramPool).
static void recycle_vram_buffer(VramBuffer& buffer)
{
if (buffer.buffer == VK_NULL_HANDLE)
return;
{
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
if (g_vramPool.size() < kVramPoolLimit)
{
g_vramPool.push_back(buffer);
buffer = VramBuffer{};
return;
}
}
destroy_vram_buffer(buffer);
}
static void free_vram_pool()
{
std::vector<VramBuffer> pool;
{
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
pool.swap(g_vramPool);
}
for (VramBuffer& buffer : pool)
destroy_vram_buffer(buffer);
}
static void destroy_vram_image(VramImage& image)
{
if (image.memory != VK_NULL_HANDLE)
g_vk.free(g_vk.device, image.memory, nullptr);
if (image.image != VK_NULL_HANDLE)
g_vk.destroyImage(g_vk.device, image.image, nullptr);
image = VramImage{};
}
static bool create_vram_image(VramImage& image, UINT width, UINT height, VkFormat format)
{
VkImageCreateInfo info = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.imageType = VK_IMAGE_TYPE_2D;
info.format = format;
info.extent = {width, height, 1};
info.mipLevels = 1;
info.arrayLayers = 1;
info.samples = VK_SAMPLE_COUNT_1_BIT;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
VkResult result = g_vk.createImage(g_vk.device, &info, nullptr, &image.image);
VkMemoryRequirements requirements = {};
if (result == VK_SUCCESS)
g_vk.imageRequirements(g_vk.device, image.image, &requirements);
const int type = device_local_type(requirements.memoryTypeBits);
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = static_cast<uint32_t>(type);
if (result == VK_SUCCESS && type < 0)
result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
if (result == VK_SUCCESS)
result = g_vk.allocate(g_vk.device, &allocation, nullptr, &image.memory);
if (result == VK_SUCCESS)
result = g_vk.bindImage(g_vk.device, image.image, image.memory, 0);
if (result != VK_SUCCESS)
{
logf("VRAM interop: %ux%u conversion image failed: %d", width, height, result);
destroy_vram_image(image);
return false;
}
image.width = width;
image.height = height;
image.format = format;
return true;
}
static size_t canonical_texel_bytes(Plane plane)
{
return (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
}
static size_t vk_texel_bytes(VkFormat format)
{
switch (format)
{
case VK_FORMAT_R16G16B16A16_SFLOAT:
case VK_FORMAT_R16G16B16A16_UINT:
case VK_FORMAT_R16G16B16A16_SINT:
case VK_FORMAT_R16G16B16A16_UNORM:
return 8;
case VK_FORMAT_R32_SFLOAT:
case VK_FORMAT_R32_UINT:
case VK_FORMAT_R32_SINT:
case VK_FORMAT_R16G16_SFLOAT:
case VK_FORMAT_R16G16_UINT:
case VK_FORMAT_R16G16_SINT:
case VK_FORMAT_D32_SFLOAT:
case VK_FORMAT_D32_SFLOAT_S8_UINT: // the depth aspect alone
return 4;
case VK_FORMAT_R16_SFLOAT:
case VK_FORMAT_R16_UNORM:
case VK_FORMAT_R16_UINT:
return 2;
default:
return 0;
}
}
// D4R_SHIM_INPLACE (experimental): a game texture that vkd3d-proton created linear and exportable is imported
// into CUDA once, so DLSS writes its texels without a game-side copy. Imports and their source resources
// stay alive for the process lifetime (ROCm does not free mapped external memory, see g_vramPool).
struct InPlaceTexture
{
CudaDevicePtr texels = 0;
size_t rowPitch = 0;
};
struct InPlaceImport
{
uint64_t memory;
CudaDevicePtr base;
ID3D12Resource* owner; // retained: prevents reuse of the cached VkDeviceMemory handle
};
static std::mutex g_inPlaceMutex;
static std::vector<InPlaceImport> g_inPlaceImports;
// Game thread.
static bool describe_in_place(ID3D12Resource* resource, InPlaceTexture& texture)
{
UINT64 memory = 0, size = 0, offset = 0, pitch = 0;
if (g_vk.linear == nullptr || FAILED(g_vk.linear->GetVulkanLinearImageInfo(resource, &memory, &size, &offset, &pitch)))
return false;
CudaDevicePtr base = 0;
{
std::lock_guard<std::mutex> lock(g_inPlaceMutex);
for (const auto& entry : g_inPlaceImports)
if (entry.memory == memory)
base = entry.base;
}
if (base == 0)
{
void* external = nullptr;
const int imported = g.worker.call([&] {
return g_vk.import(g_vk.device, memory, size, &base, &external);
});
if (imported != 0 || base == 0)
{
logf("in-place: importing %llu bytes of texture memory failed (%d)", static_cast<unsigned long long>(size), imported);
return false;
}
logf("in-place: imported %llu bytes of game texture memory (offset %llu, row pitch %llu)",
static_cast<unsigned long long>(size), static_cast<unsigned long long>(offset),
static_cast<unsigned long long>(pitch));
std::lock_guard<std::mutex> lock(g_inPlaceMutex);
resource->AddRef();
g_inPlaceImports.push_back({memory, base, resource});
}
texture.texels = base + offset;
texture.rowPitch = static_cast<size_t>(pitch);
return true;
}
// How one input or the output moves between its D3D12 texture and a buffer.
struct VramCopy
{
ID3D12Resource* resource = nullptr;
VkImage image = VK_NULL_HANDLE;
VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT;
bool convert = false; // blit to/from the plane's canonical format
UINT width = 0, height = 0;
};
static bool vram_blit_supported(VkFormat source, VkFormat destination)
{
VkFormatProperties sourceProperties = {}, destinationProperties = {};
g_vk.formatProperties(g_vk.physical, source, &sourceProperties);
g_vk.formatProperties(g_vk.physical, destination, &destinationProperties);
return (sourceProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) != 0 &&
(destinationProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_DST_BIT) != 0;
}
// sRGB is deliberately excluded: the host staging path treats its bytes as
// UNORM and a Vulkan blit would apply an sRGB transfer function instead.
static bool vram_color_blit_supported(VkFormat format, bool output)
{
switch (format)
{
case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
case VK_FORMAT_R8G8B8A8_UNORM:
case VK_FORMAT_B8G8R8A8_UNORM:
case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
break;
default:
return false;
}
return output ? vram_blit_supported(VK_FORMAT_R16G16B16A16_SFLOAT, format)
: vram_blit_supported(format, VK_FORMAT_R16G16B16A16_SFLOAT);
}
// The host path takes only the first (R) component of an exposure texel.
static bool vram_exposure_blit_supported(VkFormat format)
{
switch (format)
{
case VK_FORMAT_R16_SFLOAT:
case VK_FORMAT_R16G16B16A16_SFLOAT:
case VK_FORMAT_R32G32B32A32_SFLOAT:
return vram_blit_supported(format, VK_FORMAT_R32_SFLOAT);
default:
return false;
}
}
// Depth in a 16-bit format is widened to R32F by a blit (UNORM normalizes, SFLOAT converts).
static bool vram_depth_blit_supported(VkFormat format)
{
switch (format)
{
case VK_FORMAT_R16_SFLOAT:
case VK_FORMAT_R16_UNORM:
return vram_blit_supported(format, VK_FORMAT_R32_SFLOAT);
default:
return false;
}
}
// Motion vectors in a wider float format keep their first two components (as the host path does).
static bool vram_motion_blit_supported(VkFormat format)
{
switch (format)
{
case VK_FORMAT_R16G16B16A16_SFLOAT:
case VK_FORMAT_R32G32_SFLOAT:
case VK_FORMAT_R32G32B32A32_SFLOAT:
case VK_FORMAT_R8G8B8A8_UNORM:
return vram_blit_supported(format, VK_FORMAT_R16G16_SFLOAT);
default:
return false;
}
}
// Whether `resource` can be copied raw (or, for colour/motion/exposure, blitted)
// to/from the canonical layout of `plane`.
static bool describe_vram_copy(ID3D12Resource* resource, Plane plane, VramCopy& copy, bool output = false)
{
UINT64 handle = 0, offset = 0;
VkFormat format = VK_FORMAT_UNDEFINED;
if (FAILED(g_vk.interop->GetVulkanResourceInfo1(resource, &handle, &offset, &format)) || handle == 0)
return false;
D3D12_RESOURCE_DESC desc;
resource->GetDesc(&desc);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1)
return false;
copy.resource = resource;
copy.image = reinterpret_cast<VkImage>(handle);
copy.width = static_cast<UINT>(desc.Width);
copy.height = desc.Height;
copy.aspect = (format == VK_FORMAT_D32_SFLOAT || format == VK_FORMAT_D32_SFLOAT_S8_UINT)
? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
if (plane == Plane::Color)
{
copy.convert = vram_color_blit_supported(format, output);
return copy.convert || format == VK_FORMAT_R16G16B16A16_SFLOAT;
}
if (plane == Plane::Exposure)
copy.convert = vram_exposure_blit_supported(format);
else if (plane == Plane::Motion)
copy.convert = vram_motion_blit_supported(format);
else if (plane == Plane::Depth)
copy.convert = vram_depth_blit_supported(format);
return copy.convert || vk_texel_bytes(format) == canonical_texel_bytes(plane);
}
static const VkMemoryBarrier kBeforeTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_MEMORY_WRITE_BIT,
VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT};
static const VkMemoryBarrier kAfterTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT};
static VkBufferImageCopy full_region(const VramCopy& copy)
{
VkBufferImageCopy region = {};
region.imageSubresource.aspectMask = copy.aspect;
region.imageSubresource.layerCount = 1;
region.imageExtent = {copy.width, copy.height, 1};
return region;
}
// Records the input copies into the slot's buffers. The resources are in
// COPY_SOURCE state already.
static bool record_vram_inputs(Feature& feature, ID3D12GraphicsCommandList* list, InputSlot& slot,
const VramCopy* copies, int count)
{
VkCommandBuffer cmd = VK_NULL_HANDLE;
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
return false;
g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
nullptr, 0, nullptr);
for (int index = 0; index < count; ++index)
{
const VramCopy& copy = copies[index];
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, &layout);
VkBufferImageCopy region = full_region(copy);
static const Plane kPlanes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
if (feature.linearInputs)
region.bufferRowLength = static_cast<uint32_t>(slot.host[index].rowBytes / canonical_texel_bytes(kPlanes[index]));
if (!copy.convert)
{
g_vk.copyImageToBuffer(cmd, copy.image, layout, slot.vram[index].buffer, 1, &region);
continue;
}
// Convert colour to RGBA16F, motion to RG16F or exposure's first component
// to R32F, then copy from the canonical image into the shared buffer.
VkImage conversion = kPlanes[index] == Plane::Exposure ? feature.exposureConversion.image
: kPlanes[index] == Plane::Motion ? feature.motionConversion.image
: kPlanes[index] == Plane::Depth ? feature.depthConversion.image
: feature.colorConversion.image;
VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toDestination.image = conversion;
toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
&toDestination);
VkImageBlit blit = {};
blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
blit.dstSubresource = blit.srcSubresource;
blit.dstOffsets[1] = blit.srcOffsets[1];
g_vk.blit(cmd, copy.image, layout, conversion, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit,
VK_FILTER_NEAREST);
VkImageMemoryBarrier toSource = toDestination;
toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
&toSource);
g_vk.copyImageToBuffer(cmd, conversion, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, slot.vram[index].buffer, 1,
&region);
}
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
nullptr, 0, nullptr);
return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
}
// Records the copy of a finished result into the output texture, which is in
// COPY_DEST state already.
static bool record_vram_output(ID3D12GraphicsCommandList* list, const VramBuffer& buffer, const VramCopy& copy,
const VramImage& conversion)
{
VkCommandBuffer cmd = VK_NULL_HANDLE;
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
return false;
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_DEST, &layout);
const VkBufferImageCopy region = full_region(copy);
g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
nullptr, 0, nullptr);
if (!copy.convert)
g_vk.copyBufferToImage(cmd, buffer.buffer, copy.image, layout, 1, &region);
else
{
VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
toDestination.image = conversion.image;
toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
&toDestination);
g_vk.copyBufferToImage(cmd, buffer.buffer, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
VkImageMemoryBarrier toSource = toDestination;
toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
&toSource);
VkImageBlit blit = {};
blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
blit.dstSubresource = blit.srcSubresource;
blit.dstOffsets[1] = blit.srcOffsets[1];
g_vk.blit(cmd, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, copy.image, layout, 1, &blit,
VK_FILTER_NEAREST);
}
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
nullptr, 0, nullptr);
return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
}
// --- Split frames -------------------------------------------------------------------
//
// To present frame N's own result, everything the game records after its DLSS
// call must wait for DLSS. With the d4r vkd3d-proton patch, evaluate() splits
// the game's command list right after the input copies and the frame marker;
// vkd3d-proton submits the second half separately, waiting (at queue submission
// level, so no GPU ring is ever blocked) for splitSemaphore to reach N. The
// worker writes N's result into output slot N % kOutputSlots. Completed and
// dropped frames advance the signal only past earlier retired frames. The
// watchdog reports delays without releasing an unfinished producer.
static VkResult signal_split_semaphore(Feature& feature, uint64_t value);
static void release_split_frame(Feature* feature, uint32_t frame)
{
std::lock_guard<std::mutex> lock(feature->splitMutex);
const uint64_t value = feature->splitCompletion.retire(frame);
if (value <= feature->splitSignalled)
return;
const VkResult result = signal_split_semaphore(*feature, value);
if (result == VK_SUCCESS)
feature->splitSignalled = value;
if (result != VK_SUCCESS)
logf("split frame %llu signal failed (VkResult %d)", static_cast<unsigned long long>(value), result);
}
static void split_watchdog()
{
const auto timeout = std::chrono::milliseconds(env_uint("D4R_SHIM_SPLIT_TIMEOUT_MS", 200));
struct Seen
{
uint32_t marker = 0;
std::chrono::steady_clock::time_point at;
};
std::vector<std::pair<Feature*, Seen>> seen;
for (;;)
{
std::this_thread::sleep_for(std::chrono::milliseconds(5));
std::lock_guard<std::mutex> lock(g_featuresMutex);
const auto now = std::chrono::steady_clock::now();
for (Feature* feature : g_features)
{
if (!feature->split || feature->markerValue == nullptr)
continue;
auto it = std::find_if(seen.begin(), seen.end(), [feature](const auto& entry) { return entry.first == feature; });
if (it == seen.end())
it = seen.insert(seen.end(), {feature, Seen{}});
const uint32_t marker = *feature->markerValue;
uint64_t signalled;
{
std::lock_guard<std::mutex> splitLock(feature->splitMutex);
signalled = feature->splitSignalled;
}
if (marker <= signalled)
continue; // nothing on the GPU is waiting on us
if (it->second.marker != marker)
it->second = Seen{marker, now};
else if (now - it->second.at > timeout)
{
// A slow producer is not a completed producer. Releasing this
// wait would race the game's output copy with a late CUDA write.
logf("split frame %u still waiting for DLSS; preserving output synchronization", marker);
it->second.at = now;
}
}
// Forget released features.
seen.erase(std::remove_if(seen.begin(), seen.end(),
[](const auto& entry) {
return std::find(g_features.begin(), g_features.end(), entry.first) ==
g_features.end();
}),
seen.end());
}
}
static bool create_split_semaphore(Feature& feature)
{
VkSemaphoreTypeCreateInfo type = {VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
type.initialValue = 0;
const VkSemaphoreCreateInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type};
if (g_vk.createSemaphore(g_vk.device, &info, nullptr, &feature.splitSemaphore) != VK_SUCCESS)
{
logf("split frames: vkCreateSemaphore failed");
return false;
}
static std::once_flag watchdog;
std::call_once(watchdog, [] { std::thread(split_watchdog).detach(); });
return true;
}
// Game thread: grows a buffer; an outgrown one is kept until the feature is
// released since queued command lists may still reference it.
static bool ensure_vram_buffer(Feature& feature, VramBuffer& buffer, size_t bytes)
{
if (buffer.buffer != VK_NULL_HANDLE && buffer.bytes >= bytes)
return true;
if (buffer.buffer != VK_NULL_HANDLE)
feature.retiredBuffers.push_back(buffer);
buffer = VramBuffer{};
return create_vram_buffer(buffer, bytes);
}
static bool ensure_conversion_image(Feature& feature, VramImage& image, UINT width, UINT height, VkFormat format)
{
if (image.image != VK_NULL_HANDLE && image.width == width && image.height == height && image.format == format)
return true;
if (image.image != VK_NULL_HANDLE)
feature.retiredImages.push_back(image);
image = VramImage{};
return create_vram_image(image, width, height, format);
}
static void release_vram(Feature& feature)
{
if (feature.splitSemaphore != VK_NULL_HANDLE)
g_vk.destroySemaphore(g_vk.device, feature.splitSemaphore, nullptr);
for (InputSlot& slot : feature.inputs)
for (VramBuffer& buffer : slot.vram)
recycle_vram_buffer(buffer);
for (OutputSlot& slot : feature.outputs)
recycle_vram_buffer(slot.vram);
for (VramBuffer& buffer : feature.retiredBuffers)
recycle_vram_buffer(buffer);
feature.retiredBuffers.clear();
destroy_vram_image(feature.colorConversion);
destroy_vram_image(feature.depthConversion);
destroy_vram_image(feature.motionConversion);
destroy_vram_image(feature.exposureConversion);
destroy_vram_image(feature.outputConversion);
for (VramImage& image : feature.retiredImages)
destroy_vram_image(image);
feature.retiredImages.clear();
}
// Prep stage: converts one readback staging buffer into the canonical layout.
static void stage_plane(Plane plane, const Staging& staging, HostPlane& host, FrameTiming* timing, int timingIndex)
{
const size_t texelBytes = (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
reserve_host(host, texelBytes * staging.width * staging.height);
host.width = staging.width;
host.height = staging.height;
host.rowBytes = texelBytes * staging.width;
uint8_t* rows = host.bytes;
const size_t rowBytes = host.rowBytes;
const uint8_t* source = staging.mapped + staging.layout.Offset;
const size_t sourcePitch = staging.layout.Footprint.RowPitch;
const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
if (canonical_input(plane, staging.format))
copy_rows(g_inputRows, rows, rowBytes, source, sourcePitch, rowBytes, staging.height);
else
g_inputRows.run(staging.height, rowBytes * staging.height, [&](UINT begin, UINT end) {
for (UINT y = begin; y < end; ++y)
if (plane == Plane::Color && staging.format == DXGI_FORMAT_R11G11B10_FLOAT)
convert_r11g11b10_row(source + y * sourcePitch,
reinterpret_cast<uint16_t*>(rows + rowBytes * y), staging.width);
else
convert_row_in(plane, staging.format, source + y * sourcePitch, rows + rowBytes * y,
staging.width);
});
if (timing != nullptr)
timing->convert[timingIndex] = profile_ms(conversionStart, ProfileClock::now());
}
// Worker stage: uploads one staged plane into its CUDA array.
// D4R_SHIM_MV_STATS=N logs, every N frames, what the game's motion vectors
// look like (staged RG16F plane): how much of the texture is populated, the
// populated bounding box (a render-size sub-rectangle means render-resolution
// vectors in a display-size texture), magnitudes in texture units, and the
// global image shift between the previous and current colour frames measured
// in render pixels, so the vectors' unit can be read off their ratio.
static std::vector<float> downsampled_luma(const HostPlane& color, UINT& width, UINT& height)
{
// RGBA16F colour, 2x2 box-downsampled luma.
width = color.width / 2, height = color.height / 2;
std::vector<float> luma(static_cast<size_t>(width) * height);
const uint8_t* bytes = color.bytes != nullptr ? color.bytes : color.fallback.data();
for (UINT y = 0; y < height; ++y)
for (UINT x = 0; x < width; ++x)
{
float sum = 0.0f;
for (UINT dy = 0; dy < 2; ++dy)
{
const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + (2 * y + dy) * color.rowBytes);
for (UINT dx = 0; dx < 2; ++dx)
for (UINT c = 0; c < 3; ++c)
sum += static_cast<float>(std::bit_cast<_Float16>(row[(2 * x + dx) * 4 + c]));
}
luma[static_cast<size_t>(y) * width + x] = sum / (sum + 12.0f); // tone-mapped (HDR input)
}
return luma;
}
static void log_motion_stats(const HostPlane& host, const HostPlane& color, uint32_t frame)
{
static const uint32_t every = env_uint("D4R_SHIM_MV_STATS", 0);
static std::vector<float> previousLuma;
static uint32_t previousFrame = 0;
if (every == 0 || host.width == 0 || color.width == 0)
return;
const bool report = frame % every == 0;
if (!report && (frame + 1) % every != 0)
return;
UINT lumaWidth = 0, lumaHeight = 0;
std::vector<float> luma = downsampled_luma(color, lumaWidth, lumaHeight);
if (!report)
{
previousLuma = std::move(luma);
previousFrame = frame;
return;
}
const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
size_t nonzero = 0;
double sumMagnitude = 0.0;
float maxMagnitude = 0.0f;
UINT minX = host.width, minY = host.height, maxX = 0, maxY = 0;
std::vector<float> xs, ys;
for (UINT y = 0; y < host.height; ++y)
{
const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + y * host.rowBytes);
for (UINT x = 0; x < host.width; ++x)
{
const float mx = static_cast<float>(std::bit_cast<_Float16>(row[x * 2]));
const float my = static_cast<float>(std::bit_cast<_Float16>(row[x * 2 + 1]));
if ((x % 8) == 0 && (y % 8) == 0)
{
xs.push_back(mx);
ys.push_back(my);
}
if (mx == 0.0f && my == 0.0f)
continue;
++nonzero;
const float magnitude = std::sqrt(mx * mx + my * my);
sumMagnitude += magnitude;
maxMagnitude = std::max(maxMagnitude, magnitude);
minX = std::min(minX, x), maxX = std::max(maxX, x);
minY = std::min(minY, y), maxY = std::max(maxY, y);
}
}
auto median = [](std::vector<float>& values) {
if (values.empty())
return 0.0f;
std::nth_element(values.begin(), values.begin() + values.size() / 2, values.end());
return values[values.size() / 2];
};
const float medianX = median(xs), medianY = median(ys);
// Global shift d (render pixels, current -> previous, the vectors'
// convention) minimising |cur(p) - prev(p + d)| over the central region.
int bestX = 0, bestY = 0;
double bestCost = -1.0, zeroCost = 0.0;
if (previousFrame + 1 == frame && previousLuma.size() == luma.size())
{
const int range = 80, rangeY = 40; // in downsampled pixels
const int x0 = range, x1 = static_cast<int>(lumaWidth) - range;
const int y0 = rangeY, y1 = static_cast<int>(lumaHeight) - rangeY;
for (int dy = -rangeY; dy <= rangeY; ++dy)
for (int dx = -range; dx <= range; ++dx)
{
double cost = 0.0;
for (int y = y0; y < y1; y += 2)
for (int x = x0; x < x1; x += 2)
cost += std::fabs(luma[static_cast<size_t>(y) * lumaWidth + x] -
previousLuma[static_cast<size_t>(y + dy) * lumaWidth + (x + dx)]);
if (dx == 0 && dy == 0)
zeroCost = cost;
if (bestCost < 0.0 || cost < bestCost)
bestCost = cost, bestX = dx, bestY = dy;
}
}
const double pixels = static_cast<double>(host.width) * host.height;
logf("MV_STATS frame=%u size=%ux%u nonzero=%.4f bbox=%u,%u-%u,%u median=(%.3f,%.3f) mean_abs=%.3f max=%.3f "
"image_shift_render_px=(%d,%d) match=%.3f",
frame, host.width, host.height, nonzero / pixels, nonzero ? minX : 0, nonzero ? minY : 0, maxX, maxY,
medianX, medianY, nonzero ? sumMagnitude / nonzero : 0.0, maxMagnitude, 2 * bestX, 2 * bestY,
zeroCost > 0.0 ? bestCost / zeroCost : 1.0);
}
static bool upload_plane(Plane plane, const HostPlane& host, CudaImage& image, uint32_t frame, FrameTiming* timing,
int timingIndex)
{
if (!ensure_cuda_image(image, host.width, host.height, plane_format(plane), plane_channels(plane), false,
plane == Plane::Color))
return false;
if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
{
uint64_t hash = 1469598103934665603ull;
for (size_t index = 0; index < host.size(); ++index)
hash = (hash ^ host.bytes[index]) * 1099511628211ull;
logf("frame %u upload plane=%u array=%p object=0x%llx bytes=%zu hash=%016llx", frame,
static_cast<unsigned int>(plane), image.array,
static_cast<unsigned long long>(image.object), host.size(),
static_cast<unsigned long long>(hash));
}
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_HOST;
copy.srcHost = host.bytes;
copy.srcPitch = host.rowBytes;
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
copy.dstArray = image.array;
copy.WidthInBytes = host.rowBytes;
copy.Height = host.height;
const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
const int result = g.cu.memcpy2D(&copy);
if (timing != nullptr)
timing->upload[timingIndex] = profile_ms(copyStart, ProfileClock::now());
if (result != 0)
logf("cuMemcpy2D upload failed: %d", result);
if (result != 0)
return false;
if (GetEnvironmentVariableA("D4R_SHIM_SYNC_UPLOAD", nullptr, 0) != 0)
{
const int syncResult = g.cu.ctxSynchronize();
if (syncResult != 0)
logf("cuCtxSynchronize after upload failed: %d", syncResult);
return syncResult == 0;
}
return true;
}
static void log_output_hash(const HostPlane& host, uint32_t frame)
{
if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) == 0)
return;
uint64_t hash = 1469598103934665603ull;
for (size_t index = 0; index < host.size(); ++index)
hash = (hash ^ host.bytes[index]) * 1099511628211ull;
logf("frame %u output hash=%016llx", frame, static_cast<unsigned long long>(hash));
}
static std::atomic<uint32_t> g_captureFirst{0}; // first frame of a triggered capture
static bool dump_frame_selected(uint32_t frame, const char* startName, const char* countName, const char* everyName)
{
char trigger[MAX_PATH];
const DWORD triggerLength = GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH);
if (triggerLength > 0 && triggerLength < MAX_PATH)
{
uint32_t captured = g_captureFirst.load();
if (captured == 0 && GetFileAttributesA(trigger) != INVALID_FILE_ATTRIBUTES)
{
uint32_t expected = 0;
if (g_captureFirst.compare_exchange_strong(expected, frame))
logf("raw input/output capture triggered on frame %u", frame);
captured = g_captureFirst.load();
}
// D4R_SHIM_CAPTURE_COUNT=N keeps capturing N consecutive frames (replay sequences).
static const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
return captured != 0 && frame >= captured && frame - captured < count;
}
const uint32_t start = env_uint(startName, 1);
const uint32_t count = env_uint(countName, 1);
const uint32_t every = std::max(1u, env_uint(everyName, 1));
return frame >= start && count != 0 && (frame - start) / every < count && (frame - start) % every == 0;
}
static bool output_dump_path(uint32_t frame, char (&path)[MAX_PATH])
{
char directory[MAX_PATH];
const DWORD length = GetEnvironmentVariableA("D4R_SHIM_OUTPUT_DUMP_DIR", directory, MAX_PATH);
if (length == 0 || length >= MAX_PATH)
return false;
if (!dump_frame_selected(frame, "D4R_SHIM_OUTPUT_DUMP_START", "D4R_SHIM_OUTPUT_DUMP_COUNT",
"D4R_SHIM_OUTPUT_DUMP_EVERY"))
return false;
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u.rgba16f", directory, frame);
return written > 0 && written < MAX_PATH;
}
// Capture files. With D4R_SHIM_CAPTURE_MEMORY=1 they are kept in memory and
// written by a background thread once the triggered capture is complete, so a
// sequence can be recorded at the game's normal frame rate.
static std::mutex g_captureMutex;
static std::vector<std::pair<std::string, std::vector<uint8_t>>> g_captureFiles;
// Pinned staging for memory captures (D4R_SHIM_CAPTURE_POOL_MB, default 3072):
// device readbacks land in it directly, so a capture costs a few ms per frame.
struct CapturePool
{
uint8_t* base = nullptr;
size_t size = 0, used = 0;
bool tried = false;
};
static CapturePool g_capturePool;
static std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> g_capturePinned;
// Worker thread (a CUDA context is current). Returns nullptr when memory
// captures are off or the pool is exhausted; the caller then uses ordinary memory.
static uint8_t* capture_pinned(const char* path, size_t bytes)
{
if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 || g.cu.memAllocHost == nullptr)
return nullptr;
std::lock_guard<std::mutex> lock(g_captureMutex);
CapturePool& pool = g_capturePool;
if (!pool.tried)
{
pool.tried = true;
for (size_t mb = env_uint("D4R_SHIM_CAPTURE_POOL_MB", 3072); mb >= 256 && pool.base == nullptr; mb /= 2)
{
void* memory = nullptr;
if (g.cu.memAllocHost(&memory, mb << 20) == 0)
pool.base = static_cast<uint8_t*>(memory), pool.size = mb << 20;
}
logf("capture pool: %zu MB pinned", pool.size >> 20);
}
const size_t aligned = (bytes + 4095) & ~size_t(4095);
if (pool.base == nullptr || pool.used + aligned > pool.size)
return nullptr;
uint8_t* block = pool.base + pool.used;
pool.used += aligned;
g_capturePinned.emplace_back(path, std::make_pair(block, bytes));
return block;
}
static void write_capture_file(uint32_t frame, const char* what, const char* path, const void* data, size_t bytes)
{
static const bool memory = env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) != 0;
if (memory)
{
if (uint8_t* pinned = capture_pinned(path, bytes))
{
std::memcpy(pinned, data, bytes);
return;
}
const auto* first = static_cast<const uint8_t*>(data);
std::lock_guard<std::mutex> lock(g_captureMutex);
g_captureFiles.emplace_back(path, std::vector<uint8_t>(first, first + bytes));
return;
}
FILE* file = std::fopen(path, "wb");
if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
logf("frame %u %s dump failed: %s", frame, what, path);
else
logf("frame %u %s saved: %s", frame, what, path);
if (file != nullptr)
std::fclose(file);
}
// Called after each frame's output dump: once the last triggered frame is in,
// the buffered files go to disk.
static void flush_capture_if_complete(uint32_t frame)
{
char trigger[MAX_PATH];
if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 ||
GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH) == 0)
return;
const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
const uint32_t first = g_captureFirst.load();
if (first == 0 || frame != first + count - 1)
return;
std::vector<std::pair<std::string, std::vector<uint8_t>>> files;
std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> pinned;
{
std::lock_guard<std::mutex> lock(g_captureMutex);
files.swap(g_captureFiles);
pinned.swap(g_capturePinned);
}
logf("capture of frames %u-%u complete in memory; writing %zu files", first, frame, files.size() + pinned.size());
// The pinned pool is not reused, so its blocks stay valid for the writer.
std::thread([files = std::move(files), pinned = std::move(pinned), first, frame] {
size_t failed = 0;
auto write = [&failed](const std::string& path, const uint8_t* data, size_t bytes) {
FILE* file = std::fopen(path.c_str(), "wb");
if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
++failed;
if (file != nullptr)
std::fclose(file);
};
for (const auto& [path, block] : pinned)
write(path, block.first, block.second);
for (const auto& [path, bytes] : files)
write(path, bytes.data(), bytes.size());
logf("capture of frames %u-%u written (%zu files, %zu failed)", first, frame, files.size() + pinned.size(),
failed);
}).detach();
}
static void dump_output(const HostPlane& host, uint32_t frame, const char* path)
{
write_capture_file(frame, "raw DLSS output", path, host.bytes, host.size());
flush_capture_if_complete(frame);
}
static bool input_dump_directory(uint32_t frame, char (&directory)[MAX_PATH])
{
const DWORD length = GetEnvironmentVariableA("D4R_SHIM_INPUT_DUMP_DIR", directory, MAX_PATH);
return length != 0 && length < MAX_PATH &&
dump_frame_selected(frame, "D4R_SHIM_INPUT_DUMP_START", "D4R_SHIM_INPUT_DUMP_COUNT",
"D4R_SHIM_INPUT_DUMP_EVERY");
}
static void dump_input_planes(const HostPlane (&planes)[4], uint32_t frame, const FrameParams& p,
const char* directory)
{
constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
for (int index = 0; index < (planes[3].width != 0 ? 4 : 3); ++index)
{
const HostPlane& host = planes[index];
const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
if (bytes == nullptr || host.width == 0 || host.height == 0)
continue;
char path[MAX_PATH];
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
if (written > 0 && written < MAX_PATH)
write_capture_file(frame, names[index], path, bytes, host.size());
}
// The evaluation parameters, so the harness can replay the sequence.
char path[MAX_PATH];
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-params.txt", directory, frame);
if (written <= 0 || written >= MAX_PATH)
return;
char text[1024];
const int length = std::snprintf(
text, sizeof(text),
"jitter %.9g %.9g\nmv_scale %.9g %.9g\nsharpness %.9g\npre_exposure %.9g\n"
"exposure_scale %.9g\nframe_time %.9g\nreset %d\ninvert %d %d\nrender %u %u\n"
"color_base %u %u\ndepth_base %u %u\nmv_base %u %u\noutput_base %u %u\nhas_exposure %d\n",
p.jitterX, p.jitterY, p.mvScaleX, p.mvScaleY, p.sharpness, p.preExposure, p.exposureScale, p.frameTime,
p.reset, p.invertX, p.invertY, p.renderWidth, p.renderHeight, p.colorBaseX, p.colorBaseY, p.depthBaseX,
p.depthBaseY, p.mvBaseX, p.mvBaseY, p.outputBaseX, p.outputBaseY, p.hasExposure ? 1 : 0);
if (length > 0)
write_capture_file(frame, "params", path, text, static_cast<size_t>(length));
}
// VRAM interop without D4R_SHIM_VRAM_VERIFY: reads the selected frame's input
// planes straight from the imported buffers DLSS is about to consume.
static void dump_vram_inputs(const InputSlot& slot, uint32_t frame, const FrameParams& p, int count)
{
char directory[MAX_PATH];
if (!input_dump_directory(frame, directory))
return;
HostPlane planes[4];
constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
bool allPinned = true;
for (int index = 0; index < count; ++index)
{
const HostPlane& geometry = slot.host[index];
HostPlane& plane = planes[index];
plane.width = geometry.width, plane.height = geometry.height, plane.rowBytes = geometry.rowBytes;
char path[MAX_PATH];
std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
plane.bytes = capture_pinned(path, geometry.size());
if (plane.bytes == nullptr)
{
allPinned = false;
plane.fallback.resize(geometry.size());
}
CudaMemcpy2D readback = {};
readback.srcMemoryType = CUDA_MEMORY_DEVICE;
readback.srcDevice = slot.vram[index].device;
readback.srcPitch = geometry.rowBytes;
readback.dstMemoryType = CUDA_MEMORY_HOST;
readback.dstHost = plane.bytes != nullptr ? plane.bytes : plane.fallback.data();
readback.dstPitch = geometry.rowBytes;
readback.WidthInBytes = geometry.rowBytes;
readback.Height = geometry.height;
if (g.cu.memcpy2D(&readback) != 0)
{
logf("frame %u: VRAM input readback for the dump failed", frame);
return;
}
}
// Pinned planes are already registered for writing; only the rest (and the
// parameters) go through dump_input_planes.
for (HostPlane& plane : planes)
if (plane.bytes != nullptr)
plane.width = 0;
(void)allPinned;
dump_input_planes(planes, frame, p, directory);
}
// Worker stage: downloads the DLSS output into host memory.
static bool download_output(Feature& feature, HostPlane& host, uint32_t frame, FrameTiming* timing)
{
reserve_host(host, static_cast<size_t>(feature.output.width) * 8 * feature.output.height);
host.width = feature.output.width;
host.height = feature.output.height;
host.rowBytes = static_cast<size_t>(feature.output.width) * 8;
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
copy.srcArray = feature.output.array;
copy.dstMemoryType = CUDA_MEMORY_HOST;
copy.dstHost = host.bytes;
copy.dstPitch = host.rowBytes;
copy.WidthInBytes = host.rowBytes;
copy.Height = host.height;
const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
const int result = g.cu.memcpy2D(&copy);
if (timing != nullptr)
timing->download = profile_ms(copyStart, ProfileClock::now());
if (result != 0)
{
logf("cuMemcpy2D download failed: %d", result);
return false;
}
log_output_hash(host, frame);
char dumpPath[MAX_PATH];
if (output_dump_path(frame, dumpPath))
dump_output(host, frame, dumpPath);
return true;
}
// Finish stage: converts a downloaded result into an upload staging buffer.
static void write_output(Staging& staging, const HostPlane& host, FrameTiming* timing)
{
const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
uint8_t* destination = staging.mapped + staging.layout.Offset;
const size_t destinationPitch = staging.layout.Footprint.RowPitch;
const UINT width = staging.width < host.width ? staging.width : host.width;
const UINT height = staging.height < host.height ? staging.height : host.height;
const uint8_t* rows = host.bytes;
if (staging.format == DXGI_FORMAT_R16G16B16A16_FLOAT || staging.format == DXGI_FORMAT_R16G16B16A16_TYPELESS)
copy_rows(g_outputRows, destination, destinationPitch, rows, host.rowBytes, static_cast<size_t>(width) * 8,
height);
else
g_outputRows.run(height, host.rowBytes * height, [&](UINT begin, UINT end) {
for (UINT y = begin; y < end; ++y)
convert_row_out(staging.format, reinterpret_cast<const uint16_t*>(rows + host.rowBytes * y),
destination + y * destinationPitch, width);
});
if (timing != nullptr)
timing->outputConvert = profile_ms(conversionStart, ProfileClock::now());
}
static ID3D12Resource* get_resource(void* parameters, const char* name)
{
ID3D12Resource* resource = nullptr;
if (d4r_ngx_get_d3d12_resource(parameters, name, &resource) == NGX_SUCCESS && resource != nullptr)
return resource;
void* raw = nullptr;
if (d4r_ngx_get_void(parameters, name, &raw) == NGX_SUCCESS)
return static_cast<ID3D12Resource*>(raw);
return nullptr;
}
// --- DLSSD.OutputAlpha: the denoiser's own output ---------------------------------
//
// nvsdk_ngx_defs_dlssd.h:98 makes this an OUTPUT the denoiser writes, not another G-buffer
// input, and it carries its own Subrect base (101/102), so it has to come back to the caller
// rather than be read from it.
//
// The surface is float32, single channel: the vendor's transformer output kernel stores this
// value as a nullable scalar float32 pixel (sust.p.2d.b32.zero against param+248), so the
// surface object is created over a 1-channel float array and converted once on the way out.
//
// Everything else is borrowed from the colour output's proven lifecycle - a ring of slots, a
// claim so a staged result is not overwritten while a recorded command list still reads it,
// and the result presented a frame later - rather than being a second, asymmetric mechanism.
// The single-channel formats a caller may register, and how to write each back.
static bool alpha_output_layout(DXGI_FORMAT format, uint32_t& writeback)
{
switch (format)
{
case DXGI_FORMAT_R8_UNORM: writeback = 1; return true;
case DXGI_FORMAT_R8_UINT: writeback = 2; return true;
case DXGI_FORMAT_R16_FLOAT: writeback = 3; return true;
case DXGI_FORMAT_R16_UNORM: writeback = 4; return true;
case DXGI_FORMAT_R16_UINT: writeback = 5; return true;
case DXGI_FORMAT_R32_FLOAT: writeback = 6; return true;
default: return false;
}
}
static size_t alpha_texel_bytes(uint32_t writeback)
{
return writeback <= 2 ? 1 : (writeback <= 5 ? 2 : 4);
}
static void write_alpha_texel(uint32_t writeback, uint8_t* destination, float value)
{
switch (writeback)
{
case 1:
{
const uint32_t v = to_unorm(value, 8);
std::memcpy(destination, &v, 1);
return;
}
case 2:
{
const uint8_t v = value <= 0.0f ? 0u : (value >= 255.0f ? 255u : static_cast<uint8_t>(value));
std::memcpy(destination, &v, sizeof(v));
return;
}
case 3:
{
const uint16_t v = float_to_half(value);
std::memcpy(destination, &v, sizeof(v));
return;
}
case 4:
{
const uint16_t v = static_cast<uint16_t>(to_unorm(value, 16));
std::memcpy(destination, &v, sizeof(v));
return;
}
case 5:
{
const uint32_t v = value <= 0.0f ? 0u : (value >= 65535.0f ? 65535u : static_cast<uint32_t>(value));
const uint16_t narrow = static_cast<uint16_t>(v);
std::memcpy(destination, &narrow, sizeof(narrow));
return;
}
default:
{
std::memcpy(destination, &value, sizeof(value));
return;
}
}
}
// Game thread. Records which DLSSD.OutputAlpha texture this frame wants written, and holds a
// reference on it until the frame retires. Refuses loudly a texture this file cannot fill:
// accepting the frame and leaving the caller's surface untouched is exactly the silent wrong
// answer this whole route exists to avoid. Sets nothing on the CUDA parameter object - that is
// the worker's, and this runs on the game thread.
static bool capture_alpha_output(Feature& feature, void* parameters, int slotIndex, uint32_t frame)
{
InputSlot& slot = feature.inputs[slotIndex];
ID3D12Resource* const resource = get_resource(parameters, "DLSSD.OutputAlpha");
if (resource == nullptr)
{
slot.alphaRequested = false;
return true;
}
D3D12_RESOURCE_DESC desc;
resource->GetDesc(&desc);
uint32_t writeback = 0;
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.DepthOrArraySize != 1 ||
desc.Width == 0 || desc.Height == 0 || desc.Width > 0xFFFFFFFFull || desc.SampleDesc.Count != 1 ||
!alpha_output_layout(desc.Format, writeback))
{
logf("Ray Reconstruction: DLSSD.OutputAlpha is %s %llux%u array=%u samples=%u fmt=0x%x, which this "
"file cannot write back; this frame's evaluation is refused rather than accepted with the "
"texture left untouched",
desc.Dimension == D3D12_RESOURCE_DIMENSION_TEXTURE2D ? "a texture" : "not a texture", desc.Width,
desc.Height, desc.DepthOrArraySize, desc.SampleDesc.Count, desc.Format);
return false;
}
// The caller's Subrect. The surface is allocated at the texture's FULL extent and the base
// is forwarded, so the denoiser stores where the caller expects; only the evaluated region
// is copied back, leaving the caller's pixels outside it untouched.
const UINT baseX = get_uint_or(parameters, "DLSSD.OutputAlpha.Subrect.Base.X", 0);
const UINT baseY = get_uint_or(parameters, "DLSSD.OutputAlpha.Subrect.Base.Y", 0);
// Guard the subtractions themselves: a base outside the texture underflows below.
if (baseX > desc.Width || baseY > desc.Height)
{
logf("Ray Reconstruction: DLSSD.OutputAlpha subrect origin (%u,%u) is outside its %llux%u texture; "
"this frame's evaluation is refused",
baseX, baseY, desc.Width, desc.Height);
return false;
}
// The evaluated region is the output size placed at that origin, and it must fit; it is not
// clamped silently into whatever happens to fit.
const UINT remainingW = static_cast<UINT>(desc.Width) - baseX;
const UINT remainingH = desc.Height - baseY;
UINT extentW = get_uint_or(parameters, "OutWidth", feature.outWidth);
UINT extentH = get_uint_or(parameters, "OutHeight", feature.outHeight);
if (extentW == 0 || extentW > remainingW || extentH == 0 || extentH > remainingH)
{
logf("Ray Reconstruction: DLSSD.OutputAlpha needs a %ux%u region at (%u,%u) inside a %llux%u texture, "
"which does not fit; this frame's evaluation is refused",
extentW, extentH, baseX, baseY, desc.Width, desc.Height);
return false;
}
if (slot.alphaResource != nullptr && slot.alphaResource != resource)
{
slot.alphaResource->Release();
slot.alphaResource = nullptr; // so the new resource is always AddRef'd below
}
if (slot.alphaResource == nullptr)
resource->AddRef();
slot.alphaResource = resource;
slot.alphaFormat = desc.Format;
slot.alphaWidth = static_cast<UINT>(desc.Width); // full extent: where the vendor stores
slot.alphaHeight = desc.Height;
slot.alphaBaseX = baseX;
slot.alphaBaseY = baseY;
slot.alphaExtentW = extentW; // what is actually copied back
slot.alphaExtentH = extentH;
slot.alphaWriteback = writeback;
slot.alphaRequested = true;
if (frame <= 2)
logf("Ray Reconstruction: DLSSD.OutputAlpha %ux%u fmt=0x%x, subrect (%u,%u) region %ux%u", desc.Width,
desc.Height, desc.Format, baseX, baseY, extentW, extentH);
return true;
}
// Worker. Gives the denoiser a float32 single-channel CUDA surface object for this frame's
// alpha and forwards the subrect origin, so its stores land where the caller expects. Clearing
// the parameter when no texture was registered keeps the denoiser from writing into a surface
// belonging to a frame that no longer wants one; the image itself is kept, since dropping it
// would mean a fresh GPU allocation on the next frame that does ask.
static bool publish_alpha_output(Feature& feature, const InputSlot& slot)
{
if (!slot.alphaRequested)
{
d4r_ngx_set_void(feature.cudaParams, "DLSSD.OutputAlpha", nullptr);
return true;
}
if (!ensure_cuda_image(feature.alphaImage, slot.alphaWidth, slot.alphaHeight, CUDA_FORMAT_FLOAT, 1, true, false))
return false;
d4r_ngx_set_void(feature.cudaParams, "DLSSD.OutputAlpha", &feature.alphaImage.object);
d4r_ngx_set_uint(feature.cudaParams, "DLSSD.OutputAlpha.Subrect.Base.X", slot.alphaBaseX);
d4r_ngx_set_uint(feature.cudaParams, "DLSSD.OutputAlpha.Subrect.Base.Y", slot.alphaBaseY);
return true;
}
// Worker. The colour output's exact claim: the GPU marker is the only thing that proves a
// recorded copy has run, so a slot is reusable only once the GPU is past the frame that last
// read it, and the newest result is never taken from under a reader.
static int claim_alpha_slot(Feature& feature)
{
const auto start = std::chrono::steady_clock::now();
for (;;)
{
{
std::lock_guard<std::mutex> lock(feature.outputMutex);
const uint32_t gpuFrame = *feature.markerValue;
for (int index = 0; index < kOutputSlots; ++index)
{
if (index == feature.latestAlpha.load())
continue;
if (static_cast<int32_t>(gpuFrame - feature.alpha[index].lastReadFrame.load()) > 0)
{
feature.alpha[index].lastReadFrame = 0x7fffffffu + gpuFrame;
return index;
}
}
}
if (std::chrono::steady_clock::now() - start > std::chrono::milliseconds(50))
return -1;
d4r_sleep_us(200);
}
}
// Worker. Reads back the surface the denoiser just wrote and converts the evaluated region into
// the claimed slot's upload buffer. A failure fails the frame: an accepted frame with a missing
// alpha would leave the caller believing it was written.
static bool download_alpha_output(Feature& feature, uint32_t frame)
{
const InputSlot& slot = feature.inputs[frame % kSlots];
if (!slot.alphaRequested)
return true;
if (feature.alphaImage.array == nullptr)
return false;
const int index = claim_alpha_slot(feature);
if (index < 0)
{
logf("frame %u: no free DLSSD.OutputAlpha result slot; the evaluation is dropped", frame);
return false;
}
AlphaSlot& target = feature.alpha[index];
if (!ensure_staging(target.staging, slot.alphaResource, D3D12_HEAP_TYPE_UPLOAD))
{
logf("frame %u: no upload buffer for the DLSSD.OutputAlpha result; the evaluation is dropped", frame);
return false;
}
reserve_host(feature.alphaHost, sizeof(float) * slot.alphaWidth * slot.alphaHeight);
if (feature.alphaHost.bytes == nullptr ||
feature.alphaHost.capacity < sizeof(float) * slot.alphaWidth * slot.alphaHeight)
{
logf("frame %u: no room to read the DLSSD.OutputAlpha surface back; the evaluation is dropped", frame);
return false;
}
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
copy.srcArray = feature.alphaImage.array;
copy.dstMemoryType = CUDA_MEMORY_HOST;
copy.dstHost = feature.alphaHost.bytes;
copy.dstPitch = sizeof(float) * slot.alphaWidth;
copy.WidthInBytes = sizeof(float) * slot.alphaWidth;
copy.Height = slot.alphaHeight;
// Conversion reads the host bytes immediately, so this transfer must complete on return.
if (g.cu.memcpy2D(&copy) != 0)
{
logf("frame %u: reading the DLSSD.OutputAlpha surface back failed; the evaluation is dropped", frame);
return false;
}
// Only the evaluated rectangle is written, at its position in the buffer. The footprint
// offset from GetCopyableFootprints is left alone - it must stay 512-byte aligned - and the
// region is instead selected by the source box of the copy below.
const float* const source = reinterpret_cast<const float*>(feature.alphaHost.bytes);
uint8_t* const destination = target.staging.mapped + target.staging.layout.Offset;
const size_t rowPitch = target.staging.layout.Footprint.RowPitch;
const size_t texel = alpha_texel_bytes(slot.alphaWriteback);
for (UINT row = 0; row < slot.alphaExtentH; ++row)
for (UINT x = 0; x < slot.alphaExtentW; ++x)
write_alpha_texel(slot.alphaWriteback,
destination + static_cast<size_t>(slot.alphaBaseY + row) * rowPitch +
static_cast<size_t>(slot.alphaBaseX) * texel + x * texel,
source[static_cast<size_t>(slot.alphaBaseY + row) * slot.alphaWidth + slot.alphaBaseX + x]);
{
// Published under the same lock the claim uses, exactly as the colour output publishes.
std::lock_guard<std::mutex> lock(feature.outputMutex);
target.format = slot.alphaFormat;
target.baseX = slot.alphaBaseX;
target.baseY = slot.alphaBaseY;
target.width = slot.alphaExtentW;
target.height = slot.alphaExtentH;
target.writeback = slot.alphaWriteback;
target.lastReadFrame = 0;
target.producedFrame = frame;
feature.latestAlpha = index;
}
return true;
}
// Game thread. Presents the newest finished alpha result into the caller's texture. The slot's
// lastReadFrame is stamped before the copy is recorded and producedFrame is NOT cleared: the
// colour output's claim relies on both, and clearing here would let the worker overwrite a
// result whose command list has not run yet.
static bool record_alpha_copyback(Feature& feature, ID3D12GraphicsCommandList* list, InputSlot& slot,
D3D12_RESOURCE_STATES outputState, uint32_t colorFrame)
{
if (!slot.alphaRequested || slot.alphaResource == nullptr)
return true;
if (colorFrame == 0)
return true; // no colour result to pair with this frame
int index = -1;
{
std::lock_guard<std::mutex> lock(feature.outputMutex);
// The frame whose colour is about to be presented is `colorFrame`; only an alpha result
// produced by that very same frame may go with it. Picking "the newest alpha" instead
// would pair alpha N with colour N-1, since the alpha read-back completes before the
// colour's, and the caller would composite two different frames. No match, no copy: the
// colour is presented on its own and the alpha waits for its own frame to come round.
for (int candidate = 0; candidate < kOutputSlots; ++candidate)
if (feature.alpha[candidate].producedFrame.load() == colorFrame)
{
index = candidate;
break;
}
if (index < 0)
return true; // the matching frame has not been read back yet; colour goes alone
AlphaSlot& chosen = feature.alpha[index];
if (chosen.staging.buffer == nullptr || chosen.producedFrame.load() == 0)
return true;
if (chosen.width != slot.alphaExtentW || chosen.height != slot.alphaExtentH ||
chosen.format != slot.alphaFormat || chosen.baseX != slot.alphaBaseX ||
chosen.baseY != slot.alphaBaseY)
{
// The caller's rectangle changed under a result already in flight; presenting it
// would write where the caller no longer expects. Leave it for the next result.
chosen.producedFrame = 0;
return true;
}
chosen.lastReadFrame = feature.frame;
}
AlphaSlot& chosen = feature.alpha[index];
transition(list, slot.alphaResource, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
D3D12_TEXTURE_COPY_LOCATION destination = {};
destination.pResource = slot.alphaResource;
destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
destination.SubresourceIndex = 0;
D3D12_TEXTURE_COPY_LOCATION source = {};
source.pResource = chosen.staging.buffer;
source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
source.PlacedFootprint = chosen.staging.layout;
// The source box picks out the evaluated rectangle inside the full-extent staging buffer,
// and the destination origin places it at the caller's subrect. Together they leave every
// pixel of the caller's texture outside the rectangle untouched.
D3D12_BOX box = {};
box.left = chosen.baseX;
box.top = chosen.baseY;
box.right = chosen.baseX + chosen.width;
box.bottom = chosen.baseY + chosen.height;
box.back = 1;
list->CopyTextureRegion(&destination, chosen.baseX, chosen.baseY, 0, &source, &box);
transition(list, slot.alphaResource, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
return true;
}
// --- Ray Reconstruction: moving the auxiliary surfaces --------------------------
// How long a caller may stop registering a surface before it is treated as dropped. Long
// enough that a game which registers its G-buffer lazily, or only on the frames where it
// changes, keeps it; short enough that a resolution change - which reallocates every resource
// and can reuse a freed address - does not leave a surface bound to somebody else's texture.
constexpr uint32_t kAuxStaleFrames = 300;
static AuxSurface* find_aux_surface(Feature& feature, ID3D12Resource* resource)
{
for (AuxSurface& surface : feature.aux)
if (surface.resource == resource)
return &surface;
return nullptr;
}
static std::string join_aux_names(const std::vector<const char*>& names)
{
std::string joined;
for (const char* name : names)
joined += std::string(joined.empty() ? "" : ", ") + name;
return joined;
}
// Worker. Frees the CUDA objects of a surface and the caller's reference to its resource.
// Only ever called once no queued frame names it any more.
static void destroy_aux_surface(AuxSurface& surface)
{
if (surface.image.object != 0)
g.cu.texObjectDestroy(surface.image.object);
if (surface.image.array != nullptr)
g.cu.arrayDestroy(surface.image.array);
for (Staging& staging : surface.staging)
staging.release();
if (surface.resource != nullptr)
surface.resource->Release();
logf("Ray Reconstruction: released denoiser input %s (%ux%u fmt=0x%x)", join_aux_names(surface.publish).c_str(),
surface.width, surface.height, surface.format);
}
// Game thread. Finds the auxiliary surfaces the caller registered this frame and records a
// readback copy of each into this frame's slot, ahead of the frame marker so the worker can
// read them as soon as that marker is in. The result is this slot's binding snapshot: the
// worker publishes exactly these, so a surface the caller did not register this frame is
// neither sampled nor named, whatever it did last frame.
//
// A texture the caller did supply but this file cannot hand to the denoiser (a shape or a
// format with no CUDA layout) fails the evaluation. Silently omitting a guide the denoiser
// expects is exactly the failure that produces a plausible-looking wrong image, so the frame
// is refused and the reason named instead.
static bool gather_aux_inputs(Feature& feature, ID3D12GraphicsCommandList* list, void* parameters, int slotIndex,
D3D12_RESOURCE_STATES inputState, uint32_t frame)
{
InputSlot& slot = feature.inputs[slotIndex];
std::lock_guard<std::mutex> lock(feature.auxMutex);
std::vector<AuxBinding> bindings;
for (const AuxInput& input : kAuxInputs)
{
ID3D12Resource* const resource = get_resource(parameters, input.source);
if (resource == nullptr)
continue;
AuxSurface* surface = find_aux_surface(feature, resource);
if (surface == nullptr)
{
feature.aux.emplace_back();
surface = &feature.aux.back();
surface->resource = resource;
resource->AddRef(); // held until every frame naming it has retired
logf("Ray Reconstruction: denoiser input %s is resource %p", input.source, static_cast<void*>(resource));
}
surface->dropping = false;
surface->lastSeen = frame;
// One binding per name: two names for the same texture are two denoiser parameters.
AuxBinding* binding = nullptr;
for (AuxBinding& candidate : bindings)
if (candidate.surface == surface)
{
binding = &candidate;
break;
}
if (binding == nullptr)
{
bindings.emplace_back();
binding = &bindings.back();
binding->surface = surface;
binding->frame = frame;
}
bool listed = false;
for (const char* name : surface->publish)
listed = listed || strcmp(name, input.publish) == 0;
if (!listed)
surface->publish.push_back(input.publish);
bool bound = false;
for (const char* name : binding->names)
bound = bound || strcmp(name, input.publish) == 0;
if (!bound)
binding->names.push_back(input.publish);
D3D12_RESOURCE_DESC desc;
resource->GetDesc(&desc);
uint32_t arrayFormat = 0, channels = 0;
AuxLayout how = AuxLayout::Verbatim;
if (desc.DepthOrArraySize != 1 || desc.Width > 0xFFFFFFFFull ||
!aux_array_layout(desc.Format, arrayFormat, channels, how))
{
logf("Ray Reconstruction: denoiser input %s is %llux%u array=%u fmt=0x%x, which has no single-plane "
"CUDA layout; this frame's evaluation is refused rather than run without it",
join_aux_names(binding->names).c_str(), desc.Width, desc.Height, desc.DepthOrArraySize, desc.Format);
return false;
}
surface->format = desc.Format;
surface->arrayFormat = arrayFormat;
surface->channels = channels;
surface->how = how;
surface->width = static_cast<UINT>(desc.Width);
surface->height = static_cast<UINT>(desc.Height);
if (!ensure_staging(surface->staging[slotIndex], resource, D3D12_HEAP_TYPE_READBACK))
return false;
copy_to_staging(list, resource, surface->staging[slotIndex], inputState);
}
// Drop surfaces the caller stopped registering, but only once no queued frame names one:
// erasing a surface whose handle is still in a parameter object would leave the denoiser
// sampling a freed texture object.
for (AuxSurface& surface : feature.aux)
{
if (surface.holds != 0 || frame - surface.lastSeen < kAuxStaleFrames)
continue;
surface.dropping = true;
logf("Ray Reconstruction: denoiser input %s was not registered for %u frames",
join_aux_names(surface.publish).c_str(), kAuxStaleFrames);
}
// Committed only now. An earlier failure path returns from this function with `bindings`
// half-built, and a hold taken there would never be given back - the surface could then
// never be dropped, and its reference to the caller's resource would never be released.
for (const AuxBinding& binding : bindings)
++binding.surface->holds; // given back by retire_aux_bindings when this frame passes
slot.auxBindings = std::move(bindings);
return true;
}
// Worker. Copies this frame's bound surfaces into their CUDA arrays verbatim - same bytes,
// same row order, only the destination is the GPU - and mints the texture objects the
// denoiser samples. A failure fails the evaluation: a denoiser handed fewer guides than the
// caller registered produces a wrong image, not a visible error.
static bool upload_aux_inputs(Feature& feature, int slotIndex, uint32_t frame)
{
for (const AuxBinding& binding : feature.inputs[slotIndex].auxBindings)
{
AuxSurface& surface = *binding.surface;
const Staging& staging = surface.staging[slotIndex];
if (!ensure_cuda_image(surface.image, surface.width, surface.height, surface.arrayFormat,
surface.channels, false, false))
{
logf("frame %u: Ray Reconstruction input %s: a CUDA texture object over %ux%u fmt=0x%x could not be "
"created",
frame, join_aux_names(binding.names).c_str(), surface.width, surface.height, surface.format);
return false;
}
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_HOST;
if (surface.how == AuxLayout::Verbatim)
{
// Use HostPlane staging: direct ROCm image copies from Vulkan-mapped memory
// truncate guide rows on gfx1201, even when the source footprint is tightly packed.
HostPlane& host = surface.host[slotIndex];
const size_t rowBytes = aux_texel_bytes(surface.arrayFormat, surface.channels) * surface.width;
reserve_host(host, rowBytes * surface.height);
if (host.bytes == nullptr || host.capacity < rowBytes * surface.height)
return false;
host.width = surface.width;
host.height = surface.height;
host.rowBytes = rowBytes;
for (UINT row = 0; row < surface.height; ++row)
memcpy(host.bytes + static_cast<size_t>(row) * rowBytes,
staging.mapped + staging.layout.Offset +
static_cast<size_t>(row) * staging.layout.Footprint.RowPitch, rowBytes);
copy.srcHost = host.bytes;
copy.srcPitch = rowBytes;
}
else
{
// Packed, integer and BGRA inputs are unpacked into floats first: a CUDA array
// has no three-channel layout, a uint array is sampled normalized, and BGRA bytes
// are in the wrong order for a read as RGBA.
HostPlane& host = surface.host[slotIndex];
const size_t rowBytes = sizeof(float) * surface.channels * surface.width;
reserve_host(host, rowBytes * surface.height);
if (host.bytes == nullptr || host.capacity < rowBytes * surface.height)
{
logf("frame %u: Ray Reconstruction input %s: no room for %zu unpacked bytes",
frame, join_aux_names(binding.names).c_str(), rowBytes * surface.height);
return false;
}
host.width = surface.width;
host.height = surface.height;
host.rowBytes = rowBytes;
const uint8_t* const base = staging.mapped + staging.layout.Offset;
for (UINT row = 0; row < surface.height; ++row)
convert_aux_row(surface.format, base + static_cast<size_t>(row) * staging.layout.Footprint.RowPitch,
reinterpret_cast<float*>(host.bytes + static_cast<size_t>(row) * rowBytes),
surface.width, surface.channels);
copy.srcHost = host.bytes;
copy.srcPitch = rowBytes;
}
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
copy.dstArray = surface.image.array;
copy.WidthInBytes = aux_texel_bytes(surface.arrayFormat, surface.channels) * surface.width;
copy.Height = surface.height;
if (copy_2d(copy) != 0)
{
logf("frame %u: Ray Reconstruction input %s: the upload of %ux%u failed", frame,
join_aux_names(binding.names).c_str(), surface.width, surface.height);
return false;
}
surface.handle = surface.image.object;
}
return true;
}
// Worker. Points the denoiser's parameters at this frame's texture objects and forwards the
// settings and camera matrices the caller registered for this frame. Every name this file
// ever published is cleared first: the parameter object outlives the frame, so a name left
// holding last frame's handle would be read as a live texture.
static void publish_aux_parameters(Feature& feature, FrameParams& p, const InputSlot& slot)
{
void* params = feature.cudaParams;
for (size_t index = 0; index < sizeof(kAuxScalars) / sizeof(kAuxScalars[0]); ++index)
if (p.auxScalarSet[index])
d4r_ngx_set_int(params, kAuxScalars[index].name, p.auxScalar[index]);
// Always written, never conditionally: the parameter object outlives this frame, so a
// matrix pointer left over from the previous one would dangle the moment this frame's
// FrameParams goes out of scope. Absent, it is cleared - the denoiser then knows the
// caller supplied none, rather than reading a stale one.
for (size_t index = 0; index < sizeof(kAuxMatrices) / sizeof(kAuxMatrices[0]); ++index)
d4r_ngx_set_void(params, kAuxMatrices[index],
p.auxMatrixSet[index] ? static_cast<void*>(p.auxMatrix[index]) : nullptr);
for (size_t index = 0; index < sizeof(kAuxSubrectNames) / sizeof(kAuxSubrectNames[0]); ++index)
{
static std::string nameX, nameY;
nameX.assign(kAuxSubrectNames[index]).append(".Subrect.Base.X");
nameY.assign(kAuxSubrectNames[index]).append(".Subrect.Base.Y");
d4r_ngx_set_uint(params, nameX.c_str(), p.auxSubrect[index][0]);
d4r_ngx_set_uint(params, nameY.c_str(), p.auxSubrect[index][1]);
}
for (const AuxSurface& surface : feature.aux)
for (const char* name : surface.publish)
d4r_ngx_set_void(params, name, nullptr);
// Pointers to the handles, which is what a CUDA resource parameter is: a variable holding
// the object, not the object itself. Two surfaces naming one denoiser input (a caller that
// binds GBuffer.DiffuseAlbedo and DLSS.Input.DiffuseAlbedo to different textures) leave the
// later one winning, which is the caller's ambiguity to resolve, not this file's.
for (const AuxBinding& binding : slot.auxBindings)
if (binding.surface->handle != 0)
for (const char* name : binding.names)
d4r_ngx_set_void(params, name, &binding.surface->handle);
}
// Worker, once the frame this slot was gathering for has passed. Each bound surface keeps the
// caller's resource alive until here, and a dropped surface is freed only once its last frame
// is gone.
static void retire_aux_bindings(Feature& feature, int slotIndex)
{
std::lock_guard<std::mutex> lock(feature.auxMutex);
for (AuxBinding& binding : feature.inputs[slotIndex].auxBindings)
if (binding.surface->holds > 0)
--binding.surface->holds;
feature.inputs[slotIndex].auxBindings.clear();
for (auto it = feature.aux.begin(); it != feature.aux.end();)
{
if (!it->dropping || it->holds != 0)
{
++it;
continue;
}
destroy_aux_surface(*it);
it = feature.aux.erase(it);
}
}
static void set_evaluation_parameters(Feature& feature, const FrameParams& p)
{
void* params = feature.cudaParams;
feature.colorHandle = feature.color.object;
feature.depthHandle = feature.depth.object;
feature.motionHandle = feature.motion.object;
feature.exposureHandle = feature.exposure.object;
feature.outputHandle = feature.output.object;
// CUDA resource parameters point to variables holding the object handles.
d4r_ngx_set_void(params, "Color", &feature.colorHandle);
d4r_ngx_set_void(params, "Depth", &feature.depthHandle);
d4r_ngx_set_void(params, "MotionVectors", &feature.motionHandle);
d4r_ngx_set_void(params, "Output", &feature.outputHandle);
d4r_ngx_set_void(params, "ExposureTexture", p.hasExposure ? &feature.exposureHandle : nullptr);
d4r_ngx_set_void(params, "TransparencyMask", nullptr);
d4r_ngx_set_void(params, "DLSS.Input.Bias.Current.Color.Mask", nullptr);
d4r_ngx_set_float(params, "Jitter.Offset.X", p.jitterX);
d4r_ngx_set_float(params, "Jitter.Offset.Y", p.jitterY);
d4r_ngx_set_float(params, "MV.Scale.X", p.mvScaleX);
d4r_ngx_set_float(params, "MV.Scale.Y", p.mvScaleY);
d4r_ngx_set_float(params, "Sharpness", p.sharpness);
// The DLSS library stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output in this setup; the SDK's
// Disable.Watermark parameter turns that off. D4R_SHIM_WATERMARK=1 keeps it.
static const bool watermark = env_uint("D4R_SHIM_WATERMARK", 0) != 0;
d4r_ngx_set_int(params, "Disable.Watermark", watermark ? 0 : 1);
d4r_ngx_set_int(params, "Reset", p.reset);
d4r_ngx_set_float(params, "DLSS.Pre.Exposure", p.preExposure);
d4r_ngx_set_float(params, "DLSS.Exposure.Scale", p.exposureScale);
d4r_ngx_set_float(params, "FrameTimeDeltaInMsec", p.frameTime);
d4r_ngx_set_int(params, "DLSS.Indicator.Invert.X.Axis", p.invertX);
d4r_ngx_set_int(params, "DLSS.Indicator.Invert.Y.Axis", p.invertY);
d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Width", p.renderWidth);
d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Height", p.renderHeight);
d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.X", p.colorBaseX);
d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.Y", p.colorBaseY);
d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.X", p.depthBaseX);
d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.Y", p.depthBaseY);
d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.X", p.mvBaseX);
d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.Y", p.mvBaseY);
d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.X", p.outputBaseX);
d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.Y", p.outputBaseY);
if (feature.scratch != 0)
{
d4r_ngx_set_void(params, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature.scratch)));
d4r_ngx_set_ull(params, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
}
}
// DLSS evaluations run as a three-stage pipeline so the CPU copies of one
// frame overlap the GPU work of another:
// prepare_inputs (prep thread) waits for the game's input copies and
// converts the readback staging into host planes;
// run_evaluation (CUDA worker) uploads them, evaluates, downloads the output;
// publish_output (finish thread) writes it into an upload staging buffer and
// makes it the result evaluate() presents.
// Every NGX-core and CUDA call stays on the worker. A stage only reuses a
// buffer once the next stage has released it (hostBusy / outputHostBusy).
static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing);
static int claim_output_slot(Feature* feature);
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
const FrameTiming& timing, ProfileClock::time_point start);
// Spins until `flag` clears; false after two seconds.
static bool wait_for_release(const std::atomic<bool>& flag)
{
const auto start = std::chrono::steady_clock::now();
while (flag.load())
{
if (std::chrono::steady_clock::now() - start > std::chrono::seconds(2))
return false;
d4r_sleep_us(100);
}
return true;
}
static void release_split_frame(Feature* feature, uint32_t frame);
// Every queued frame ends in exactly one call of this (published or dropped).
static void frame_retired(Feature* feature, uint32_t frame)
{
if (feature->split)
release_split_frame(feature, frame);
// The denoiser inputs this frame bound belong to the slot it used, and the caller stops
// owing them a reference only now that its evaluate has returned.
if (feature->rayReconstruction)
{
retire_aux_bindings(*feature, static_cast<int>(frame % kSlots));
}
feature->inFlight.fetch_sub(1);
}
// Overlapped staging, worker thread only. Uploads the staged planes with async
// copies into linear buffers on the "up" stream, then moves them into the CUDA
// arrays device-to-device (on the null stream, ahead of NGX's kernels).
static bool upload_inputs_async(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
FrameTiming* timing)
{
CopyStreams* streams = copy_streams();
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
const int count = hasExposure ? 4 : 3;
const auto start = ProfileClock::now();
for (int index = 0; index < count; ++index)
{
const HostPlane& host = slot.host[index];
if (!ensure_cuda_image(*images[index], host.width, host.height, plane_format(planes[index]),
plane_channels(planes[index]), false, planes[index] == Plane::Color) ||
!ensure_linear(feature.planeLinear[index], feature.planeLinearBytes[index], host.size()))
return false;
if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
{
uint64_t hash = 1469598103934665603ull;
for (size_t byte = 0; byte < host.size(); ++byte)
hash = (hash ^ host.bytes[byte]) * 1099511628211ull;
logf("frame %u upload plane=%d bytes=%zu hash=%016llx (async)", frame, index, host.size(),
static_cast<unsigned long long>(hash));
}
const auto issue = ProfileClock::now();
const int result = g.cu.memcpyHtoDAsync(feature.planeLinear[index], host.bytes, host.size(), streams->up);
if (timing != nullptr)
timing->upload[index] = profile_ms(issue, ProfileClock::now());
if (result != 0)
{
logf("cuMemcpyHtoDAsync failed: %d", result);
return false;
}
}
int result = g.cu.streamSynchronize(streams->up);
for (int index = 0; index < count && result == 0; ++index)
{
const HostPlane& host = slot.host[index];
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_DEVICE;
copy.srcDevice = feature.planeLinear[index];
copy.srcPitch = host.rowBytes;
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
copy.dstArray = images[index]->array;
copy.WidthInBytes = host.rowBytes;
copy.Height = host.height;
result = copy_2d(copy);
}
if (timing != nullptr)
timing->h2dTotal = profile_ms(start, ProfileClock::now());
if (result != 0)
logf("staged upload failed: %d", result);
return result == 0;
}
// Starts the output download: array -> linear buffer, then an async copy on
// the "down" stream that the next frame's uploads overlap.
static bool start_download(Feature& feature, HostPlane& host, FrameTiming* timing)
{
const size_t rowBytes = static_cast<size_t>(feature.output.width) * 8;
const size_t bytes = rowBytes * feature.output.height;
reserve_host(host, bytes);
host.width = feature.output.width;
host.height = feature.output.height;
host.rowBytes = rowBytes;
if (!ensure_linear(feature.outputLinear, feature.outputLinearBytes, bytes))
return false;
const auto start = ProfileClock::now();
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
copy.srcArray = feature.output.array;
copy.dstMemoryType = CUDA_MEMORY_DEVICE;
copy.dstDevice = feature.outputLinear;
copy.dstPitch = rowBytes;
copy.WidthInBytes = rowBytes;
copy.Height = host.height;
int result = feature.outputRedirected ? 0 : copy_2d(copy);
// The null-stream copy must land before the non-blocking stream reads it.
if (result == 0)
result = g.cu.ctxSynchronize();
if (result == 0)
result = g.cu.memcpyDtoHAsync(host.bytes, feature.outputLinear, bytes, copy_streams()->down);
if (timing != nullptr)
timing->d2hIssue = profile_ms(start, ProfileClock::now());
if (result != 0)
logf("staged download failed: %d", result);
return result == 0;
}
// A result whose download is still in flight so the next frame's uploads can
// overlap it. Worker thread only.
struct PendingDownload
{
Feature* feature = nullptr;
int hostIndex = 0;
uint32_t frame = 0;
FrameParams params;
FrameTiming timing;
};
static PendingDownload g_pendingDownload;
static void post_publish(Feature* feature, int hostIndex, uint32_t frame, const FrameParams& params,
FrameTiming timing)
{
if (timing.enabled)
timing.finishQueued = ProfileClock::now();
g.finish.post([feature, hostIndex, frame, params, timing] {
publish_output(feature, hostIndex, frame, params, timing);
});
}
// Waits for the in-flight download, if any, and hands it to publish_output.
// `inline_completion` marks the case without a next frame to overlap: the wait
// then belongs to this frame's worker time.
static void complete_pending_download(bool inline_completion = false)
{
PendingDownload pending = g_pendingDownload;
if (pending.feature == nullptr)
return;
g_pendingDownload = {};
const auto waitStart = ProfileClock::now();
const int result = g.cu.streamSynchronize(copy_streams()->down);
if (pending.timing.enabled)
{
pending.timing.d2hWait = profile_ms(waitStart, ProfileClock::now());
pending.timing.download = pending.timing.d2hIssue + pending.timing.d2hWait;
if (inline_completion)
pending.timing.worker += pending.timing.d2hWait;
}
if (result != 0)
{
logf("frame %u: output download failed: %d", pending.frame, result);
pending.feature->outputHostBusy[pending.hostIndex] = false;
frame_retired(pending.feature, pending.frame);
return;
}
log_output_hash(pending.feature->outputHost[pending.hostIndex], pending.frame);
char dumpPath[MAX_PATH];
if (output_dump_path(pending.frame, dumpPath))
dump_output(pending.feature->outputHost[pending.hostIndex], pending.frame, dumpPath);
post_publish(pending.feature, pending.hostIndex, pending.frame, pending.params, pending.timing);
}
// Linear inputs, worker thread: pitch-linear texture objects on the slot's interop buffers (recreated
// when a buffer or its geometry changes), reported to NGX as arrays by the bridge.
static bool ensure_linear_inputs(InputSlot& slot, bool hasExposure)
{
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
const int count = hasExposure ? 4 : 3;
for (int index = 0; index < count; ++index)
{
const HostPlane& geometry = slot.host[index];
const VramBuffer& buffer = slot.vram[index];
if (slot.linearTexture[index] != 0 && slot.linearPointer[index] == buffer.device &&
slot.linearWidth[index] == geometry.width && slot.linearHeight[index] == geometry.height &&
slot.linearPitch[index] == geometry.rowBytes)
continue;
if (slot.linearTexture[index] != 0)
g.cu.texObjectDestroy(slot.linearTexture[index]);
slot.linearTexture[index] = 0;
struct Pitch2D
{
CudaDevicePtr pointer;
uint32_t format, channels;
size_t width, height, pitch;
} pitch2D = {buffer.device, plane_format(planes[index]), plane_channels(planes[index]), geometry.width,
geometry.height, geometry.rowBytes};
static_assert(sizeof(Pitch2D) == 40, "CUDA_RESOURCE_DESC pitch2D");
CudaResourceDesc resource = {};
resource.resType = 3; // CU_RESOURCE_TYPE_PITCH2D
memcpy(&resource.res, &pitch2D, sizeof(pitch2D));
const CudaTextureDesc sampler = input_sampler(geometry.width, pitch2D.channels, planes[index] == Plane::Color);
CudaObject object = 0;
int result = g.cu.texObjectCreate(&object, &resource, &sampler, nullptr);
if (result == 0)
result = g.cu.registerLinearTexture(object, geometry.width, geometry.height, pitch2D.format, pitch2D.channels);
if (result != 0)
{
logf("linear input texture %d (%ux%u, pitch %zu) failed: %d", index, geometry.width, geometry.height,
geometry.rowBytes, result);
if (object != 0)
g.cu.texObjectDestroy(object);
return false;
}
slot.linearTexture[index] = object;
slot.linearPointer[index] = buffer.device;
slot.linearWidth[index] = geometry.width;
slot.linearHeight[index] = geometry.height;
slot.linearPitch[index] = geometry.rowBytes;
}
return true;
}
// VRAM interop, worker thread: moves the planes from the imported buffers
// Opt-in thin-feature motion coverage. Keep the original imported velocity buffer
// immutable; the new texture is owned by this input slot until its GPU work retires.
static CudaObject prepare_dilated_motion(Feature& feature, InputSlot& slot, const FrameParams& p)
{
const unsigned radius = env_uint("D4R_MOTION_DILATION", 0);
if (radius == 0)
return 0;
const auto& depth = slot.host[1];
const auto& motion = slot.host[2];
D4rMotionDilationParams args = {slot.vram[1].device, slot.vram[2].device, slot.dilatedMotion,
depth.rowBytes, motion.rowBytes, p.renderWidth, p.renderHeight, motion.width, motion.height,
p.jitterX, p.jitterY, radius, (feature.flags & 8u) != 0};
const bool motionGeometry = d4r_motion_dilation_geometry(&args, feature.flags,
feature.outWidth, feature.outHeight);
const bool supported = radius <= 2 && p.vram && feature.linearInputs && g.cu.dilateMotion != nullptr &&
motionGeometry && p.colorBaseX == 0 && p.colorBaseY == 0 && p.depthBaseX == 0 &&
p.depthBaseY == 0 && p.mvBaseX == 0 && p.mvBaseY == 0 && p.outputBaseX == 0 && p.outputBaseY == 0 &&
p.renderWidth > 0 && p.renderHeight > 0 && std::isfinite(p.jitterX) && std::isfinite(p.jitterY) &&
p.renderWidth <= depth.width && p.renderHeight <= depth.height;
if (!supported)
{
if (!feature.dilationReported)
logf("motion-vector dilation unavailable for this feature; original motion vectors retained");
feature.dilationReported = true;
return 0;
}
const size_t bytes = motion.size();
const bool recreate = slot.dilatedBytes < bytes || slot.dilatedWidth != motion.width ||
slot.dilatedHeight != motion.height || slot.dilatedPitch != motion.rowBytes;
if (recreate)
{
if (slot.dilatedTexture != 0)
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
if (slot.dilatedMotion != 0)
g.cu.memFree(slot.dilatedMotion), slot.dilatedMotion = 0;
slot.dilatedBytes = 0;
if (g.cu.memAlloc(&slot.dilatedMotion, bytes) != 0)
return 0;
slot.dilatedBytes = bytes;
struct Pitch2D { CudaDevicePtr pointer; uint32_t format, channels; size_t width, height, pitch; };
const Pitch2D pitch = {slot.dilatedMotion, plane_format(Plane::Motion), 2, motion.width, motion.height, motion.rowBytes};
CudaResourceDesc resource = {};
resource.resType = 3;
static_assert(sizeof(pitch) == 40);
std::memcpy(&resource.res, &pitch, sizeof(pitch));
const CudaTextureDesc sampler = input_sampler(motion.width, 2, false);
int result = g.cu.texObjectCreate(&slot.dilatedTexture, &resource, &sampler, nullptr);
if (result == 0)
result = g.cu.registerLinearTexture(slot.dilatedTexture, motion.width, motion.height, pitch.format, 2);
if (result != 0)
{
if (slot.dilatedTexture != 0)
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
return 0;
}
slot.dilatedWidth = motion.width, slot.dilatedHeight = motion.height, slot.dilatedPitch = motion.rowBytes;
}
if (slot.dilatedTexture == 0)
return 0;
args.output = slot.dilatedMotion;
// Preserve allocation padding, which DLSS may sample at the active edge.
// The dilation kernel only writes the active render grid for low-res MVs.
if (args.motion_width != motion.width || args.motion_height != motion.height)
{
CudaMemcpy2D copy = {};
copy.srcMemoryType = copy.dstMemoryType = CUDA_MEMORY_DEVICE;
copy.srcDevice = args.motion;
copy.dstDevice = args.output;
copy.srcPitch = copy.dstPitch = motion.rowBytes;
copy.WidthInBytes = motion.rowBytes;
copy.Height = motion.height;
if (copy_2d(copy) != 0)
return 0;
}
const int result = g.cu.dilateMotion(&args);
if (!feature.dilationReported)
logf("motion-vector dilation %s: radius=%u render pixels, GPU status=%d, %s active=%ux%u allocation=%ux%u",
result == 0 ? "enabled" : "unavailable", radius, result,
(feature.flags & 2u) != 0 ? "low-resolution" : "display-resolution",
args.motion_width, args.motion_height, motion.width, motion.height);
feature.dilationReported = true;
return result == 0 ? slot.dilatedTexture : 0;
}
// VRAM interop, worker thread: moves the planes from the imported buffers
// into the CUDA arrays on the GPU. With D4R_SHIM_VRAM_VERIFY also compares
// them with the host-staged planes of the same frame.
static bool upload_inputs_vram(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
FrameTiming* timing)
{
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
const int count = hasExposure ? 4 : 3;
const bool verify = env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
const auto start = ProfileClock::now();
int result = 0;
for (int index = 0; index < count && result == 0; ++index)
{
const HostPlane& geometry = slot.host[index];
if (!ensure_cuda_image(*images[index], geometry.width, geometry.height, plane_format(planes[index]),
plane_channels(planes[index]), false, planes[index] == Plane::Color))
return false;
if (verify)
{
std::vector<uint8_t> vram(geometry.size());
CudaMemcpy2D readback = {};
readback.srcMemoryType = CUDA_MEMORY_DEVICE;
readback.srcDevice = slot.vram[index].device;
readback.srcPitch = geometry.rowBytes;
readback.dstMemoryType = CUDA_MEMORY_HOST;
readback.dstHost = vram.data();
readback.dstPitch = geometry.rowBytes;
readback.WidthInBytes = geometry.rowBytes;
readback.Height = geometry.height;
size_t differing = 0;
if (g.cu.memcpy2D(&readback) == 0)
for (size_t byte = 0; byte < vram.size(); ++byte)
differing += vram[byte] != geometry.bytes[byte];
if (differing != 0 || frame <= 3 || frame % 120 == 0)
logf("frame %u VRAM verify plane=%d: %zu of %zu bytes differ from host staging", frame, index,
differing, vram.size());
}
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_DEVICE;
copy.srcDevice = slot.vram[index].device;
copy.srcPitch = geometry.rowBytes;
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
copy.dstArray = images[index]->array;
copy.WidthInBytes = geometry.rowBytes;
copy.Height = geometry.height;
result = copy_2d(copy);
// D4R_SHIM_VRAM_ARRAY_VERIFY=1: read the CUDA array DLSS samples back
// and compare it with the imported buffer it was filled from.
static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
if (arrayVerify && result == 0 && g.cu.ctxSynchronize() == 0)
{
std::vector<uint8_t> fromBuffer(geometry.size()), fromArray(geometry.size());
CudaMemcpy2D readBuffer = {};
readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
readBuffer.srcDevice = slot.vram[index].device;
readBuffer.srcPitch = geometry.rowBytes;
readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
readBuffer.dstHost = fromBuffer.data();
readBuffer.dstPitch = geometry.rowBytes;
readBuffer.WidthInBytes = geometry.rowBytes;
readBuffer.Height = geometry.height;
CudaMemcpy2D readArray = {};
readArray.srcMemoryType = CUDA_MEMORY_ARRAY;
readArray.srcArray = images[index]->array;
readArray.dstMemoryType = CUDA_MEMORY_HOST;
readArray.dstHost = fromArray.data();
readArray.dstPitch = geometry.rowBytes;
readArray.WidthInBytes = geometry.rowBytes;
readArray.Height = geometry.height;
if (g.cu.memcpy2D(&readBuffer) == 0 && g.cu.memcpy2D(&readArray) == 0)
{
size_t differing = 0, firstRow = geometry.height;
for (size_t byte = 0; byte < fromArray.size(); ++byte)
if (fromArray[byte] != fromBuffer[byte])
{
if (differing++ == 0)
firstRow = byte / geometry.rowBytes;
}
if (differing != 0 || frame <= 3 || frame % 120 == 0)
logf("frame %u VRAM array verify plane=%d: %zu of %zu bytes differ (first row %zu)", frame, index,
differing, fromArray.size(), firstRow);
}
}
}
if (timing != nullptr)
timing->h2dTotal = profile_ms(start, ProfileClock::now());
if (result != 0)
logf("VRAM input copy failed: %d", result);
return result == 0;
}
// Only called on the CUDA worker. A completion event covers queued work on
// the default stream; do not rely on HIP's blocking flag to release the CPU.
static int synchronize_default_stream(Feature& feature, bool* querySleep = nullptr, double* waitEmaUs = nullptr)
{
if (feature.outputReadyEvent != nullptr)
{
const int recorded = g.cu.eventRecord(feature.outputReadyEvent, nullptr);
if (recorded == 0)
{
// D4R_SHIM_OUTPUT_POLL_US: N > 0 polls every N us (200 was the fixed interval before), 0 yields between
// queries (spins). Unset: adaptive - sleep until shortly before the predicted completion (smoothed
// wait of earlier frames), then poll every 20 us, backing off to 200 us when the GPU runs late.
static const int fixedPoll = [] {
const char* v = std::getenv("D4R_SHIM_OUTPUT_POLL_US");
return v != nullptr && *v != '\0' ? static_cast<int>(env_uint("D4R_SHIM_OUTPUT_POLL_US", 200)) : -1;
}();
const auto waitStart = std::chrono::steady_clock::now();
unsigned polls = 0;
double lastNotReadyUs = -1.0; // elapsed time of the last query that found the work unfinished
const int result = d4r_wait_event(
[&] { return g.cu.eventQuery(feature.outputReadyEvent); },
[&] {
if (fixedPoll >= 0)
{
d4r_sleep_us(static_cast<unsigned>(fixedPoll));
return;
}
constexpr double kMarginUs = 120.0, kStepUs = 20.0, kChunkUs = 200.0, kSpinAfterUs = 150.0;
const double elapsed = std::chrono::duration<double, std::micro>(
std::chrono::steady_clock::now() - waitStart).count();
lastNotReadyUs = elapsed; // this sleep follows a query that returned NOT_READY
++polls;
const double predicted = waitEmaUs != nullptr ? *waitEmaUs : 0.0;
const double remaining = predicted - kMarginUs - elapsed;
// far from the predicted completion: sleep in chunks of at most 200 us (a single long sleep
// can overshoot by far more under Wine); near it: 20 us polls; late: back off to 200 us
if (remaining > kStepUs)
d4r_sleep_us(static_cast<unsigned>(std::min(remaining, kChunkUs)));
else if (elapsed < predicted + kSpinAfterUs)
d4r_sleep_us(0); // final stretch around the predicted completion: spin (SwitchToThread);
// a Wine sleep rounds up to ~50 us, which would be the detection latency
else
d4r_sleep_us(static_cast<unsigned>(std::min(
kStepUs * (1.0 + (elapsed - predicted - kSpinAfterUs) / 160.0), kChunkUs)));
});
if (result == 0 && fixedPoll < 0 && waitEmaUs != nullptr)
{
const double waited = std::chrono::duration<double, std::micro>(
std::chrono::steady_clock::now() - waitStart).count();
// Learn the completion time, not our own wait: the work finished between the last NOT_READY query
// and the successful one (at most one 200 us chunk apart), so the midpoint cannot ratchet upwards
double sample = lastNotReadyUs >= 0.0 ? 0.5 * (lastNotReadyUs + waited) : waited;
sample = std::min(sample, 20000.0);
*waitEmaUs = *waitEmaUs == 0.0 ? sample : 0.8 * *waitEmaUs + 0.2 * sample;
}
if (result == 0)
{
if (querySleep != nullptr) *querySleep = true;
// The event is already complete; this also flushes bridge kernel profiles.
return g.cu.eventSynchronize(feature.outputReadyEvent);
}
logf("output event query failed (%d); using context synchronization", result);
}
else
logf("output event record failed (%d); using context synchronization", recorded);
}
return g.cu.ctxSynchronize();
}
// A frame recorded for packed output whose output kernel then did not pack it (it was not the native one
// after all): the RGBA16F result is converted on the host, rounding as the kernel store and the game-side
// blit do. Slow, and only expected around a kernel change.
static int pack_output_on_host(Feature& feature, CudaMemcpy2D copy, size_t packedPitch, uint32_t frame)
{
const size_t width = feature.output.width, height = feature.output.height;
std::vector<uint16_t> halves(width * height * 4);
std::vector<uint32_t> packed(width * height);
const CudaDevicePtr target = copy.dstDevice;
copy.dstMemoryType = CUDA_MEMORY_HOST;
copy.dstHost = halves.data();
copy.dstPitch = width * 8;
int result = g.cu.memcpy2D(&copy);
if (result != 0)
return result;
const auto unorm = [](uint16_t half, float scale) {
float value = half_to_float(half);
value = value > 0.0f ? value : 0.0f;
value = value < 1.0f ? value : 1.0f;
return static_cast<uint32_t>(value * scale + 0.5f);
};
for (size_t texel = 0; texel < packed.size(); ++texel)
packed[texel] = unorm(halves[texel * 4], 1023.0f) | unorm(halves[texel * 4 + 1], 1023.0f) << 10 |
unorm(halves[texel * 4 + 2], 1023.0f) << 20 | unorm(halves[texel * 4 + 3], 3.0f) << 30;
CudaMemcpy2D upload = {};
upload.srcMemoryType = CUDA_MEMORY_HOST;
upload.srcHost = packed.data();
upload.srcPitch = width * 4;
upload.dstMemoryType = CUDA_MEMORY_DEVICE;
upload.dstDevice = target;
upload.dstPitch = packedPitch;
upload.WidthInBytes = width * 4;
upload.Height = height;
result = g.cu.memcpy2D(&upload);
logf("frame %u: output kernel did not pack its result; converted on the host (%d)", frame, result);
return result;
}
// VRAM interop, worker thread: copies the finished result into a free output
// slot's imported buffer and publishes it; evaluate() copies it on to the game.
static void publish_vram(Feature* feature, uint32_t frame, const FrameParams& params, FrameTiming& timing,
ProfileClock::time_point workerStart)
{
const auto start = ProfileClock::now();
// A split frame's second half reads slot frame % kOutputSlots; its previous
// reader, frame - kOutputSlots, ran before this frame's marker.
const int target = params.split ? static_cast<int>(frame % kOutputSlots) : claim_output_slot(feature);
if (target < 0)
{
logf("frame %u: no free output slot; dropping result", frame);
frame_retired(feature, frame);
return;
}
VramBuffer buffer = feature->outputs[target].vram;
const size_t rowBytes = static_cast<size_t>(feature->output.width) * 8;
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
copy.srcArray = feature->output.array;
copy.dstMemoryType = CUDA_MEMORY_DEVICE;
copy.dstDevice = buffer.device;
copy.dstPitch = rowBytes;
copy.WidthInBytes = rowBytes;
copy.Height = feature->output.height;
if (params.outputInPlace != 0)
{
// no game-side copy was recorded: a result the output kernel did not redirect goes to the texture here
buffer.device = copy.dstDevice = params.outputInPlace;
copy.dstPitch = params.outputInPlacePitch;
buffer.bytes = static_cast<size_t>(params.outputInPlacePitch) * feature->output.height;
}
int result = rowBytes / (params.packedOutput ? 2 : 1) * feature->output.height > buffer.bytes ? -1
: feature->outputRedirected ? 0
: params.packedOutput ? pack_output_on_host(*feature, copy, params.outputInPlace != 0 ? copy.dstPitch
: rowBytes / 2, frame)
: copy_2d(copy);
if (result == 0)
{
const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
const double cpuStart = timing.enabled ? profile_thread_cpu_ms() : -1.0;
result = synchronize_default_stream(*feature, &timing.outputSyncBlocking, &feature->publishWaitEmaUs);
if (timing.enabled)
{
timing.outputSyncWall = profile_ms(syncStart, ProfileClock::now());
const double cpuEnd = profile_thread_cpu_ms();
if (cpuStart >= 0.0 && cpuEnd >= 0.0)
timing.outputSyncCpu = std::max(0.0, cpuEnd - cpuStart);
}
}
if (result == 0 && timing.gpuEventsRecorded)
{
float gpuEvalMs = -1.0f;
if (g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) == 0)
timing.gpuEval = gpuEvalMs;
}
if (result != 0)
{
logf("frame %u: VRAM output copy failed: %d", frame, result);
std::lock_guard<std::mutex> lock(feature->outputMutex);
feature->outputs[target].lastReadFrame = 0;
frame_retired(feature, frame);
return;
}
static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
if (arrayVerify)
{
// The result as DLSS left it in its array vs the buffer the game copies from.
std::vector<uint8_t> fromArray(rowBytes * feature->output.height), fromBuffer(fromArray.size());
CudaMemcpy2D readArray = copy;
readArray.dstMemoryType = CUDA_MEMORY_HOST;
readArray.dstHost = fromArray.data();
CudaMemcpy2D readBuffer = {};
readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
readBuffer.srcDevice = buffer.device;
readBuffer.srcPitch = rowBytes;
readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
readBuffer.dstHost = fromBuffer.data();
readBuffer.dstPitch = rowBytes;
readBuffer.WidthInBytes = rowBytes;
readBuffer.Height = feature->output.height;
if (g.cu.memcpy2D(&readArray) == 0 && g.cu.memcpy2D(&readBuffer) == 0)
{
size_t differing = 0;
for (size_t byte = 0; byte < fromArray.size(); ++byte)
differing += fromArray[byte] != fromBuffer[byte];
if (differing != 0 || frame <= 3 || frame % 120 == 0)
logf("frame %u VRAM output verify: %zu of %zu bytes differ between result array and buffer", frame,
differing, fromArray.size());
}
}
char dumpPath[MAX_PATH];
const bool saveRaw = output_dump_path(frame, dumpPath);
if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) != 0 || saveRaw)
{
HostPlane host;
uint8_t* pinned = saveRaw ? capture_pinned(dumpPath, rowBytes * feature->output.height) : nullptr;
if (pinned == nullptr)
host.fallback.resize(rowBytes * feature->output.height);
host.bytes = pinned != nullptr ? pinned : host.fallback.data();
host.width = feature->output.width;
host.height = feature->output.height;
host.rowBytes = rowBytes;
copy.dstMemoryType = CUDA_MEMORY_HOST;
copy.dstHost = host.bytes;
if (g.cu.memcpy2D(&copy) == 0)
{
log_output_hash(host, frame);
if (saveRaw && pinned != nullptr)
flush_capture_if_complete(frame); // already registered for writing
else if (saveRaw)
dump_output(host, frame, dumpPath);
}
else if (saveRaw)
logf("frame %u raw DLSS output readback failed", frame);
}
if (timing.enabled)
{
timing.d2hIssue = profile_ms(start, ProfileClock::now());
timing.download = timing.d2hIssue;
timing.worker = profile_ms(workerStart, ProfileClock::now());
}
finish_publish(feature, target, frame, params, timing, start);
}
// Worker. Streamline can create the feature at one quality's render size while the game still renders at
// another's (PRAGMATA starts with an Ultra Performance 854x480 frame on a Performance 1280x720 feature), and NGX
// rejects a subrect outside the feature's dynamic range on every frame until the game recreates it: a black
// screen. Recreate the NGX feature at the size actually rendered; the game's handle and output stay the same.
static bool recreate_for_render_size(Feature& feature, FrameParams& params, uint32_t frame)
{
if ((params.renderWidth == feature.cudaWidth && params.renderHeight == feature.cudaHeight) ||
(params.renderWidth == feature.refusedWidth && params.renderHeight == feature.refusedHeight))
return false;
logf("frame %u: NGX rejected render %ux%u on a %ux%u feature; recreating it at %ux%u", frame, params.renderWidth,
params.renderHeight, feature.cudaWidth, feature.cudaHeight, params.renderWidth, params.renderHeight);
// earlier frames' kernels may still be running on the old feature
synchronize_default_stream(feature);
if (feature.cudaHandle != nullptr)
g.ngx.releaseFeature(feature.cudaHandle);
feature.cudaHandle = nullptr;
void* p = feature.cudaParams;
d4r_ngx_set_uint(p, "Width", params.renderWidth);
d4r_ngx_set_uint(p, "Height", params.renderHeight);
const unsigned int featureId =
feature.rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING;
NgxResult created = g.ngx.createFeature(featureId, p, &feature.cudaHandle);
unsigned int width = params.renderWidth, height = params.renderHeight;
if (created != NGX_SUCCESS)
{
logf("frame %u: recreating the NGX feature at %ux%u failed (0x%08x); restoring %ux%u", frame, width, height,
created, feature.cudaWidth, feature.cudaHeight);
width = feature.cudaWidth;
height = feature.cudaHeight;
d4r_ngx_set_uint(p, "Width", width);
d4r_ngx_set_uint(p, "Height", height);
if (width != 0 && height != 0)
created = g.ngx.createFeature(featureId, p, &feature.cudaHandle);
feature.refusedWidth = created == NGX_SUCCESS ? params.renderWidth : 0;
feature.refusedHeight = created == NGX_SUCCESS ? params.renderHeight : 0;
if (created != NGX_SUCCESS)
{
logf("frame %u: restoring the NGX feature failed (0x%08x)", frame, created);
feature.cudaHandle = nullptr;
width = height = 0;
}
}
// a new feature has no history
feature.cudaWidth = width;
feature.cudaHeight = height;
params.reset = 1;
d4r_ngx_set_int(p, "Reset", 1);
return feature.cudaHandle != nullptr && width == params.renderWidth && height == params.renderHeight;
}
static bool wait_for_input_marker(Feature* feature, int slotIndex, uint32_t frame);
static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
InputSlot& slot = feature->inputs[slotIndex];
const auto start = ProfileClock::now();
if (timing.enabled)
timing.workerWait = profile_ms(timing.workerQueued, start);
FrameTiming* stages = timing.enabled ? &timing : nullptr;
const bool overlapped = async_copies() && copy_streams() != nullptr;
if (timing.markerDeferred)
{
const auto markerStart = ProfileClock::now();
if (!wait_for_input_marker(feature, slotIndex, frame))
return;
if (timing.enabled)
timing.markerWait = profile_ms(markerStart, ProfileClock::now());
}
if (params.vram && env_uint("D4R_SHIM_VRAM_VERIFY", 0) == 0)
dump_vram_inputs(slot, frame, params, params.hasExposure ? 4 : 3);
const bool linearInputs = params.vram && feature->linearInputs;
bool ok = linearInputs ? ensure_linear_inputs(slot, params.hasExposure)
: params.vram ? upload_inputs_vram(*feature, slot, params.hasExposure, frame, stages)
: overlapped
? upload_inputs_async(*feature, slot, params.hasExposure, frame, stages)
: upload_plane(Plane::Color, slot.host[0], feature->color, frame, stages, 0) &&
upload_plane(Plane::Depth, slot.host[1], feature->depth, frame, stages, 1) &&
upload_plane(Plane::Motion, slot.host[2], feature->motion, frame, stages, 2) &&
(!params.hasExposure ||
upload_plane(Plane::Exposure, slot.host[3], feature->exposure, frame, stages, 3));
ok = ok && ensure_cuda_image(feature->output, feature->outputWidth, feature->outputHeight, CUDA_FORMAT_HALF, 4,
true, false);
const auto dilationStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
const CudaObject dilatedMotion = ok ? prepare_dilated_motion(*feature, slot, params) : 0;
// Ray Reconstruction: the auxiliary surfaces are copied in the same window as the four
// main planes, so the one synchronize below covers them as well. The lock is taken only
// around the copies and, later, around the evaluate - never across the synchronize, which
// is the one long wait here and would otherwise stall the game thread's next gather.
bool auxUploaded = false;
if (ok && feature->rayReconstruction)
{
{
std::lock_guard<std::mutex> lock(feature->auxMutex);
auxUploaded = upload_aux_inputs(*feature, slotIndex, frame);
}
if (!auxUploaded)
{
slot.busy = false;
frame_retired(feature, frame);
return;
}
}
// Device-to-array copies (VRAM interop, staged uploads) return before they
// finish, and NGX does not evaluate on the stream they were issued on, so
// without this DLSS can sample a mix of this frame's and the previous
// frame's colour/depth/motion: invisible when the camera is still, beaded
// thin geometry in motion. D4R_SHIM_INPUT_SYNC=0 restores the old behaviour.
static const bool inputSync = env_uint("D4R_SHIM_INPUT_SYNC", 1) != 0;
// Linear inputs are already ready; other copies need completion before NGX samples them.
if (ok && (dilatedMotion != 0 || auxUploaded || (inputSync && !linearInputs)))
{
const int syncResult = synchronize_default_stream(*feature);
if (syncResult != 0)
{
logf("frame %u: input copy synchronize failed: %d", frame, syncResult);
ok = false;
}
}
if (dilatedMotion != 0 && ok)
{
params.motionDilation = env_uint("D4R_MOTION_DILATION", 0);
if (timing.enabled)
timing.motionDilationWall = profile_ms(dilationStart, ProfileClock::now());
}
slot.hostBusy = false; // the host planes are no longer needed
// The previous frame's download ran alongside these uploads.
complete_pending_download();
if (!ok)
{
slot.busy = false;
frame_retired(feature, frame);
return;
}
// The first frame NGX evaluates for a feature starts its history. Decided
// here, in evaluation order, so it does not depend on thread timing.
if (feature->evaluatedFrames++ == 0)
params.reset = 1;
set_evaluation_parameters(*feature, params);
// Held across the denoiser's evaluate too: the parameter object it is reading names
// &auxSurface.handle, which the game thread must not be able to move or erase meanwhile.
std::unique_lock<std::mutex> auxLock(feature->auxMutex, std::defer_lock);
if (feature->rayReconstruction)
{
auxLock.lock();
publish_aux_parameters(*feature, params, slot);
if (!publish_alpha_output(*feature, slot))
{
auxLock.unlock();
slot.busy = false;
frame_retired(feature, frame);
return;
}
}
if (linearInputs)
{
// the parameters point at these variables
feature->colorHandle = slot.linearTexture[0];
feature->depthHandle = slot.linearTexture[1];
feature->motionHandle = dilatedMotion != 0 ? dilatedMotion : slot.linearTexture[2];
feature->exposureHandle = params.hasExposure ? slot.linearTexture[3] : feature->exposureHandle;
}
static const bool outputDirect = env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0;
// The redirect is carried out by the native output kernel's stores (ZLUDA's compile of NVIDIA's kernel
// writes the array), so it is set while the previous frame's output kernel was the native one and
// undone below when this frame's was not.
const bool nativeOutput = g.cu.outputKernelNative == nullptr || g.cu.outputKernelNative() == 1;
if (outputDirect && feature->outputDirectAllowed && g.cu.setArrayRedirect != nullptr && feature->output.array != nullptr &&
(nativeOutput || feature->outputRedirected))
{
// split frames write their own slot; the staged path downloads from outputLinear
size_t rowBytes = static_cast<size_t>(feature->output.width) * 8;
const size_t bytes = rowBytes * feature->output.height;
CudaDevicePtr target = 0;
if (params.packedOutput)
rowBytes = static_cast<size_t>(feature->output.width) * 4;
if (params.vram && params.split && params.outputInPlace != 0)
{
target = params.outputInPlace;
rowBytes = params.outputInPlacePitch;
}
else if (params.vram && params.split && feature->outputs[frame % kOutputSlots].vram.bytes >= bytes)
target = feature->outputs[frame % kOutputSlots].vram.device;
if (params.packedOutput)
rowBytes |= 1; // the bridge's flag for R10G10B10A2 texels (row pitches are multiples of 8)
else if (!params.vram && overlapped && ensure_linear(feature->outputLinear, feature->outputLinearBytes, bytes))
target = feature->outputLinear;
const bool redirect = nativeOutput && target != 0 &&
g.cu.setArrayRedirect(feature->output.array, target, static_cast<uint32_t>(rowBytes)) == 0;
if (!redirect && feature->outputRedirected)
g.cu.setArrayRedirect(feature->output.array, 0, 0);
static bool logged = false;
if (redirect && !logged)
{
logf("frame %u: direct output (the native output kernel writes the shared buffer; no array copy)", frame);
logged = true;
}
feature->outputRedirected = redirect;
}
const int eventStartResult = timing.enabled && feature->profileEventsReady
? g.cu.eventRecord(feature->profileStart, nullptr) : -1;
const auto evalStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
// The core dispatches feature 13 to the denoiser it loaded, so this call is the same
// either way; only the feature id that created it differs.
NgxResult result = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
if (result == NGX_FAIL_INVALID_PARAMETER && recreate_for_render_size(*feature, params, frame))
result = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
if (auxLock.owns_lock())
auxLock.unlock();
if (feature->outputRedirected && g.cu.outputKernelNative != nullptr && g.cu.outputKernelNative() != 1)
{
// this frame's output kernel ignored the redirect: publish from the array as usual
g.cu.setArrayRedirect(feature->output.array, 0, 0);
feature->outputRedirected = false;
}
feature->packedReady = feature->outputRedirected;
const auto evalReturned = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
const int eventEndResult = eventStartResult == 0 ? g.cu.eventRecord(feature->profileEnd, nullptr) : -1;
const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
// D4R_SHIM_EVAL_SYNC=0 (VRAM interop): no wait between NGX's kernels and the output copy queued
// behind them on the same stream; publish_vram synchronises before the result is released.
static const bool evalSync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0;
const int syncResult = params.vram && !evalSync ? 0 : synchronize_default_stream(*feature, nullptr, &feature->evalWaitEmaUs);
const auto evaluated = ProfileClock::now();
const bool gpuEventsRecorded = eventStartResult == 0 && eventEndResult == 0;
float gpuEvalMs = -1.0f;
if (gpuEventsRecorded && syncResult == 0 && (!params.vram || evalSync) &&
g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) != 0)
gpuEvalMs = -1.0f;
// Keep the input slot owned until output completion, including linear reads.
if (result != NGX_SUCCESS || syncResult != 0)
{
logf("frame %u: CUDA_EvaluateFeature -> 0x%08x, sync %d", frame, result, syncResult);
slot.busy = false;
frame_retired(feature, frame);
return;
}
// The alpha surface is complete once the synchronize above has passed. A failure here drops
// the whole frame, exactly as a failed synchronize does: an accepted frame with an unwritten
// alpha would leave the caller believing it was written.
if (feature->rayReconstruction && !download_alpha_output(*feature, frame))
{
slot.busy = false;
frame_retired(feature, frame);
return;
}
if (timing.enabled)
{
timing.ngxHost = profile_ms(evalStart, evalReturned);
timing.ctxSync = profile_ms(syncStart, evaluated);
timing.gpuEval = gpuEvalMs;
timing.gpuEventsRecorded = gpuEventsRecorded && params.vram && !evalSync;
}
if (params.vram)
{
publish_vram(feature, frame, params, timing, start);
slot.busy = false; // output synchronization completed all reads of this input slot
return;
}
const int hostIndex = static_cast<int>(feature->nextOutputHost++ % 2);
if (!wait_for_release(feature->outputHostBusy[hostIndex]))
{
logf("frame %u: output host buffer still busy; dropping result", frame);
frame_retired(feature, frame);
return;
}
if (overlapped)
{
if (!start_download(*feature, feature->outputHost[hostIndex], stages))
{
frame_retired(feature, frame);
return;
}
feature->outputHostBusy[hostIndex] = true;
if (timing.enabled)
timing.worker = profile_ms(start, ProfileClock::now());
g_pendingDownload = {feature, hostIndex, frame, params, timing};
// Without a queued frame to overlap, finish it now rather than add latency.
if (g.worker.pending() == 0)
complete_pending_download(true);
return;
}
if (!download_output(*feature, feature->outputHost[hostIndex], frame, stages))
{
frame_retired(feature, frame);
return;
}
feature->outputHostBusy[hostIndex] = true;
if (timing.enabled)
timing.worker = profile_ms(start, ProfileClock::now());
post_publish(feature, hostIndex, frame, params, timing);
}
// Waits for this frame's own input marker (the GPU reached the game's input copies). false: the frame was
// dropped (retired feature, skipped submission, timeout) and has been retired here.
static bool wait_for_input_marker(Feature* feature, int slotIndex, uint32_t frame)
{
InputSlot& slot = feature->inputs[slotIndex];
const auto start = std::chrono::steady_clock::now();
// A later input marker does not prove this slot's copies were submitted:
// the game may have discarded this frame's command list. Require its own
// tag, otherwise stale input data can enter DLSS's temporal history.
const volatile uint32_t* inputMarker = feature->markerValue + 1 + slotIndex;
while (*inputMarker != frame)
{
std::atomic_thread_fence(std::memory_order_acquire);
const bool skipped = static_cast<int32_t>(*feature->markerValue - frame) > 0;
// The GPU may have filled our slot between the loop condition and
// the aggregate-marker read. Recheck before deciding it was skipped.
if (*inputMarker == frame)
break;
if (feature->retiring.load() || skipped || std::chrono::steady_clock::now() - start > std::chrono::seconds(5))
{
logf("frame %u: %s waiting for GPU marker (at %u); dropping", frame,
feature->retiring.load() ? "feature retired while" : skipped ? "input submission skipped while" : "timed out",
*inputMarker);
slot.busy = false;
frame_retired(feature, frame);
return false;
}
// D4R_SHIM_MARKER_POLL_US: poll interval (0 = yield); detection latency is on the critical path
static const unsigned markerPollUs = env_uint("D4R_SHIM_MARKER_POLL_US", 200);
if (markerPollUs == 0)
std::this_thread::yield();
else
d4r_sleep_us(markerPollUs);
}
return true;
}
static void prepare_inputs(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
InputSlot& slot = feature->inputs[slotIndex];
const auto start = ProfileClock::now();
// D4R_SHIM_DEFER_MARKER (default on): the worker waits for the game's copies itself, avoiding a thread
// wake-up between readiness and evaluation. The wait must precede every input read, including CUDA
// uploads and diagnostic readbacks; waiting only before NGX would copy stale data into its arrays.
static const bool deferMarker = env_uint("D4R_SHIM_DEFER_MARKER", 1) != 0 &&
env_uint("D4R_MOTION_DILATION", 0) == 0;
if (params.vram && deferMarker && env_uint("D4R_SHIM_VRAM_VERIFY", 0) == 0)
{
timing.markerDeferred = true;
if (timing.enabled)
{
timing.prepStart = start;
timing.prep = profile_ms(start, ProfileClock::now());
timing.workerQueued = ProfileClock::now();
}
g.worker.post([feature, slotIndex, frame, params, timing] {
run_evaluation(feature, slotIndex, frame, params, timing);
});
return;
}
if (!wait_for_input_marker(feature, slotIndex, frame))
return;
const auto ready = ProfileClock::now();
// With VRAM interop the inputs are already on the GPU; host staging runs
// only to cross-check them (D4R_SHIM_VRAM_VERIFY).
if (!params.vram || env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0)
{
if (!wait_for_release(slot.hostBusy))
{
logf("frame %u: host planes of slot %d still busy; dropping", frame, slotIndex);
slot.busy = false;
frame_retired(feature, frame);
return;
}
FrameTiming* stages = timing.enabled ? &timing : nullptr;
stage_plane(Plane::Color, slot.color, slot.host[0], stages, 0);
stage_plane(Plane::Depth, slot.depth, slot.host[1], stages, 1);
stage_plane(Plane::Motion, slot.motion, slot.host[2], stages, 2);
log_motion_stats(slot.host[2], slot.host[0], frame);
if (params.hasExposure)
stage_plane(Plane::Exposure, slot.exposure, slot.host[3], stages, 3);
char dumpDirectory[MAX_PATH];
if (input_dump_directory(frame, dumpDirectory))
dump_input_planes(slot.host, frame, params, dumpDirectory);
slot.hostBusy = true;
}
if (!params.vram)
slot.busy = false; // the readback buffers are no longer needed
if (timing.enabled)
{
timing.prepStart = start;
timing.markerWait = profile_ms(start, ready);
timing.prep = profile_ms(start, ProfileClock::now());
timing.workerQueued = ProfileClock::now();
}
g.worker.post([feature, slotIndex, frame, params, timing] {
run_evaluation(feature, slotIndex, frame, params, timing);
});
}
// Picks a result slot that no queued command list still copies from, and
// marks it so evaluate() does not present it while it is rewritten. A reader
// frame's output copy is recorded after its marker, so the GPU is past it
// only once a later frame's marker has landed. Waits briefly for that.
static int claim_output_slot(Feature* feature)
{
const auto start = std::chrono::steady_clock::now();
for (;;)
{
{
std::lock_guard<std::mutex> lock(feature->outputMutex);
const uint32_t gpuFrame = *feature->markerValue;
for (int index = 0; index < kOutputSlots; ++index)
{
if (index == feature->latestOutput.load())
continue;
if (static_cast<int32_t>(gpuFrame - feature->outputs[index].lastReadFrame.load()) > 0)
{
feature->outputs[index].lastReadFrame = 0x7fffffffu + gpuFrame;
return index;
}
}
}
if (std::chrono::steady_clock::now() - start > std::chrono::milliseconds(50))
return -1;
d4r_sleep_us(200);
}
}
// Makes `target` the result evaluate() presents and retires the frame.
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
const FrameTiming& timing, ProfileClock::time_point start)
{
{
std::lock_guard<std::mutex> lock(feature->outputMutex);
feature->outputs[target].lastReadFrame = 0;
feature->outputs[target].producedFrame = frame;
feature->latestOutput = target;
}
const uint32_t completed = ++feature->completedFrames;
frame_retired(feature, frame);
const auto finished = ProfileClock::now();
if (timing.enabled)
{
static thread_local ProfileClock::time_point lastFinished;
const double interval = lastFinished.time_since_epoch().count() != 0
? profile_ms(lastFinished, finished) : -1.0;
lastFinished = finished;
const InputSlot& slot = feature->inputs[frame % kSlots];
logf("D4R_PROFILE frame=%u render=%ux%u output=%ux%u color=%ux%u depth=%ux%u "
"motion=%ux%u exposure=%d quality=%d flags=0x%x preset=%u "
"jitter=%.6f,%.6f mv_scale=%.3f,%.3f reset=%d pre_exposure=%.6f exposure_scale=%.6f "
"frame_dt=%.3f game_interval=%.3f "
"queue_depth=%zu queue_wait=%.3f marker_wait=%.3f slot_wait=%.3f "
"d3d_input_record=%.3f d3d_output_record=%.3f game_call=%.3f "
"color_convert=%.3f depth_convert=%.3f motion_convert=%.3f exposure_convert=%.3f "
"color_h2d=%.3f depth_h2d=%.3f motion_h2d=%.3f exposure_h2d=%.3f "
"ngx_host=%.3f ctx_sync=%.3f gpu_eval=%.3f output_sync_wall=%.3f "
"output_sync_cpu=%.3f output_sync_blocking=%d output_sync_method=%s d2h=%.3f output_convert=%.3f "
"h2d_total=%.3f d2h_issue=%.3f d2h_wait=%.3f "
"prep=%.3f worker_wait=%.3f worker=%.3f finish_wait=%.3f finish=%.3f pipeline=%.3f "
"throughput_interval=%.3f presented=%u age=%d motion_dilation=%u motion_dilation_wall=%.3f",
frame, params.renderWidth, params.renderHeight, feature->outWidth, feature->outHeight,
slot.host[0].width, slot.host[0].height, slot.host[1].width, slot.host[1].height,
slot.host[2].width, slot.host[2].height, params.hasExposure, feature->quality, feature->flags,
feature->preset, params.jitterX, params.jitterY, params.mvScaleX, params.mvScaleY,
params.reset, params.preExposure, params.exposureScale, params.frameTime, timing.gameInterval,
timing.queueDepth, profile_ms(timing.queuedAt, timing.prepStart), timing.markerWait, timing.slotWait,
timing.inputRecord, timing.outputRecord, timing.gameCall,
timing.convert[0], timing.convert[1], timing.convert[2], timing.convert[3],
timing.upload[0], timing.upload[1], timing.upload[2], timing.upload[3],
timing.ngxHost, timing.ctxSync, timing.gpuEval, timing.outputSyncWall, timing.outputSyncCpu,
timing.outputSyncBlocking, timing.outputSyncBlocking ? "event_query_sleep" : "context",
timing.download, timing.outputConvert,
timing.h2dTotal, timing.d2hIssue, timing.d2hWait,
timing.prep, timing.workerWait, timing.worker, timing.finishWait, profile_ms(start, finished),
profile_ms(timing.prepStart, finished), interval,
timing.presentedFrame, timing.presentedFrame != 0 ? static_cast<int>(frame - timing.presentedFrame) : -1,
params.motionDilation, timing.motionDilationWall);
}
if (completed <= 5 || completed % 120 == 0)
logf("frame %u done (slot %d)", frame, target);
}
static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
const auto start = ProfileClock::now();
if (timing.enabled)
timing.finishWait = profile_ms(timing.finishQueued, start);
const int target = claim_output_slot(feature);
if (target < 0)
{
feature->outputHostBusy[hostIndex] = false;
logf("frame %u: no free output slot; dropping result", frame);
frame_retired(feature, frame);
return;
}
write_output(feature->outputs[target].staging, feature->outputHost[hostIndex], timing.enabled ? &timing : nullptr);
feature->outputHostBusy[hostIndex] = false;
finish_publish(feature, target, frame, params, timing, start);
}
// Waits until every frame queued so far has passed through all three stages.
static void drain_pipeline()
{
g.prep.call([] { return 0; });
g.worker.call([] {
complete_pending_download();
return 0;
});
g.finish.call([] { return 0; });
}
static void release_feature(Feature* feature);
static void destroy_recorded_resources(Feature* feature);
static void finish_shutdown();
class DeferredResources final : public IUnknown
{
public:
explicit DeferredResources(Feature* feature) : feature_(feature) { ++g.resourceOwners; }
HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, void** out) override
{
if (out == nullptr) return E_POINTER;
*out = nullptr;
if (iid != __uuidof(IUnknown)) return E_NOINTERFACE;
*out = static_cast<IUnknown*>(this);
AddRef();
return S_OK;
}
ULONG STDMETHODCALLTYPE AddRef() override { return ++refs_; }
ULONG STDMETHODCALLTYPE Release() override
{
const ULONG remaining = --refs_;
if (remaining == 0)
g.cleanup.post([this] {
destroy_recorded_resources(feature_);
delete this;
--g.resourceOwners;
finish_shutdown();
});
return remaining;
}
private:
std::atomic<ULONG> refs_{1};
Feature* feature_;
};
// --- exported NGX D3D12 API ------------------------------------------------------
#define D4R_EXPORT extern "C" __declspec(dllexport)
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init(unsigned long long applicationId, const wchar_t* dataPath,
ID3D12Device* device, const NgxFeatureCommonInfo* featureInfo,
unsigned int sdkVersion)
{
logf("NVSDK_NGX_D3D12_Init(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_Ext(unsigned long long applicationId, const wchar_t* dataPath,
ID3D12Device* device, unsigned int sdkVersion,
const NgxFeatureCommonInfo* featureInfo)
{
logf("NVSDK_NGX_D3D12_Init_Ext(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_ProjectID(const char* projectId, int engineType, const char* engineVersion,
const wchar_t* dataPath, ID3D12Device* device,
unsigned int sdkVersion, const NgxFeatureCommonInfo* featureInfo)
{
logf("NVSDK_NGX_D3D12_Init_ProjectID(project=%s, engine=%d %s, device=%p, sdk=0x%x)",
projectId != nullptr ? projectId : "", engineType, engineVersion != nullptr ? engineVersion : "", device,
sdkVersion);
ProjectIdentity project;
project.id = projectId != nullptr ? projectId : "";
project.engineType = engineType;
project.engineVersion = engineVersion != nullptr ? engineVersion : "";
return initialize(0, dataPath, device, featureInfo, sdkVersion, &project);
}
// Parameter objects handed to callers are the shim's own implementation: the
// CUDA-path objects of the official core cannot hold D3D12 resources.
static std::mutex g_parametersMutex;
static std::vector<void*> g_ownedParameters;
static void* new_parameters()
{
void* parameters = d4r_ngx_parameters_create();
std::lock_guard<std::mutex> lock(g_parametersMutex);
g_ownedParameters.push_back(parameters);
return parameters;
}
// Copies the official capability values (availability, driver requirements,
// and the optimal-settings/stats callbacks, which work on any parameter
// implementation) into one of our parameter objects.
// Reads the denoising family out of the core's own capability parameters. Worker only.
static void refresh_denoiser_capability()
{
void* official = nullptr;
if (g.ngx.getCapabilityParameters == nullptr || g.ngx.getCapabilityParameters(&official) != NGX_SUCCESS ||
official == nullptr)
return;
DenoiserCapability capability;
int available = 0;
capability.available =
d4r_ngx_get_int(official, "SuperSamplingDenoising.Available", &available) == NGX_SUCCESS ? available : 0;
int initResult = 0;
capability.initResult =
d4r_ngx_get_int(official, "SuperSamplingDenoising.FeatureInitResult", &initResult) == NGX_SUCCESS
? initResult
: static_cast<int>(NGX_FAIL_NOT_INITIALIZED);
capability.known = true;
g.ngx.destroyParameters(official);
std::lock_guard<std::mutex> lock(g_denoiserCapabilityMutex);
g_denoiserCapability = capability;
}
static NgxResult fill_capabilities(void* parameters)
{
return g.worker.call([&]() -> NgxResult {
void* official = nullptr;
const NgxResult result = g.ngx.getCapabilityParameters(&official);
if (result != NGX_SUCCESS || official == nullptr)
return result;
const char* integers[] = {"SuperSampling.Available", "SuperSampling.NeedsUpdatedDriver",
"SuperSampling.FeatureInitResult"};
for (const char* name : integers)
{
int value = 0;
if (d4r_ngx_get_int(official, name, &value) == NGX_SUCCESS)
d4r_ngx_set_int(parameters, name, value);
}
const char* unsignedValues[] = {"SuperSampling.MinDriverVersionMajor", "SuperSampling.MinDriverVersionMinor"};
for (const char* name : unsignedValues)
{
unsigned int value = 0;
if (d4r_ngx_get_uint(official, name, &value) == NGX_SUCCESS)
d4r_ngx_set_uint(parameters, name, value);
}
// DLSSDOptimalSettingsCallback is not optional decoration: the SDK's helpers call it to
// work out the render dimensions, so a Ray Reconstruction route that dropped it would
// misbehave in a way that looks like a game bug. Forwarded exactly as the Super
// Sampling pair already is - no new ABI is invented here.
const char* pointers[] = {"DLSSOptimalSettingsCallback", "DLSSGetStatsCallback",
"DLSSDOptimalSettingsCallback", "DLSSDGetStatsCallback"};
for (const char* name : pointers)
{
void* value = nullptr;
if (d4r_ngx_get_void(official, name, &value) == NGX_SUCCESS && value != nullptr)
d4r_ngx_set_void(parameters, name, value);
}
// The denoising family is the core's own account of the denoiser it loaded, so it is
// passed through rather than re-derived here - and cached, because it is also what
// decides whether Ray Reconstruction may be created at all.
refresh_denoiser_capability();
const char* denoisingInts[] = {"SuperSamplingDenoising.Available",
"SuperSamplingDenoising.NeedsUpdatedDriver",
"SuperSamplingDenoising.FeatureInitResult"};
for (const char* name : denoisingInts)
{
int value = 0;
if (d4r_ngx_get_int(official, name, &value) == NGX_SUCCESS)
d4r_ngx_set_int(parameters, name, value);
}
const char* denoisingUnsigned[] = {"SuperSamplingDenoising.MinDriverVersionMajor",
"SuperSamplingDenoising.MinDriverVersionMinor"};
for (const char* name : denoisingUnsigned)
{
unsigned int value = 0;
if (d4r_ngx_get_uint(official, name, &value) == NGX_SUCCESS)
d4r_ngx_set_uint(parameters, name, value);
}
// D4R_RR_ENABLE=0 is this file's decision and outranks what the core reports: with the
// route switched off, availability reads 0 and the callbacks are cleared, so a game
// cannot reach a denoiser route that is not being offered.
if (!rr_enabled())
{
d4r_ngx_set_int(parameters, "SuperSamplingDenoising.Available", 0);
d4r_ngx_set_int(parameters, "SuperSamplingDenoising.FeatureInitResult",
static_cast<int>(NGX_FAIL_FEATURE_NOT_SUPPORTED));
d4r_ngx_set_void(parameters, "DLSSDOptimalSettingsCallback", nullptr);
d4r_ngx_set_void(parameters, "DLSSDGetStatsCallback", nullptr);
}
const DenoiserCapability capability = denoiser_capability();
logf("denoiser capabilities from the core: available=%d init=0x%08x (Ray Reconstruction %s)",
capability.available, static_cast<unsigned int>(capability.initResult),
rr_denial_reason());
g.ngx.destroyParameters(official);
return NGX_SUCCESS;
});
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetParameters(void** parameters)
{
if (!g.ngxInitialized)
return NGX_FAIL_NOT_INITIALIZED;
*parameters = new_parameters();
return fill_capabilities(*parameters);
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_AllocateParameters(void** parameters)
{
if (!g.ngxInitialized)
return NGX_FAIL_NOT_INITIALIZED;
*parameters = new_parameters();
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetCapabilityParameters(void** parameters)
{
if (!g.ngxInitialized)
return NGX_FAIL_NOT_INITIALIZED;
*parameters = new_parameters();
const NgxResult result = fill_capabilities(*parameters);
int available = -1, initResult = -1, needsDriver = -1, denoising = -1, denoisingInit = -1;
d4r_ngx_get_int(*parameters, "SuperSampling.Available", &available);
d4r_ngx_get_int(*parameters, "SuperSampling.FeatureInitResult", &initResult);
d4r_ngx_get_int(*parameters, "SuperSampling.NeedsUpdatedDriver", &needsDriver);
d4r_ngx_get_int(*parameters, "SuperSamplingDenoising.Available", &denoising);
d4r_ngx_get_int(*parameters, "SuperSamplingDenoising.FeatureInitResult", &denoisingInit);
logf("NVSDK_NGX_D3D12_GetCapabilityParameters -> 0x%08x, SuperSampling.Available=%d FeatureInitResult=0x%08x "
"NeedsUpdatedDriver=%d, SuperSamplingDenoising.Available=%d FeatureInitResult=0x%08x",
result, available, static_cast<unsigned int>(initResult), needsDriver, denoising,
static_cast<unsigned int>(denoisingInit));
return result;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_DestroyParameters(void* parameters)
{
{
std::lock_guard<std::mutex> lock(g_parametersMutex);
for (auto it = g_ownedParameters.begin(); it != g_ownedParameters.end(); ++it)
if (*it == parameters)
{
g_ownedParameters.erase(it);
d4r_ngx_parameters_destroy(parameters);
return NGX_SUCCESS;
}
}
return NGX_FAIL_INVALID_PARAMETER;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetScratchBufferSize(unsigned int, const void*, size_t* size)
{
// Scratch memory for the CUDA path is allocated internally.
if (size != nullptr)
*size = 0;
return NGX_SUCCESS;
}
// nvsdk_ngx.h: NVSDK_NGX_D3D12_GetFeatureRequirements(IDXGIAdapter *Adapter,
// const NVSDK_NGX_FeatureDiscoveryInfo *FeatureDiscoveryInfo,
// NVSDK_NGX_FeatureRequirement *OutSupported). The feature is NOT a separate argument: it is
// FeatureDiscoveryInfo->FeatureID (defs.h, the field after SDKVersion), and the first argument
// is the display adapter, which must never be read as a feature id.
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetFeatureRequirements(void* adapter, const void* discovery,
NgxFeatureRequirement* requirement)
{
if (requirement == nullptr)
return NGX_SUCCESS;
// NVSDK_NGX_FeatureDiscoveryInfo: { NVSDK_NGX_Version SDKVersion; NVSDK_NGX_Feature FeatureID; ... },
// both enums, so FeatureID is the unsigned int at offset 4.
unsigned int feature = 0;
if (discovery != nullptr)
std::memcpy(&feature, static_cast<const char*>(discovery) + sizeof(unsigned int), sizeof(feature));
std::memset(requirement, 0, sizeof(*requirement));
ensure_portable_config();
// The local switch is checked before the core is asked: with D4R_RR_ENABLE=0 this process
// is not offering Ray Reconstruction, and the core's authoritative "supported" must not
// be passed on to a game that would then create a feature this shim would refuse.
if (feature == NGX_FEATURE_RAY_RECONSTRUCTION && !rr_enabled())
{
requirement->FeatureSupported = 1; // not supported
logf("Ray Reconstruction is not supported: %s", rr_denial_reason());
return NGX_SUCCESS;
}
{
std::lock_guard<std::mutex> lock(g.mutex);
// Requirements must not depend on a prior Init. Load the provider without
// initializing NGX here: the later Init retains the game's identity and device.
if (ensure_workers_started() != NGX_SUCCESS || (g.core == nullptr && !load_libraries()))
return NGX_FAIL_PLATFORM_ERROR;
if (feature == NGX_FEATURE_RAY_RECONSTRUCTION)
load_denoiser_library();
}
// Otherwise the core's answer is authoritative - it consulted the denoiser it loaded - and
// this file only steps in when the core declines.
if (g.ngx.getFeatureRequirements != nullptr)
{
// The adapter this export receives is not what the core's CUDA entry point wants: it
// takes a CUdevice ordinal. Passing the adapter pointer through would read its low 32
// bits as an ordinal, so the current context's device is asked for instead.
int device = 0;
const NgxResult result = g.worker.call([&] {
if (g.cu.ctxGetDevice != nullptr && g.cu.ctxGetDevice(&device) != 0)
device = 0; // no current context: device 0 is this process's only CUDA device
return g.ngx.getFeatureRequirements(device, discovery, requirement);
});
if (result == NGX_SUCCESS)
{
logf("GetFeatureRequirements(feature=%u, cuda device %d) from the NGX core: supported=%u minHW=0x%x "
"'%s'",
feature, device, requirement->FeatureSupported, requirement->MinHWArchitecture,
requirement->MinOSVersion);
return result;
}
logf("GetFeatureRequirements(feature=%u) from the NGX core -> 0x%08x; answering here instead", feature,
result);
}
if (feature == NGX_FEATURE_RAY_RECONSTRUCTION)
{
// Requirements are legal before Init. Ask the denoiser's discovery export,
// not its Init stub, without binding NGX to an application prematurely.
if (!g.ngxInitialized && g.denoiser.cudaCapable)
{
using RequirementsFn = NgxResult (*)(void*, const void*, NgxFeatureRequirement*);
RequirementsFn denoiserRequirements = nullptr;
if (load_export(g.denoiserModule, "NVSDK_NGX_D3D12_GetFeatureRequirements", denoiserRequirements))
{
const NgxResult result = g.worker.call([&] {
return denoiserRequirements(adapter, discovery, requirement);
});
logf("GetFeatureRequirements before Init from DLSS-Denoiser -> 0x%08x, supported=%u",
result, requirement->FeatureSupported);
return result;
}
}
const bool supported = ray_reconstruction_available();
if (!supported)
{
requirement->FeatureSupported = 1;
logf("Ray Reconstruction is not supported: %s",
g.ngxInitialized ? rr_denial_reason() : "no CUDA-capable DLSS-Denoiser for pre-Init discovery");
return NGX_SUCCESS;
}
}
if (feature != NGX_FEATURE_SUPER_SAMPLING && feature != NGX_FEATURE_RAY_RECONSTRUCTION)
{
requirement->FeatureSupported = 1; // not supported
return NGX_SUCCESS;
}
requirement->FeatureSupported = 0; // supported
requirement->MinHWArchitecture = 0x160;
return NGX_SUCCESS;
}
static unsigned int get_uint_or(void* parameters, const char* name, unsigned int fallback)
{
unsigned int value = fallback;
if (d4r_ngx_get_uint(parameters, name, &value) != NGX_SUCCESS)
{
int signedValue = 0;
if (d4r_ngx_get_int(parameters, name, &signedValue) == NGX_SUCCESS)
return static_cast<unsigned int>(signedValue);
return fallback;
}
return value;
}
static int get_int_or(void* parameters, const char* name, int fallback)
{
int value = fallback;
if (d4r_ngx_get_int(parameters, name, &value) != NGX_SUCCESS)
{
unsigned int unsignedValue = 0;
if (d4r_ngx_get_uint(parameters, name, &unsignedValue) == NGX_SUCCESS)
return static_cast<int>(unsignedValue);
return fallback;
}
return value;
}
static float get_float_or(void* parameters, const char* name, float fallback)
{
float value = fallback;
return d4r_ngx_get_float(parameters, name, &value) == NGX_SUCCESS ? value : fallback;
}
static bool engine_requested();
// Called on the game thread with the feature API lock held. All CUDA resources are made on its worker.
// Vulkan-only K features retain just their creation settings until a frame actually needs CUDA.
static NgxResult create_cuda_feature(Feature& feature)
{
static const char* const kSuperSamplingPresets[] = {
"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance",
"DLSS.Hint.Render.Preset.UltraQuality"};
static const char* const kRayReconstructionPresets[] = {
"RayReconstruction.Hint.Render.Preset.DLAA", "RayReconstruction.Hint.Render.Preset.Quality",
"RayReconstruction.Hint.Render.Preset.Balanced", "RayReconstruction.Hint.Render.Preset.Performance",
"RayReconstruction.Hint.Render.Preset.UltraPerformance", "RayReconstruction.Hint.Render.Preset.UltraQuality"};
const bool rayReconstruction = feature.rayReconstruction;
const char* const* presetNames = rayReconstruction ? kRayReconstructionPresets : kSuperSamplingPresets;
const NgxResult result = g.worker.call([&]() -> NgxResult {
NgxResult allocation = g.ngx.allocateParameters(&feature.cudaParams);
if (allocation != NGX_SUCCESS)
return allocation;
void* p = feature.cudaParams;
if (rayReconstruction)
{
// The CUDA denoiser reads creation dimensions through Get(int*),
// unlike Super Sampling's Get(unsigned int*) contract.
d4r_ngx_set_int(p, "Width", static_cast<int>(feature.width));
d4r_ngx_set_int(p, "Height", static_cast<int>(feature.height));
d4r_ngx_set_int(p, "OutWidth", static_cast<int>(feature.outWidth));
d4r_ngx_set_int(p, "OutHeight", static_cast<int>(feature.outHeight));
}
else
{
d4r_ngx_set_uint(p, "Width", feature.width);
d4r_ngx_set_uint(p, "Height", feature.height);
d4r_ngx_set_uint(p, "OutWidth", feature.outWidth);
d4r_ngx_set_uint(p, "OutHeight", feature.outHeight);
}
d4r_ngx_set_int(p, "PerfQualityValue", feature.quality);
d4r_ngx_set_int(p, "DLSS.Feature.Create.Flags", feature.flags);
d4r_ngx_set_int(p, "DLSS.Enable.Output.Subrects", feature.subrects);
d4r_ngx_set_uint(p, "CreationNodeMask", 1);
d4r_ngx_set_uint(p, "VisibilityNodeMask", 1);
for (int index = 0; index < 6; ++index)
d4r_ngx_set_uint(p, presetNames[index], feature.creationPresets[index]);
if (rayReconstruction)
for (size_t index = 0; index < sizeof(kAuxScalars) / sizeof(kAuxScalars[0]); ++index)
d4r_ngx_set_int(p, kAuxScalars[index].name, feature.auxScalarBaseline[index]);
if (g.cu.memAlloc(&feature.scratch, 64ull * 1024 * 1024) == 0)
{
d4r_ngx_set_void(p, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature.scratch)));
d4r_ngx_set_ull(p, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
}
else
feature.scratch = 0;
// The core owns the denoiser: it loads nvngx_dlssd.dll, knows its kernel map, and
// dispatches feature 13 to it. Only the feature id differs from Super Sampling.
const NgxResult created = g.ngx.createFeature(
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, p, &feature.cudaHandle);
if (created != NGX_SUCCESS)
{
if (feature.cudaHandle != nullptr) g.ngx.releaseFeature(feature.cudaHandle);
feature.cudaHandle = nullptr;
g.ngx.destroyParameters(feature.cudaParams); feature.cudaParams = nullptr;
if (feature.scratch != 0) g.cu.memFree(feature.scratch);
feature.scratch = 0;
return created;
}
feature.cudaWidth = feature.width;
feature.cudaHeight = feature.height;
if (created == NGX_SUCCESS && env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0 &&
g.cu.eventCreate != nullptr && g.cu.eventRecord != nullptr &&
g.cu.eventSynchronize != nullptr && g.cu.eventQuery != nullptr && g.cu.eventDestroy != nullptr)
{
// Explicit query/sleep waits avoid HIP's spinning event-sync path.
// Timing is unnecessary for this completion event.
constexpr unsigned int kBlockingSyncNoTiming = 0x1u | 0x2u;
const int eventResult = g.cu.eventCreate(&feature.outputReadyEvent, kBlockingSyncNoTiming);
if (eventResult != 0)
{
logf("blocking output wait unavailable (cuEventCreate %d); using context sync", eventResult);
feature.outputReadyEvent = nullptr;
}
else
logf("output event query/sleep wait enabled for this feature");
}
if (created == NGX_SUCCESS && profile_enabled() && g.cu.eventCreate != nullptr &&
g.cu.eventRecord != nullptr && g.cu.eventElapsedTime != nullptr && g.cu.eventDestroy != nullptr)
{
const int startResult = g.cu.eventCreate(&feature.profileStart, 0);
const int endResult = startResult == 0 ? g.cu.eventCreate(&feature.profileEnd, 0) : -1;
feature.profileEventsReady = startResult == 0 && endResult == 0;
if (!feature.profileEventsReady)
{
logf("D4R_PROFILE: CUDA event creation failed (%d, %d)", startResult, endResult);
if (feature.profileStart != nullptr)
g.cu.eventDestroy(feature.profileStart);
if (feature.profileEnd != nullptr)
g.cu.eventDestroy(feature.profileEnd);
feature.profileStart = feature.profileEnd = nullptr;
}
}
return created;
});
logf("NVSDK_NGX_CUDA_CreateFeature(feature=%u) -> 0x%08x",
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, result);
if (result == NGX_SUCCESS) feature.cudaCreationDeferred = false;
return result;
}
static NgxResult create_feature(unsigned int featureId, void* parameters, NgxHandle** handle, bool nativeVulkan)
{
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
std::lock_guard<std::mutex> stateLock(g.mutex);
if (!g.ngxInitialized)
return NGX_FAIL_NOT_INITIALIZED;
if (!nativeVulkan && g.device == nullptr)
return NGX_FAIL_NOT_INITIALIZED;
if (!nativeVulkan)
std::call_once(g_vk.once, init_vram_interop);
if (!nativeVulkan && g_vk.lifetime == nullptr)
{
logf("DLSS requires lifetime-aware d4r d3d12.dll/d3d12core.dll; update both files with the shim");
return NGX_FAIL_PLATFORM_ERROR;
}
// Two features are served here. Super Sampling is feature 1 and Ray Reconstruction
// feature 13; the NGX core dispatches the second to the denoiser it loaded, and is
// offered feature 13 only once it has reported that denoiser working.
const bool rayReconstruction = featureId == NGX_FEATURE_RAY_RECONSTRUCTION;
if (featureId != NGX_FEATURE_SUPER_SAMPLING && !rayReconstruction)
{
logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u) unsupported", featureId);
return NGX_FAIL_FEATURE_NOT_SUPPORTED;
}
if (rayReconstruction)
{
// A game may create feature 13 without ever asking for capabilities, so the core's
// account of the denoiser is fetched here if it has not been already.
g.worker.call([] { refresh_denoiser_capability(); return 0; });
}
if (rayReconstruction && !ray_reconstruction_available())
{
logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u) unavailable: %s", featureId, rr_denial_reason());
return NGX_FAIL_FEATURE_NOT_SUPPORTED;
}
if (parameters == nullptr || handle == nullptr)
{
logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u): no parameters or handle", featureId);
return NGX_FAIL_INVALID_PARAMETER;
}
auto* feature = new Feature();
feature->nativeVulkan = nativeVulkan;
feature->width = get_uint_or(parameters, "Width", 0);
feature->height = get_uint_or(parameters, "Height", 0);
feature->outWidth = get_uint_or(parameters, "OutWidth", 0);
feature->outHeight = get_uint_or(parameters, "OutHeight", 0);
const int quality = get_int_or(parameters, "PerfQualityValue", 0);
const int flags = get_int_or(parameters, "DLSS.Feature.Create.Flags", 0);
feature->quality = quality;
feature->flags = flags;
const int subrects = get_int_or(parameters, "DLSS.Enable.Output.Subrects", 0);
// The render preset is chosen by two independent knobs. D4R_DLSS_PRESET forces the
// Super Sampling model (NVSDK_NGX_DLSS_Hint_Render_Preset, e.g. 5 = E) and touches only
// Super Sampling features; D4R_RR_PRESET forces the Ray Reconstruction model
// (NVSDK_NGX_RayReconstruction_Hint_Render_Preset, D = 4, E = 5) and touches only
// Ray Reconstruction features. Neither one silently becomes the other.
static const char* const kSuperSamplingPresets[] = {
"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance",
"DLSS.Hint.Render.Preset.UltraQuality"};
static const char* const kRayReconstructionPresets[] = {
"RayReconstruction.Hint.Render.Preset.DLAA", "RayReconstruction.Hint.Render.Preset.Quality",
"RayReconstruction.Hint.Render.Preset.Balanced", "RayReconstruction.Hint.Render.Preset.Performance",
"RayReconstruction.Hint.Render.Preset.UltraPerformance", "RayReconstruction.Hint.Render.Preset.UltraQuality"};
const char* const* presetNames = rayReconstruction ? kRayReconstructionPresets : kSuperSamplingPresets;
unsigned int presets[6];
for (int index = 0; index < 6; ++index)
presets[index] = get_uint_or(parameters, presetNames[index], 0);
const std::string forced = env_string(rayReconstruction ? "D4R_RR_PRESET" : "D4R_DLSS_PRESET");
if (!forced.empty())
{
const unsigned int value = static_cast<unsigned int>(strtoul(forced.c_str(), nullptr, 0));
const char* model = value == 5 ? "E" : value == 11 ? "K" : value == 12 ? "L" : value == 13 ? "M" : "custom";
logf("DLSS model override: D4R_DLSS_PRESET=%s (model %s) overrides game/OptiScaler presets "
"[DLAA=%u Quality=%u Balanced=%u Performance=%u UltraPerformance=%u UltraQuality=%u]; "
"using preset=%u for all quality modes",
forced.c_str(), model, presets[0], presets[1], presets[2], presets[3], presets[4], presets[5], value);
for (unsigned int& preset : presets)
preset = value;
}
feature->preset = presets[1];
// Direct output needs the native output kernel of the preset in use (verified: K = 11 through
// hiluma_engine_output, M = 13 through rrlite_downsample_kernel, its last kernel). Only a forced
// Super Sampling preset qualifies; a Ray Reconstruction preset never does here.
if (!forced.empty() && !rayReconstruction)
{
char allowed[128] = "11,13";
if (const std::string list = env_string("D4R_SHIM_OUTPUT_DIRECT_PRESETS"); !list.empty())
snprintf(allowed, sizeof(allowed), "%s", list.c_str());
for (char* token = strtok(allowed, ","); token != nullptr; token = strtok(nullptr, ","))
feature->outputDirectAllowed |= strtoul(token, nullptr, 0) == presets[1];
}
feature->rayReconstruction = rayReconstruction;
if (rayReconstruction)
for (size_t index = 0; index < sizeof(kAuxScalars) / sizeof(kAuxScalars[0]); ++index)
feature->auxScalarBaseline[index] = get_int_or(parameters, kAuxScalars[index].name, 0);
logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u%s): %ux%u -> %ux%u quality=%d flags=0x%x subrects=%d preset=%u",
featureId, rayReconstruction ? ", Ray Reconstruction" : "", feature->width, feature->height,
feature->outWidth, feature->outHeight, quality, flags, subrects, presets[1]);
feature->subrects = subrects;
std::copy(std::begin(presets), std::end(presets), feature->creationPresets.begin());
// The engine records D3D12 evaluations only; the native Vulkan frontend keeps eager CUDA creation. Presets K, L
// and M (the engine's) create the CUDA feature only when a frame falls back: NGX's L kernels take ZLUDA about 20 s
// to compile on a cold cache, which stalled the game inside CreateFeature (Overwatch, 2026-10-11).
feature->cudaCreationDeferred = !nativeVulkan && engine_requested() &&
(feature->preset == 11 || feature->preset == 12 || feature->preset == 13);
const NgxResult result = feature->cudaCreationDeferred ? NGX_SUCCESS : create_cuda_feature(*feature);
if (feature->cudaCreationDeferred)
logf("engine backend: CUDA feature creation deferred until fallback is needed");
if (result != NGX_SUCCESS)
{
release_feature(feature);
return result;
}
if (!nativeVulkan && !create_buffer(D3D12_HEAP_TYPE_READBACK, 256, &feature->marker,
reinterpret_cast<uint8_t**>(const_cast<uint32_t**>(&feature->markerValue))))
{
release_feature(feature);
return NGX_FAIL_PLATFORM_ERROR;
}
if (!nativeVulkan)
{
for (int index = 0; index <= kSlots; ++index)
feature->markerValue[index] = 0;
feature->resources = new DeferredResources(feature);
}
feature->handle.Id = g.nextHandleId++;
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
g_features.push_back(feature);
}
*handle = &feature->handle;
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned int featureId,
void* parameters, NgxHandle** handle)
{
return create_feature(featureId, parameters, handle, false);
}
static Feature* find_feature(const NgxHandle* handle)
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
for (Feature* feature : g_features)
if (&feature->handle == handle && !feature->retiring.load())
return feature;
return nullptr;
}
// --- Native engine backend (engine/): no NGX evaluation, CUDA, worker or frame split -------------
// [Engine] Enabled in d4r.ini (D4R_ENGINE=1) runs presets K and M with d4r's own shaders inside the game's command
// list. The models (the output of engine/compile_k.py / compile_m.py) are the folders engine\k, engine\k-ldr,
// engine\m and engine\m-ldr next to this DLL, or [Engine] ModelDir (D4R_ENGINE_MODEL_DIR) and the
// D4R_ENGINE_MODEL_{LDR,M,M_LDR}_DIR variables. Anything the engine does not cover falls back to the CUDA backend.
struct EngineBackend
{
std::once_flag vulkanOnce; // the game's device and the engine's Vulkan entry points, for every preset
std::once_flag once;
std::unique_ptr<d4r::Model> model; // shared by all features, kept for the life of the process
// The same network with NGX's LDR input and output stages, for games that do not set the HDR flag
// (engine\\k-ldr next to this DLL, or D4R_ENGINE_MODEL_LDR_DIR); loaded on first use.
std::once_flag ldrOnce;
std::unique_ptr<d4r::Model> modelLdr;
std::string directory, directoryLdr; // kept for the native HIP network's hipnet.bin and hip/*.hsaco
};
static EngineBackend g_engine;
static bool engine_requested()
{
static const bool requested = env_uint("D4R_ENGINE", 0) != 0;
return requested;
}
static void init_engine_vulkan()
{
VkInstance instance = VK_NULL_HANDLE;
if (g_vk.interop == nullptr || g_vk.lifetime == nullptr ||
FAILED(g_vk.interop->GetVulkanHandles(&instance, &g_vk.physical, &g_vk.device)))
{
logf("engine backend: the vkd3d-proton interop interfaces are unavailable");
return;
}
HMODULE vulkan = GetModuleHandleA("vulkan-1.dll");
PFN_vkGetInstanceProcAddr instanceProc = nullptr;
if (vulkan == nullptr || !load_export(vulkan, "vkGetInstanceProcAddr", instanceProc) ||
!d4r::loadVulkan(instanceProc, instance, g_vk.device))
{
logf("engine backend: Vulkan entry points are unavailable");
return;
}
}
static void init_engine_backend()
{
// SetEnvironmentVariable (d4r.ini) does not reach the C runtime's getenv
std::string directory = env_string("D4R_ENGINE_MODEL_DIR");
if (directory.empty())
{
char folder[MAX_PATH * 3] = {};
WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + L"\\engine\\k").c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
directory = folder;
}
if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
return;
// A Unix path names the same file through Wine's Z: drive.
const std::string path = directory[0] == '/' ? "Z:" + directory : directory;
const auto start = std::chrono::steady_clock::now();
try
{
g_engine.model = std::make_unique<d4r::Model>(g_vk.physical, g_vk.device, path);
g_engine.directory = path;
}
catch (const std::exception& error)
{
logf("engine backend: cannot load the model from %s: %s (see D4R_README.txt / engine/README.md for how to "
"create it)", path.c_str(), error.what());
return;
}
logf("engine backend: model %s loaded and pipelines compiled in %.0f ms", path.c_str(),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
}
static void init_engine_backend_ldr()
{
std::string directory = env_string("D4R_ENGINE_MODEL_LDR_DIR");
if (directory.empty())
{
char folder[MAX_PATH * 3] = {};
WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + L"\\engine\\k-ldr").c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
directory = folder;
}
if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
return;
const std::string path = directory[0] == '/' ? "Z:" + directory : directory;
try
{
g_engine.modelLdr = std::make_unique<d4r::Model>(g_vk.physical, g_vk.device, path);
g_engine.directoryLdr = path;
logf("engine backend: LDR model %s loaded", path.c_str());
}
catch (const std::exception& error)
{
logf("engine backend: cannot load the LDR model from %s: %s", path.c_str(), error.what());
}
}
// K and M borrow compatible views through the same bounded, resource-owning cache.
static void prepare_engine_view(Feature& feature, ID3D12Resource* resource, const D3D12_RESOURCE_DESC& desc,
d4r::GameImage& image, bool isOutput, bool originSupported,
D3D12_RESOURCE_STATES& state)
{
if (image.format == VK_FORMAT_D32_SFLOAT || image.format == VK_FORMAT_D32_SFLOAT_S8_UINT)
image.aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
const VkImageUsageFlags usage = isOutput ? VK_IMAGE_USAGE_STORAGE_BIT : VK_IMAGE_USAGE_SAMPLED_BIT;
VkFormatProperties properties{};
d4r::vkTable.GetPhysicalDeviceFormatProperties(g_vk.physical, image.format, &properties);
const bool enabled = env_uint(isOutput ? "D4R_ENGINE_DIRECT_OUTPUT" : "D4R_ENGINE_DIRECT_INPUTS", 1) != 0;
const bool compatible = desc.DepthOrArraySize == 1 && originSupported && (isOutput ?
(image.format == VK_FORMAT_R16G16B16A16_SFLOAT || image.format == VK_FORMAT_A2B10G10R10_UNORM_PACK32) &&
(desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS) &&
(properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) :
!(desc.Flags & D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE) &&
(properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT));
if (!enabled || !compatible)
return;
auto found = std::find_if(feature.engineViews.begin(), feature.engineViews.end(), [&](const auto& view) {
return view->resource == resource && view->format == image.format && view->aspect == image.aspect &&
view->usage == usage;
});
if (found == feature.engineViews.end())
{
VkMemoryRequirements memory{};
d4r::vkTable.GetImageMemoryRequirements(g_vk.device, image.image, &memory);
constexpr uint64_t cacheBudget = 512ull * 1024 * 1024;
if (feature.engineViews.size() < 64 && memory.size <= cacheBudget - feature.engineViewBytes)
{
auto view = std::make_unique<EngineBorrowedView>();
view->device = g_vk.device; view->format = image.format;
view->aspect = image.aspect; view->usage = usage;
VkImageViewUsageCreateInfo restricted{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; restricted.usage = usage;
VkImageViewCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, &restricted};
info.image = image.image; info.viewType = VK_IMAGE_VIEW_TYPE_2D; info.format = view->format;
info.subresourceRange = {view->aspect, 0, 1, 0, 1};
if (d4r::vkTable.CreateImageView(g_vk.device, &info, nullptr, &view->view) == VK_SUCCESS)
{
resource->AddRef(); view->resource = resource;
feature.engineViews.push_back(std::move(view));
feature.engineViewBytes += memory.size;
found = feature.engineViews.end() - 1;
}
}
else if (!feature.engineViewBudgetReported)
{
feature.engineViewBudgetReported = true;
logf("engine view cache budget reached; new textures use copies until feature retirement");
}
}
if (found != feature.engineViews.end())
{
image.view = (*found)->view;
state = isOutput ? D3D12_RESOURCE_STATE_UNORDERED_ACCESS : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE;
}
}
// Presets M and L on the engine (both run d4r::MEngine; the model directory decides which): a model per preset and
// range, engine\\m, engine\\m-ldr, engine\\l and engine\\l-ldr next to this DLL, or D4R_ENGINE_MODEL_{M,L}[_LDR]_DIR.
struct EngineBackendM
{
std::once_flag once[2][2]; // [preset L][LDR]
std::unique_ptr<d4r::MModel> models[2][2];
std::string directories[2][2];
};
static EngineBackendM g_engineM;
static void init_engine_backend_m_variant(bool presetL, bool ldr)
{
const char* const name = presetL ? "L" : "M";
std::unique_ptr<d4r::MModel>& slot = g_engineM.models[presetL][ldr];
std::string directory = env_string(presetL ? (ldr ? "D4R_ENGINE_MODEL_L_LDR_DIR" : "D4R_ENGINE_MODEL_L_DIR")
: (ldr ? "D4R_ENGINE_MODEL_M_LDR_DIR" : "D4R_ENGINE_MODEL_M_DIR"));
if (directory.empty())
{
char folder[MAX_PATH * 3] = {};
const wchar_t* const sub = presetL ? (ldr ? L"\\engine\\l-ldr" : L"\\engine\\l") : (ldr ? L"\\engine\\m-ldr" : L"\\engine\\m");
WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + sub).c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
directory = folder;
}
if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
{
logf("engine backend (%s): Vulkan interop is unavailable", name);
return;
}
const std::string path = directory[0] == '/' ? "Z:" + directory : directory;
const auto start = std::chrono::steady_clock::now();
try
{
auto model = std::make_unique<d4r::MModel>(g_vk.physical, g_vk.device, path);
if (model->isPresetL() != presetL)
throw std::runtime_error(presetL ? "the directory holds a preset M model" : "the directory holds a preset L model");
slot = std::move(model);
g_engineM.directories[presetL][ldr] = path;
}
catch (const std::exception& error)
{
logf("engine backend (%s): cannot load the model from %s: %s", name, path.c_str(), error.what());
return;
}
logf("engine backend (%s): model %s loaded and pipelines compiled in %.0f ms", name, path.c_str(),
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count());
}
// Preset M's network on d4r's native HIP layer kernels (bridge: d4rEngineNet*; engine/hip_net.h). HIP runs the
// network about 1.7x faster than the engine's cooperative-matrix shader; the stages around it stay on Vulkan in
// the game's command list, which is split at the network like a CUDA split frame. The ten layers wait on the GPU
// for the front's fill of the frame number (d4rEngineNetInputWaitReady/d4rEngineNetLaunchAt, D4R_SHIM_ENGINE_GPU_WAIT)
// when the bridge supports it; otherwise the worker waits for the CPU-visible marker before launching.
struct EngineNetCode
{
const char* name;
const void* data;
uint64_t bytes;
};
struct EngineNetExports
{
std::once_flag once;
int(WINAPI* create)(const uint8_t*, uint64_t, const EngineNetCode*, uint32_t, uint32_t, uint32_t, CudaDevicePtr,
CudaDevicePtr, void**) = nullptr;
int(WINAPI* launch)(void*, uint32_t) = nullptr;
int(WINAPI* done)(void*, float*) = nullptr;
void(WINAPI* destroy)(void*) = nullptr;
// GPU-side readiness: the front fills a u32; the HIP stream waits on it instead of a CPU-visible marker.
int(WINAPI* inputWaitReady)(void*, CudaDevicePtr, uint32_t*) = nullptr;
int(WINAPI* launchAt)(void*, uint32_t) = nullptr;
int(WINAPI* releaseInput)(void*, uint32_t) = nullptr;
int(WINAPI* setTimeline)(void*, VkDevice, uint64_t) = nullptr;
int(WINAPI* signalTimeline)(void*, uint64_t) = nullptr;
bool ready = false;
};
static EngineNetExports g_engineNet;
static VkResult signal_split_semaphore(Feature& feature, uint64_t value)
{
if (feature.engineNetTimeline)
return static_cast<VkResult>(g_engineNet.signalTimeline(feature.engineNet, value));
VkSemaphoreSignalInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO};
info.semaphore = feature.splitSemaphore;
info.value = value;
return g_vk.signalSemaphore(g_vk.device, &info);
}
static bool read_file(const std::string& path, std::vector<char>& bytes)
{
FILE* file = fopen(path.c_str(), "rb");
if (file == nullptr)
return false;
fseek(file, 0, SEEK_END);
const long size = ftell(file);
fseek(file, 0, SEEK_SET);
bytes.resize(size > 0 ? static_cast<size_t>(size) : 0);
const bool ok = size > 0 && fread(bytes.data(), 1, bytes.size(), file) == bytes.size();
fclose(file);
return ok;
}
// A storage buffer the engine's shaders and HIP both use. Pooled like the interop buffers, for the same ROCm leak.
static bool create_engine_shared_buffer(VramBuffer& target, size_t bytes, bool host = false)
{
const uint8_t kind = host ? 2 : 1;
if (take_pooled_vram_buffer(target, bytes, kind))
{
if (target.mapped != nullptr)
{
std::memset(target.mapped, 0, target.bytes);
MemoryBarrier();
}
return true;
}
VramBuffer buffer;
buffer.kind = kind;
VkExternalMemoryBufferCreateInfo external = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
info.size = bytes;
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
VkResult result = g_vk.createBuffer(g_vk.device, &info, nullptr, &buffer.buffer);
VkMemoryRequirements requirements = {};
if (result == VK_SUCCESS)
g_vk.bufferRequirements(g_vk.device, buffer.buffer, &requirements);
int type = device_local_type(requirements.memoryTypeBits);
if (host)
{
type = -1;
const auto coherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
for (int pass = 0; pass < 2 && type < 0; ++pass)
for (uint32_t index = 0; index < g_vk.memory.memoryTypeCount; ++index)
{
const auto flags = g_vk.memory.memoryTypes[index].propertyFlags;
if ((requirements.memoryTypeBits & (1u << index)) && (flags & coherent) == coherent &&
(pass != 0 || !(flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)))
{
type = static_cast<int>(index);
break;
}
}
}
VkMemoryDedicatedAllocateInfo dedicated = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
dedicated.buffer = buffer.buffer;
VkExportMemoryAllocateInfo exportInfo = {VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, &dedicated};
exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = static_cast<uint32_t>(type);
if (result == VK_SUCCESS && type < 0)
result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
if (result == VK_SUCCESS)
result = g_vk.allocate(g_vk.device, &allocation, nullptr, &buffer.memory);
if (result == VK_SUCCESS)
result = g_vk.bindBuffer(g_vk.device, buffer.buffer, buffer.memory, 0);
if (result == VK_SUCCESS && host)
result = g_vk.map(g_vk.device, buffer.memory, 0, VK_WHOLE_SIZE, 0, &buffer.mapped);
int imported = -1;
if (result == VK_SUCCESS)
imported = g.worker.call([&] {
return g_vk.import(g_vk.device, reinterpret_cast<uint64_t>(buffer.memory), requirements.size, &buffer.device,
&buffer.external);
});
if (result != VK_SUCCESS || imported != 0)
{
logf("engine backend: shared buffer of %zu bytes failed (VkResult %d, import %d)", bytes, result, imported);
if (buffer.mapped != nullptr)
g_vk.unmap(g_vk.device, buffer.memory);
if (buffer.memory != VK_NULL_HANDLE)
g_vk.free(g_vk.device, buffer.memory, nullptr);
if (buffer.buffer != VK_NULL_HANDLE)
g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
return false;
}
if (buffer.mapped != nullptr)
{
std::memset(buffer.mapped, 0, bytes);
MemoryBarrier();
}
buffer.bytes = bytes;
target = buffer;
return true;
}
// Only once no command list uses the engine and no network launch is pending.
static void release_engine_network(Feature& feature)
{
if (feature.engineNet != nullptr && g_engineNet.destroy != nullptr)
g.worker.call([&feature] {
g_engineNet.destroy(feature.engineNet);
return 0;
});
feature.engineNet = nullptr;
feature.engineNetTimeline = false;
for (VramBuffer* buffer : {&feature.engineTokens, &feature.engineHead, &feature.engineSync})
recycle_vram_buffer(*buffer);
}
// The HIP network of a feature: the buffers shared with HIP and the bridge's network over them. False: the
// engine's own Vulkan network is used instead (reason logged). `preset` is "K", "M" or "L"; `width`
// and `height` are the dimensions the bridge derives the token grid from (M: render, K: output); `tokenBytes`
// and `headBytes` size the two shared buffers (M: both MEngine::networkBufferBytes, K: Engine::networkTokenBytes
// and Engine::networkHeadBytes). The layer kernel names come from hipnet.bin, so no preset list is repeated here.
static bool create_engine_network(Feature& feature, const std::string& directory, const char* preset,
bool stagesAvailable, unsigned int width, unsigned int height, size_t tokenBytes,
size_t headBytes)
{
const std::string choice = env_string("D4R_ENGINE_NETWORK");
if (choice == "vulkan")
return false;
auto unavailable = [&](const char* reason) {
logf("engine backend (%s): HIP network unavailable (%s); using the Vulkan network", preset, reason);
release_engine_network(feature);
return false;
};
std::call_once(g_engineNet.once, [] {
g_engineNet.ready = load_export(g.cuda, "d4rEngineNetCreate", g_engineNet.create) &&
load_export(g.cuda, "d4rEngineNetLaunch", g_engineNet.launch) &&
load_export(g.cuda, "d4rEngineNetDone", g_engineNet.done) &&
load_export(g.cuda, "d4rEngineNetDestroy", g_engineNet.destroy) &&
load_export(g.cuda, "d4rEngineNetInputWaitReady", g_engineNet.inputWaitReady) &&
load_export(g.cuda, "d4rEngineNetLaunchAt", g_engineNet.launchAt) &&
load_export(g.cuda, "d4rEngineNetReleaseInput", g_engineNet.releaseInput) &&
load_export(g.cuda, "d4rEngineNetSetTimeline", g_engineNet.setTimeline) &&
load_export(g.cuda, "d4rEngineNetSignalTimeline", g_engineNet.signalTimeline);
});
if (!g_engineNet.ready)
return unavailable("the CUDA bridge lacks d4rEngineNet*");
if (!stagesAvailable)
return unavailable("the model has no stages for the native network; rebuild it with --hip");
if (g_vk.split == nullptr || g_vk.import == nullptr || feature.marker == nullptr || feature.markerValue == nullptr)
return unavailable("split frames need the d4r vkd3d-proton and VRAM interop");
std::vector<char> net;
if (!read_file(directory + "\\hipnet.bin", net))
return unavailable("no hipnet.bin in the model directory");
// The bridge dispatches on the magic alone, and the shared buffers are sized for `preset`.
const char* const magic = std::strcmp(preset, "K") == 0 ? "D4RKHIP1" : "D4RMHIP1";
if (net.size() < 16 || std::memcmp(net.data(), magic, 8) != 0)
return unavailable("hipnet.bin is not a native network of this preset");
uint32_t count = 0;
std::memcpy(&count, net.data() + 8, sizeof(count));
if (count == 0 || net.size() < 16 + 64ull * count)
return unavailable("invalid hipnet.bin");
// The kernel names live in the layer table; load the code object of each distinct one (the M tube repeats).
std::vector<std::string> names;
names.reserve(count);
for (uint32_t index = 0; index < count; ++index)
{
char name[33] = {};
std::memcpy(name, net.data() + 16 + 64 * index, 32);
if (name[0] == '\0')
return unavailable("invalid kernel name in hipnet.bin");
if (std::find(names.begin(), names.end(), name) == names.end())
names.emplace_back(name);
}
std::vector<std::vector<char>> code(names.size());
std::vector<EngineNetCode> codes(names.size());
for (size_t index = 0; index < names.size(); ++index)
{
if (!read_file(directory + "\\hip\\" + names[index] + ".hsaco", code[index]))
return unavailable("a layer kernel is missing in the model's hip folder");
codes[index] = {names[index].c_str(), code[index].data(), code[index].size()};
}
if (!create_engine_shared_buffer(feature.engineTokens, tokenBytes) ||
!create_engine_shared_buffer(feature.engineHead, headBytes) ||
!create_engine_shared_buffer(feature.engineSync, 256, true))
return unavailable("shared buffers");
const int created = g.worker.call([&] {
return g_engineNet.create(reinterpret_cast<const uint8_t*>(net.data()), net.size(), codes.data(),
static_cast<uint32_t>(codes.size()), width, height, feature.engineTokens.device,
feature.engineHead.device, &feature.engineNet);
});
if (created != 0 || feature.engineNet == nullptr)
{
feature.engineNet = nullptr;
logf("engine backend (%s): d4rEngineNetCreate failed (%d)", preset, created);
return unavailable("the kernels could not be loaded for this GPU");
}
// Input readiness on the GPU: the engine's front fills engineSync with the frame number (the shim records the
// fill right after recordFront) and the network's stream waits for it, so the layers start the moment the
// front is complete, without the CPU round trip through the readback marker. The bridge checks HIP's
// capability and a round trip on this buffer. Without wait-value support (or D4R_SHIM_ENGINE_GPU_WAIT=0)
// the worker keeps waiting for the CPU-visible marker before launching.
const bool gpuWaitEnabled = env_uint("D4R_SHIM_ENGINE_GPU_WAIT", 1) != 0;
const int waitProbe = gpuWaitEnabled ? g.worker.call([&] {
return g_engineNet.inputWaitReady(feature.engineNet, feature.engineSync.device,
static_cast<uint32_t*>(feature.engineSync.mapped));
}) : 0;
if (waitProbe < 0)
return unavailable("the HIP stream cannot complete a wait-value test");
feature.engineGpuWait = waitProbe == 1;
logf("engine backend (%s): HIP network input readiness on %s", preset, feature.engineGpuWait.load()
? "the GPU (the front publishes the frame number; the layers wait on it)"
: "the CPU (marker polling)");
if (feature.splitSemaphore == VK_NULL_HANDLE && !create_split_semaphore(feature))
return unavailable("timeline semaphore");
if (g.worker.call([&] {
return g_engineNet.setTimeline(feature.engineNet, g_vk.device,
reinterpret_cast<uint64_t>(feature.splitSemaphore));
}) != 0)
return unavailable("native HIP completion timeline");
feature.engineNetTimeline = true;
logf("engine backend (%s): network completion releases the split timeline from the native HIP callback", preset);
return true;
}
// Records the end of the front: a fill of this frame's number into the buffer the HIP stream waits on
// (d4rEngineNetLaunchAt). The engine's recordFront ends the input stage with its own ALL_COMMANDS barrier (the
// tokens are then available); the fill is ordered after it and, once it lands, the ten layers may read them.
static void record_engine_input_ready(Feature& feature, VkCommandBuffer command, uint32_t frame)
{
const VkMemoryBarrier before = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr,
VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT};
g_vk.barrier(command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &before, 0, nullptr,
0, nullptr);
g_vk.fill(command, feature.engineSync.buffer, 0, sizeof(uint32_t), frame);
const VkMemoryBarrier after = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT,
VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT};
g_vk.barrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &after, 0, nullptr,
0, nullptr);
}
static bool wait_for_input_marker(Feature* feature, int slotIndex, uint32_t frame);
static void frame_retired(Feature* feature, uint32_t frame);
// The network's inputs are written by the front of the game's list, so its stream waits for the frame's fill of
// the shared u32 (d4rEngineNetLaunchAt). A wait for a frame whose front never reached the GPU (the game recorded
// its list but did not execute it, or the game stalled) can only be satisfied by a later frame's fill: writing
// the value it waits for releases it, so the stream keeps moving. The frame's output is undefined either way.
// `disable`: the GPU-side wait itself looks broken for this feature, so later frames wait for the CPU marker.
static void release_engine_input_wait(Feature& feature, uint32_t frame, const char* reason, bool disable)
{
if (feature.engineNet == nullptr || g_engineNet.releaseInput == nullptr)
return;
const int result = g_engineNet.releaseInput(feature.engineNet, frame);
if (disable)
feature.engineGpuWait.store(false);
logf("engine frame %u: %s; releasing the GPU-side input wait (%d)", frame, reason, result);
}
// CUDA worker: runs the network of `frame` (behind the GPU-side input wait, or once the CPU marker shows the
// front has run), then lets the second part of the game's command list (the engine's reconstruction) run.
static void run_engine_network(Feature* feature, int slotIndex, uint32_t frame)
{
const bool gpuWait = feature->engineGpuWait.load() && g_engineNet.launchAt != nullptr;
if (!gpuWait && !wait_for_input_marker(feature, slotIndex, frame))
return; // dropped and retired there
const auto start = std::chrono::steady_clock::now();
int launched;
if (gpuWait)
{
feature->engineWaitFrame.store(frame);
launched = g_engineNet.launchAt(feature->engineNet, frame);
if (launched == 801 /* CUDA_ERROR_NOT_SUPPORTED: no wait was queued */)
{
feature->engineWaitFrame.store(0);
// The bridge could not queue the wait (no hipStreamWaitValue32, or the stream rejected it): wait
// for the input marker on the CPU, as without the GPU-side wait.
feature->engineGpuWait.store(false);
logf("engine frame %u: GPU-side input wait unavailable (%d); waiting for the input marker on the CPU", frame, launched);
if (!wait_for_input_marker(feature, slotIndex, frame))
return;
launched = g_engineNet.launch(feature->engineNet, frame);
}
}
else
launched = g_engineNet.launch(feature->engineNet, frame);
float gpuMs = 0.0f;
int done = launched == 0 ? 0 : -1;
bool released = false, discarded = false;
// Completion polling (as the CUDA path's output event): sleep while far from the predicted completion,
// yield around it (a Wine sleep rounds up to ~50 us), back off when the GPU runs late. The prediction
// learns from the earlier frames; with the GPU-side wait it also covers the front's remaining GPU time.
double lastNotReadyUs = -1.0;
while (done == 0)
{
done = g_engineNet.done(feature->engineNet, &gpuMs);
if (done != 0)
break;
const double elapsedUs = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - start).count();
if (gpuWait && !released)
{
// This frame's own marker (written with the aggregate one, at the end of its front) tells whether
// its front ran at all. A later frame's front running while this one's never did means the game
// discarded this list: its wait can only be satisfied by that later front, which would make the
// network read another frame's tokens. Release it and drop the frame, as the CPU marker path does.
const uint32_t slotMarker = feature->markerValue[1 + slotIndex];
if (slotMarker != frame && static_cast<int32_t>(*feature->markerValue - frame) > 0)
{
release_engine_input_wait(*feature, frame, "its command list was discarded", false);
discarded = true;
break;
}
// The front ran but the stream's wait did not start the network (a broken wait would leave every
// later frame queued behind it): release it and wait for the CPU marker from now on.
if (slotMarker == frame && elapsedUs > 1000000.0)
{
release_engine_input_wait(*feature, frame, "the GPU-side wait did not start the network", true);
released = true;
}
}
if (elapsedUs > 2000000.0)
{
done = -2;
break;
}
lastNotReadyUs = elapsedUs;
constexpr double kMarginUs = 120.0, kStepUs = 20.0, kChunkUs = 200.0, kSpinAfterUs = 150.0;
const double predictedUs = feature->engineNetWaitEmaUs;
const double remaining = predictedUs - kMarginUs - elapsedUs;
if (remaining > kStepUs)
d4r_sleep_us(static_cast<unsigned>(std::min(remaining, kChunkUs)));
else if (elapsedUs < predictedUs + kSpinAfterUs)
d4r_sleep_us(0); // polling retires resources; the native callback releases the GPU's split wait
else
d4r_sleep_us(static_cast<unsigned>(std::min(
kStepUs * (1.0 + (elapsedUs - predictedUs - kSpinAfterUs) / 160.0), kChunkUs)));
}
feature->engineWaitFrame.store(0);
const double waitedUs = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - start).count();
if (done == 1)
{
// Learn the completion time, not our own wait: the work finished between the last NOT_READY poll and the
// successful one (at most one chunk apart), so the midpoint cannot ratchet upwards.
double sample = lastNotReadyUs >= 0.0 ? 0.5 * (lastNotReadyUs + waitedUs) : waitedUs;
sample = std::min(sample, 20000.0);
feature->engineNetWaitEmaUs = feature->engineNetWaitEmaUs == 0.0
? sample : 0.8 * feature->engineNetWaitEmaUs + 0.2 * sample;
}
if (discarded)
logf("engine frame %u: the game discarded its command list; the frame is dropped", frame);
else if (done != 1)
logf("engine frame %u: HIP network failed (launch %d, state %d); the frame's output is undefined", frame, launched, done);
else if (frame <= 3 || frame % 600 == 0)
logf("engine frame %u: HIP network %.2f ms GPU, %.2f ms %s", frame, gpuMs, waitedUs / 1000.0,
gpuWait ? "from launch to completion (GPU-side input wait)" : "from input readiness to completion");
if (gpuWait && !discarded && done != 1)
release_engine_input_wait(*feature, frame, "the network failed before completing its input wait", true);
feature->inputs[slotIndex].busy = false;
frame_retired(feature, frame);
}
// Preset K and preset M record a native-network frame the same way: the front half, this frame's marker, the
// split of the game's list, the back half, and the worker that runs the network between the two halves.
// `recordPart(front)` records one half into the game's list; `sources` is {colour, depth, motion, output}, the
// first three sampled through the front. `refuse` (nothing usable recorded; the CUDA backend takes over) and
// `abandon` (recorded but the frame is unusable) belong to the caller so its logging and flags stay preset-specific.
struct EngineFrameSource
{
ID3D12Resource* resource;
d4r::GameImage* image;
unsigned int x, y, width, height;
D3D12_RESOURCE_STATES state, copyState;
};
template <typename RecordPart, typename Refuse, typename Abandon>
static bool record_engine_network_frame(Feature& feature, ID3D12GraphicsCommandList* list, EngineFrameSource* sources,
uint32_t frame, const char* preset, int reset, float mvScaleX, float mvScaleY,
float jitterX, float jitterY, RecordPart&& recordPart, Refuse&& refuse,
Abandon&& abandon)
{
// HIP waits on the GPU-published input value between the two Vulkan list parts. Its native completion
// callback releases the second part. The Wine worker retires resources, not frame timing.
const int slotIndex = static_cast<int>(frame % kSlots);
InputSlot& slot = feature.inputs[slotIndex];
const auto waitStart = std::chrono::steady_clock::now();
while (slot.busy.load() || feature.inFlight.load() >= 2)
{
if (std::chrono::steady_clock::now() - waitStart > std::chrono::seconds(2))
{
logf("engine frame %u: %d frames in flight; skipping evaluation", frame, feature.inFlight.load());
return true;
}
d4r_sleep_us(200);
}
if (FAILED(g_vk.lifetime->RetainExternalResources(list, feature.resources)))
return refuse("the command list cannot retain the engine's resources");
ID3D12GraphicsCommandList2* list2 = nullptr;
if (FAILED(list->QueryInterface(__uuidof(ID3D12GraphicsCommandList2), reinterpret_cast<void**>(&list2))))
return refuse("ID3D12GraphicsCommandList2 is unavailable");
for (int index = 0; index < 3; ++index)
transition(list, sources[index].resource, sources[index].state, sources[index].copyState);
const char* failure = recordPart(true);
if (failure != nullptr)
{
for (int index = 0; index < 3; ++index)
transition(list, sources[index].resource, sources[index].copyState, sources[index].state);
list2->Release();
return refuse(failure);
}
const D3D12_GPU_VIRTUAL_ADDRESS markerAddress = feature.marker->GetGPUVirtualAddress();
D3D12_WRITEBUFFERIMMEDIATE_PARAMETER markers[] = {
{markerAddress + sizeof(uint32_t) * (1 + slotIndex), frame},
{markerAddress, frame},
};
D3D12_WRITEBUFFERIMMEDIATE_MODE modes[] = {
D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT, D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT,
};
list2->WriteBufferImmediate(2, markers, modes);
list2->Release();
// From here on this call owns the frame: a failure leaves this frame's output undefined and hands the next
// frame to the CUDA backend (which must not record its own marker and split for this frame).
if (FAILED(g_vk.split->SplitCommandListForExternalWait(list, reinterpret_cast<UINT64>(feature.splitSemaphore), frame)))
{
for (int index = 0; index < 3; ++index)
transition(list, sources[index].resource, sources[index].copyState, sources[index].state);
return abandon("the command list could not be split");
}
transition(list, sources[3].resource, sources[3].state, sources[3].copyState);
failure = recordPart(false);
transition(list, sources[3].resource, sources[3].copyState, sources[3].state);
// The reconstruction still samples borrowed colour; keep its sampled layout through the back.
for (int index = 0; index < 3; ++index)
transition(list, sources[index].resource, sources[index].copyState, sources[index].state);
feature.split = true;
slot.busy = true;
feature.inFlight.fetch_add(1);
{
std::lock_guard<std::mutex> lock(feature.splitMutex);
feature.splitCompletion.admit(frame);
}
Feature* const owner = &feature;
g.worker.post([owner, slotIndex, frame] { run_engine_network(owner, slotIndex, frame); });
if (failure != nullptr)
return abandon(failure);
feature.engineJitter[0] = jitterX;
feature.engineJitter[1] = jitterY;
if (frame <= 3 || frame % 600 == 0)
logf("engine frame %u recorded, preset %s with the HIP network (jitter %.6f,%.6f, reset %d, mvScale %.3f,%.3f)",
frame, preset, jitterX, jitterY, reset, mvScaleX, mvScaleY);
return true;
}
// Presets M (13) and L (12): record the evaluation into `list`, borrowing compatible inputs and copying only
// fallbacks. L is M's pipeline with its own input, expansion and reconstruction stages (engine/L_NOTES.md).
static bool engine_evaluate_m(Feature& feature, ID3D12GraphicsCommandList* list, ID3D12Resource* color,
ID3D12Resource* depth, ID3D12Resource* motion, ID3D12Resource* output,
const FrameParams& p, uint32_t frame)
{
const bool presetL = feature.preset == 12;
const char* const name = presetL ? "L" : "M";
auto refuse = [&](const char* reason) {
feature.engineRefused = true;
logf("engine backend (%s): %s; feature %u uses the CUDA backend from frame %u", name, reason, feature.handle.Id, frame);
return false;
};
// NVSDK_NGX_DLSS_Feature_Flags: IsHDR 1, MVLowRes 2, MVJittered 4, DepthInverted 8, AutoExposure 0x40.
const bool ldr = (feature.flags & 0x1) == 0; // NGX's LDR stages: a model directory of its own, no exposure
std::call_once(g_engineM.once[presetL][ldr], [presetL, ldr] { init_engine_backend_m_variant(presetL, ldr); });
d4r::MModel* const model = g_engineM.models[presetL][ldr].get();
if (model == nullptr)
{
if (!ldr)
return refuse("not available");
return refuse(presetL ? "no LDR model (engine\\l-ldr) for a game without the HDR flag"
: "no LDR model (engine\\m-ldr) for a game without the HDR flag");
}
if ((feature.flags & 0x2) == 0)
return refuse("needs render-resolution motion vectors");
if (!model->hasDynamicResolution() && feature.engineM != nullptr &&
(p.renderWidth != feature.engineMRenderWidth || p.renderHeight != feature.engineMRenderHeight))
return refuse("dynamic render sizes need a model rebuilt by the current compile_m.py");
// NGX measures the exposure for this preset, so the AutoExposure flag, the game's exposure texture, the
// pre-exposure and the exposure scale are all ignored here (CUDA output is identical for each of them).
if (color == output || depth == output || motion == output)
return refuse("input/output texture aliasing is not supported");
d4r::GameFrameM game;
using Source = EngineFrameSource;
const auto inputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_INPUT_STATE", 0x40));
const auto depthState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_DEPTH_STATE", 0x40));
const auto outputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_OUTPUT_STATE", 0x8));
Source sources[4] = {
{color, &game.color, p.colorBaseX, p.colorBaseY, p.renderWidth, p.renderHeight, inputState, D3D12_RESOURCE_STATE_COPY_SOURCE},
{depth, &game.depth, p.depthBaseX, p.depthBaseY, p.renderWidth, p.renderHeight, depthState, D3D12_RESOURCE_STATE_COPY_SOURCE},
{motion, &game.motion, p.mvBaseX, p.mvBaseY, p.renderWidth, p.renderHeight, inputState, D3D12_RESOURCE_STATE_COPY_SOURCE},
{output, &game.output, p.outputBaseX, p.outputBaseY, feature.outWidth, feature.outHeight, outputState,
D3D12_RESOURCE_STATE_COPY_DEST},
};
for (Source& source : sources)
{
UINT64 handle = 0, offset = 0;
D3D12_RESOURCE_DESC desc;
source.resource->GetDesc(&desc);
if (FAILED(g_vk.interop->GetVulkanResourceInfo1(source.resource, &handle, &offset, &source.image->format)) ||
handle == 0 || desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 ||
source.x + source.width > desc.Width || source.y + source.height > desc.Height)
return refuse("a texture is not a single-sample 2D image covering its rectangle");
source.image->image = reinterpret_cast<VkImage>(handle);
source.image->x = source.x;
source.image->y = source.y;
prepare_engine_view(feature, source.resource, desc, *source.image, source.image == &game.output,
source.x == 0 && source.y == 0, source.copyState);
g_vk.interop->GetVulkanImageLayout(source.resource, source.copyState, &source.image->layout);
}
if (game.depth.format == VK_FORMAT_D32_SFLOAT || game.depth.format == VK_FORMAT_D32_SFLOAT_S8_UINT)
game.depth.aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
if (!d4r::gameColorFormat(game.color.format) || !d4r::gameMotionFormat(game.motion.format) ||
!d4r::gameDepthFormat(game.depth.format, game.depth.aspect) || !d4r::gameOutputFormat(game.output.format))
{
logf("engine backend (%s): Vulkan formats colour %d, depth %d, motion %d, output %d", name, game.color.format,
game.depth.format, game.motion.format, game.output.format);
return refuse("a texture format is not implemented");
}
game.jitter[0] = p.jitterX;
game.jitter[1] = p.jitterY;
game.motionToRender[0] = p.mvScaleX;
game.motionToRender[1] = p.mvScaleY;
if ((feature.flags & 0x4) != 0 && p.mvScaleX != 0.0f && p.mvScaleY != 0.0f)
{
game.motionOffset[0] = (p.jitterX - feature.engineJitter[0]) / p.mvScaleX;
game.motionOffset[1] = (p.jitterY - feature.engineJitter[1]) / p.mvScaleY;
}
game.renderWidth = p.renderWidth;
game.renderHeight = p.renderHeight;
game.reset = p.reset != 0 || feature.engineM == nullptr ||
p.renderWidth != feature.engineMLastWidth || p.renderHeight != feature.engineMLastHeight;
game.depthInverted = (feature.flags & 0x8) != 0;
try
{
if (feature.engineM == nullptr)
{
const size_t bytes = static_cast<size_t>(d4r::MEngine::networkBufferBytes(p.renderWidth, p.renderHeight));
const bool hip = create_engine_network(feature, g_engineM.directories[presetL][ldr], name,
model->hasExternalNetworkStages(), p.renderWidth, p.renderHeight,
bytes, bytes);
const d4r::ExternalNetwork external = {feature.engineTokens.buffer, feature.engineHead.buffer};
feature.engineM = std::make_unique<d4r::GameUpscalerM>(*model, g_vk.physical, g_vk.device, feature.outWidth,
feature.outHeight, (hip || !model->hasDynamicResolution()) ? p.renderWidth : feature.outWidth,
(hip || !model->hasDynamicResolution()) ? p.renderHeight : feature.outHeight,
hip ? &external : nullptr);
feature.engineMRenderWidth = p.renderWidth;
feature.engineMRenderHeight = p.renderHeight;
logf("engine backend (%s): feature %u runs preset %s in the game's command list (%ux%u -> %ux%u), network on %s, RGB10A2 direct output %d",
name, feature.handle.Id, name, p.renderWidth, p.renderHeight, feature.outWidth, feature.outHeight,
hip ? "the native HIP layers (list split around it)" : "Vulkan",
model->hasRgb10Output() && game.output.view != VK_NULL_HANDLE &&
game.output.format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 &&
game.output.layout == VK_IMAGE_LAYOUT_GENERAL && game.output.image != game.color.image &&
game.output.image != game.motion.image && game.output.image != game.depth.image);
}
}
catch (const std::exception& error)
{
logf("engine backend (%s): %s", name, error.what());
release_engine_network(feature);
return refuse("the engine could not be created");
}
d4r::GameUpscalerM* adapter = feature.engineM.get();
const bool resizedHip = feature.engineNet != nullptr &&
(p.renderWidth != feature.engineMRenderWidth || p.renderHeight != feature.engineMRenderHeight);
if (resizedHip)
{
try
{
if (!feature.engineMDynamic)
{
feature.engineMDynamic = std::make_unique<d4r::GameUpscalerM>(*model, g_vk.physical, g_vk.device,
feature.outWidth, feature.outHeight, feature.outWidth, feature.outHeight);
logf("engine backend (%s): dynamic render sizes use the Vulkan network; original HIP resources retained", name);
}
adapter = feature.engineMDynamic.get();
}
catch (const std::exception& error)
{
logf("engine backend (%s): %s", name, error.what());
return refuse("the dynamic engine could not be created");
}
}
feature.engineMLastWidth = p.renderWidth;
feature.engineMLastHeight = p.renderHeight;
if (feature.engineNet != nullptr && !resizedHip)
{
// The native HIP network: the game's list is split around it. Its native completion callback releases
// the second part, which records expansion, reconstruction and the final store (or the conversion blit
// for incompatible outputs). The Wine worker retires resources, not frame timing.
auto recordPart = [&](bool front) -> const char* {
VkCommandBuffer command = VK_NULL_HANDLE;
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &command)))
return "BeginVkCommandBufferInterop failed";
const char* failure = nullptr;
try
{
if (front)
{
adapter->recordFront(command, game);
// The HIP network's stream waits for this frame's number (record_engine_input_ready): the
// fill is part of the first half, so the network starts the moment the front's GPU work ends.
if (feature.engineGpuWait.load())
record_engine_input_ready(feature, command, frame);
}
else
adapter->recordBack(command, game);
}
catch (const std::exception& error)
{
logf("engine backend (%s): %s", name, error.what());
failure = "the frame could not be recorded";
}
if (FAILED(g_vk.interop->EndVkCommandBufferInterop(list)) && failure == nullptr)
failure = "EndVkCommandBufferInterop failed";
return failure;
};
auto abandon = [&](const char* reason) {
feature.engineRefused = true;
logf("engine backend (%s): %s; feature %u uses the CUDA backend after frame %u", name, reason, feature.handle.Id, frame);
return true;
};
return record_engine_network_frame(feature, list, sources, frame, name, game.reset, p.mvScaleX, p.mvScaleY,
p.jitterX, p.jitterY, recordPart, refuse, abandon);
}
if (FAILED(g_vk.lifetime->RetainExternalResources(list, feature.resources)))
return refuse("the command list cannot retain the engine's resources");
for (const Source& source : sources)
transition(list, source.resource, source.state, source.copyState);
VkCommandBuffer command = VK_NULL_HANDLE;
bool recorded = SUCCEEDED(g_vk.interop->BeginVkCommandBufferInterop(list, &command));
const char* failure = recorded ? nullptr : "BeginVkCommandBufferInterop failed";
if (recorded)
{
try
{
adapter->record(command, game);
}
catch (const std::exception& error)
{
logf("engine backend (%s): %s", name, error.what());
failure = "the frame could not be recorded";
recorded = false;
}
if (FAILED(g_vk.interop->EndVkCommandBufferInterop(list)) && recorded)
{
failure = "EndVkCommandBufferInterop failed";
recorded = false;
}
}
for (const Source& source : sources)
transition(list, source.resource, source.copyState, source.state);
if (!recorded)
return refuse(failure);
feature.engineJitter[0] = p.jitterX;
feature.engineJitter[1] = p.jitterY;
if (frame <= 3 || frame % 600 == 0)
logf("engine frame %u recorded, preset %s (jitter %.6f,%.6f, reset %d, mvScale %.3f,%.3f)", frame, name, p.jitterX, p.jitterY,
game.reset, p.mvScaleX, p.mvScaleY);
return true;
}
// Records the whole evaluation of `frame` into `list`. False: nothing usable was recorded and the
// CUDA backend takes this feature over.
static bool engine_evaluate(Feature& feature, ID3D12GraphicsCommandList* list, ID3D12Resource* color,
ID3D12Resource* depth, ID3D12Resource* motion, ID3D12Resource* output,
ID3D12Resource* exposure, const FrameParams& p, uint32_t frame)
{
if (feature.engineRefused)
return false;
auto refuse = [&](const char* reason) {
feature.engineRefused = true;
logf("engine backend: %s; feature %u uses the CUDA backend from frame %u", reason, feature.handle.Id, frame);
if (feature.engine != nullptr)
logf("engine backend: the temporal history restarts in the CUDA backend");
return false;
};
std::call_once(g_engine.vulkanOnce, init_engine_vulkan);
if (feature.preset == 13 || feature.preset == 12)
return engine_evaluate_m(feature, list, color, depth, motion, output, p, frame);
// Without the HDR flag NGX runs its LDR input and output kernels: a model directory of its own.
const bool ldr = (feature.flags & 0x1) == 0;
if (ldr)
std::call_once(g_engine.ldrOnce, init_engine_backend_ldr);
else
std::call_once(g_engine.once, init_engine_backend);
d4r::Model* const model = ldr ? g_engine.modelLdr.get() : g_engine.model.get();
if (model == nullptr)
return refuse(ldr ? "no LDR model (engine\\k-ldr) for a game without the HDR flag" : "not available");
// NVSDK_NGX_DLSS_Feature_Flags: IsHDR 1, MVLowRes 2, MVJittered 4, DepthInverted 8.
if (feature.preset != 11)
return refuse("only presets K, L and M are implemented");
// MVLowRes (0x2) clear: vectors are at output resolution (engine revision D4RO0003).
const bool displayMotion = (feature.flags & 0x2) == 0;
if (displayMotion && !model->hasDisplayMotion())
return refuse("display-resolution motion needs a model compiled by the current compile_k.py");
if ((feature.flags & 0x8) == 0 && !model->hasRegularDepth())
return refuse("regular (non-inverted) depth needs a model compiled by the current compile_k.py");
// DLSS.Indicator.Invert.X/Y.Axis (p.invertX/Y) only place NGX's debug indicator: NGX's K and M outputs are
// byte-identical with them set (harness D4R_HARNESS_INDICATOR_INVERT=1), so neither engine path reads them.
// Two exposure sources. AutoExposure (0x40) measures the frame's luma scaled by 1 / pre-exposure and multiplies
// the result by the exposure scale. Without it the game's ExposureTexture is the exposure: texel x scale /
// pre-exposure (read directly, not measured).
const bool autoExposure = (feature.flags & 0x40) != 0;
const bool gameExposure = !ldr && !autoExposure;
if (!ldr && autoExposure && !model->hasAutoExposure())
return refuse("automatic exposure needs a model with exposure shaders");
if (gameExposure && (!p.hasExposure || !model->hasGameExposure()))
return refuse("exposure without AutoExposure needs an exposure texture and a model compiled by the current compile_k.py");
if (!ldr && !(std::isfinite(p.exposureScale) && p.exposureScale > 0.0f))
return refuse("the exposure scale is not a positive number");
if (!ldr && !(std::isfinite(p.preExposure) && p.preExposure > 0.0f))
return refuse("the pre-exposure is not a positive number");
if (output == color || output == motion || output == depth)
return refuse("input/output texture aliasing is not supported");
d4r::GameFrame game;
using Source = EngineFrameSource;
const auto inputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_INPUT_STATE", 0x40));
const auto depthState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_DEPTH_STATE", 0x40));
const auto outputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_OUTPUT_STATE", 0x8));
Source sources[5] = {
{color, &game.color, p.colorBaseX, p.colorBaseY, p.renderWidth, p.renderHeight, inputState,
D3D12_RESOURCE_STATE_COPY_SOURCE},
{depth, &game.depth, p.depthBaseX, p.depthBaseY, p.renderWidth, p.renderHeight, depthState,
D3D12_RESOURCE_STATE_COPY_SOURCE},
{motion, &game.motion, p.mvBaseX, p.mvBaseY, displayMotion ? feature.outWidth : p.renderWidth,
displayMotion ? feature.outHeight : p.renderHeight, inputState, D3D12_RESOURCE_STATE_COPY_SOURCE},
{output, &game.output, p.outputBaseX, p.outputBaseY, feature.outWidth, feature.outHeight, outputState,
D3D12_RESOURCE_STATE_COPY_DEST},
{exposure, &game.exposure, 0, 0, 1, 1, inputState, D3D12_RESOURCE_STATE_COPY_SOURCE},
};
const size_t sourceCount = gameExposure ? 5 : 4;
for (size_t index = 0; index < sourceCount; ++index)
{
Source& source = sources[index];
UINT64 handle = 0, offset = 0;
D3D12_RESOURCE_DESC desc;
source.resource->GetDesc(&desc);
if (FAILED(g_vk.interop->GetVulkanResourceInfo1(source.resource, &handle, &offset, &source.image->format)) ||
handle == 0 || desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 ||
source.x + source.width > desc.Width || source.y + source.height > desc.Height)
return refuse("a texture is not a single-sample 2D image covering its rectangle");
source.image->image = reinterpret_cast<VkImage>(handle);
source.image->x = source.x;
source.image->y = source.y;
const bool isOutput = source.image == &game.output;
prepare_engine_view(feature, source.resource, desc, *source.image, isOutput,
(source.x == 0 && source.y == 0) || model->hasDirectOrigins(), source.copyState);
g_vk.interop->GetVulkanImageLayout(source.resource, source.copyState, &source.image->layout);
}
if (gameExposure && (game.exposure.view == VK_NULL_HANDLE || (game.exposure.format != VK_FORMAT_R32_SFLOAT &&
game.exposure.format != VK_FORMAT_R16_SFLOAT &&
game.exposure.format != VK_FORMAT_R32G32B32A32_SFLOAT &&
game.exposure.format != VK_FORMAT_R16G16B16A16_SFLOAT)))
return refuse("the exposure texture must be a float texture the engine samples directly");
if (game.depth.format == VK_FORMAT_D32_SFLOAT || game.depth.format == VK_FORMAT_D32_SFLOAT_S8_UINT)
game.depth.aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
if (!d4r::gameColorFormat(game.color.format) || !d4r::gameMotionFormat(game.motion.format) ||
!d4r::gameDepthFormat(game.depth.format, game.depth.aspect) || !d4r::gameOutputFormat(game.output.format))
{
logf("engine backend: Vulkan formats colour %d, depth %d, motion %d, output %d", game.color.format,
game.depth.format, game.motion.format, game.output.format);
return refuse("a texture format is not implemented");
}
if (frame <= 2)
logf("engine direct textures: color %d, motion %d, depth %d, output %d (format %d, RGB10A2 pipeline %d)",
game.color.view != VK_NULL_HANDLE, game.motion.view != VK_NULL_HANDLE,
game.depth.view != VK_NULL_HANDLE, game.output.view != VK_NULL_HANDLE,
game.output.format, model->hasRgb10Output());
game.settings.renderWidth = p.renderWidth;
game.settings.renderHeight = p.renderHeight;
game.settings.jitter[0] = p.jitterX;
game.settings.jitter[1] = p.jitterY;
game.settings.displayMotion = displayMotion;
// The engine takes motion in output pixels; MV.Scale gives render pixels (display-resolution vectors are output pixels already).
const float toOutput = displayMotion ? 1.0f : static_cast<float>(feature.outWidth) / static_cast<float>(p.renderWidth);
const float toOutputY = displayMotion ? 1.0f : static_cast<float>(feature.outHeight) / static_cast<float>(p.renderHeight);
game.settings.motionScale[0] = p.mvScaleX * toOutput;
game.settings.motionScale[1] = p.mvScaleY * toOutputY;
// MVJittered: NGX adds the change of the jitter (render pixels) to the vectors before scaling them
// (observed with MV.Scale 1: offset = jitter - previous jitter, the first frame's previous being 0).
// The offset is added to the raw vector, before motionScale's own output-pixel conversion, so it carries
// no extra output/render factor in either motion mode.
if ((feature.flags & 0x4) != 0 && p.mvScaleX != 0.0f && p.mvScaleY != 0.0f)
{
game.settings.motionOffset[0] = (p.jitterX - feature.engineJitter[0]) / p.mvScaleX;
game.settings.motionOffset[1] = (p.jitterY - feature.engineJitter[1]) / p.mvScaleY;
}
game.settings.reset = p.reset != 0 || feature.engine == nullptr;
game.settings.depthInverted = (feature.flags & 0x8) != 0;
game.settings.autoExposure = !ldr && autoExposure; // LDR: no exposure at all
game.settings.exposureScale = ldr ? 1.0f : p.exposureScale;
// NGX's automatic K measurement scales the luma by 1 / pre-exposure before the logarithm
// (cuda_luma_convert_kernel), so the stored exposure is pre times the raw measurement and the stage's
// 1 / pre factor recovers it; the game texture path applies the same stage factor.
game.settings.preExposure = ldr ? 1.0f : p.preExposure;
try
{
if (feature.engine == nullptr)
{
const size_t tokenBytes = static_cast<size_t>(d4r::Engine::networkTokenBytes(feature.outWidth, feature.outHeight));
const size_t headBytes = static_cast<size_t>(d4r::Engine::networkHeadBytes(feature.outWidth, feature.outHeight));
const bool hip = create_engine_network(feature, ldr ? g_engine.directoryLdr : g_engine.directory, "K", true,
feature.outWidth, feature.outHeight, tokenBytes, headBytes);
const d4r::ExternalNetwork external = {feature.engineTokens.buffer, feature.engineHead.buffer};
feature.engine = std::make_unique<d4r::GameUpscaler>(*model, g_vk.physical, g_vk.device,
feature.outWidth, feature.outHeight,
feature.outWidth, feature.outHeight,
hip ? &external : nullptr);
logf("engine backend: feature %u runs preset K in the game's command list (%ux%u -> %ux%u), network on %s",
feature.handle.Id, p.renderWidth, p.renderHeight, feature.outWidth, feature.outHeight,
hip ? "the native HIP layers (list split around it)" : "Vulkan");
}
}
catch (const std::exception& error)
{
logf("engine backend: %s", error.what());
release_engine_network(feature);
return refuse("the engine could not be created");
}
if (feature.engineNet != nullptr)
{
// The native HIP network: the game's list is split around it. Its native completion callback releases
// the second part, which records the final store (or the conversion blit for incompatible outputs).
auto recordPart = [&](bool front) -> const char* {
VkCommandBuffer command = VK_NULL_HANDLE;
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &command)))
return "BeginVkCommandBufferInterop failed";
const char* failure = nullptr;
try
{
if (front)
{
feature.engine->recordFront(command, game);
// The HIP network's stream waits for this frame's number (record_engine_input_ready): the
// fill is part of the first half, so the network starts the moment the front's GPU work ends.
if (feature.engineGpuWait.load())
record_engine_input_ready(feature, command, frame);
}
else
feature.engine->recordBack(command, game);
}
catch (const std::exception& error)
{
logf("engine backend (K): %s", error.what());
failure = "the frame could not be recorded";
}
if (FAILED(g_vk.interop->EndVkCommandBufferInterop(list)) && failure == nullptr)
failure = "EndVkCommandBufferInterop failed";
return failure;
};
auto abandon = [&](const char* reason) {
feature.engineRefused = true;
logf("engine backend (K): %s; feature %u uses the CUDA backend after frame %u", reason, feature.handle.Id, frame);
return true;
};
return record_engine_network_frame(feature, list, sources, frame, "K", game.settings.reset,
game.settings.motionScale[0], game.settings.motionScale[1], p.jitterX,
p.jitterY, recordPart, refuse, abandon);
}
if (FAILED(g_vk.lifetime->RetainExternalResources(list, feature.resources)))
return refuse("the command list cannot retain the engine's resources");
for (const Source& source : sources)
transition(list, source.resource, source.state, source.copyState);
VkCommandBuffer command = VK_NULL_HANDLE;
bool recorded = SUCCEEDED(g_vk.interop->BeginVkCommandBufferInterop(list, &command));
const char* failure = recorded ? nullptr : "BeginVkCommandBufferInterop failed";
if (recorded)
{
try
{
feature.engine->record(command, game);
}
catch (const std::exception& error)
{
logf("engine backend: %s", error.what());
failure = "the frame could not be recorded";
recorded = false;
}
if (FAILED(g_vk.interop->EndVkCommandBufferInterop(list)) && recorded)
{
failure = "EndVkCommandBufferInterop failed";
recorded = false;
}
}
for (const Source& source : sources)
transition(list, source.resource, source.copyState, source.state);
if (!recorded)
return refuse(failure);
feature.engineJitter[0] = p.jitterX;
feature.engineJitter[1] = p.jitterY;
if (frame <= 3 || frame % 600 == 0)
logf("engine frame %u recorded (jitter %.6f,%.6f, reset %d, mvScale %.3f,%.3f, render %ux%u)", frame,
p.jitterX, p.jitterY, game.settings.reset, p.mvScaleX, p.mvScaleY, p.renderWidth, p.renderHeight);
return true;
}
static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* handle, void* parameters)
{
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
FrameTiming timing;
timing.enabled = profile_enabled();
if (timing.enabled)
{
timing.gameStart = ProfileClock::now();
static thread_local ProfileClock::time_point lastGameStart;
if (lastGameStart.time_since_epoch().count() != 0)
timing.gameInterval = profile_ms(lastGameStart, timing.gameStart);
lastGameStart = timing.gameStart;
}
Feature* feature = find_feature(handle);
if (feature == nullptr || feature->nativeVulkan || list == nullptr || parameters == nullptr)
return NGX_FAIL_INVALID_PARAMETER;
ID3D12Resource* color = get_resource(parameters, "Color");
ID3D12Resource* depth = get_resource(parameters, "Depth");
ID3D12Resource* motion = get_resource(parameters, "MotionVectors");
ID3D12Resource* output = get_resource(parameters, "Output");
ID3D12Resource* exposure = get_resource(parameters, "ExposureTexture");
const uint32_t frame = ++feature->frame;
if (color == nullptr || depth == nullptr || motion == nullptr || output == nullptr)
{
if (frame <= 3)
logf("evaluate frame %u: missing resource color=%p depth=%p mv=%p output=%p", frame, color, depth, motion,
output);
return NGX_FAIL_INVALID_PARAMETER;
}
D3D12_RESOURCE_DESC colorDesc, depthDesc, motionDesc, outputDesc;
color->GetDesc(&colorDesc);
depth->GetDesc(&depthDesc);
motion->GetDesc(&motionDesc);
output->GetDesc(&outputDesc);
if (frame == 1)
logf("evaluate: color %llux%u fmt=%d, depth %llux%u fmt=%d, mv %llux%u fmt=%d, output %llux%u fmt=%d, "
"exposure=%p",
colorDesc.Width, colorDesc.Height, colorDesc.Format, depthDesc.Width, depthDesc.Height, depthDesc.Format,
motionDesc.Width, motionDesc.Height, motionDesc.Format, outputDesc.Width, outputDesc.Height,
outputDesc.Format, exposure);
const bool colorSupported = supported_input(Plane::Color, colorDesc.Format);
const bool depthSupported = supported_input(Plane::Depth, depthDesc.Format);
const bool motionSupported = supported_input(Plane::Motion, motionDesc.Format);
const bool outputSupported = supported_output(outputDesc.Format);
if (!colorSupported || !depthSupported || !motionSupported || !outputSupported)
{
// Log the first failures even if they occur well after startup, then periodically.
const uint32_t rejected = ++feature->rejectedFormatCount;
if (rejected <= 3 || rejected % 120 == 0)
logf("evaluate feature %u frame %u: unsupported format (rejection %u): "
"color %llux%u fmt=%d%s, depth %llux%u fmt=%d%s, "
"mv %llux%u fmt=%d%s, output %llux%u fmt=%d%s",
feature->handle.Id, frame, rejected,
colorDesc.Width, colorDesc.Height, colorDesc.Format, colorSupported ? "" : " (unsupported)",
depthDesc.Width, depthDesc.Height, depthDesc.Format, depthSupported ? "" : " (unsupported)",
motionDesc.Width, motionDesc.Height, motionDesc.Format, motionSupported ? "" : " (unsupported)",
outputDesc.Width, outputDesc.Height, outputDesc.Format, outputSupported ? "" : " (unsupported)");
return NGX_FAIL_UNSUPPORTED_FORMAT;
}
if (exposure != nullptr)
{
D3D12_RESOURCE_DESC exposureDesc;
exposure->GetDesc(&exposureDesc);
if (!supported_input(Plane::Exposure, exposureDesc.Format))
exposure = nullptr;
}
FrameParams p;
p.jitterX = get_float_or(parameters, "Jitter.Offset.X", 0.0f);
p.jitterY = get_float_or(parameters, "Jitter.Offset.Y", 0.0f);
p.mvScaleX = get_float_or(parameters, "MV.Scale.X", 1.0f);
p.mvScaleY = get_float_or(parameters, "MV.Scale.Y", 1.0f);
p.sharpness = get_float_or(parameters, "Sharpness", 0.0f);
p.preExposure = get_float_or(parameters, "DLSS.Pre.Exposure", 1.0f);
p.exposureScale = get_float_or(parameters, "DLSS.Exposure.Scale", 1.0f);
p.frameTime = get_float_or(parameters, "FrameTimeDeltaInMsec", 16.6f);
p.reset = get_int_or(parameters, "Reset", 0);
p.invertX = get_int_or(parameters, "DLSS.Indicator.Invert.X.Axis", 0);
p.invertY = get_int_or(parameters, "DLSS.Indicator.Invert.Y.Axis", 0);
p.renderWidth = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Width", 0);
p.renderHeight = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Height", 0);
if (p.renderWidth == 0 || p.renderHeight == 0)
{
p.renderWidth = feature->width;
p.renderHeight = feature->height;
}
p.colorBaseX = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.X", 0);
p.colorBaseY = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.Y", 0);
p.depthBaseX = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.X", 0);
p.depthBaseY = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.Y", 0);
p.mvBaseX = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.X", 0);
p.mvBaseY = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.Y", 0);
p.outputBaseX = get_uint_or(parameters, "DLSS.Output.Subrect.Base.X", 0);
p.outputBaseY = get_uint_or(parameters, "DLSS.Output.Subrect.Base.Y", 0);
p.hasExposure = exposure != nullptr;
// Ray Reconstruction: the denoiser's own per-frame settings and camera matrices are read
// here, on the game thread, while the caller's parameter object is certainly alive, and
// carried to the worker by value.
if (feature->rayReconstruction)
{
for (size_t index = 0; index < sizeof(kAuxScalars) / sizeof(kAuxScalars[0]); ++index)
{
int value = 0;
const bool given = d4r_ngx_get_int(parameters, kAuxScalars[index].name, &value) == NGX_SUCCESS;
// Not given this frame: back to what the caller asked for at creation, so a
// setting it has stopped varying does not keep an old per-frame override.
p.auxScalar[index] = given ? value : feature->auxScalarBaseline[index];
p.auxScalarSet[index] = true;
}
// Each guide's Subrect origin, read here while the caller's object is alive. An origin
// the caller stops registering resets to zero rather than lingering.
for (size_t index = 0; index < sizeof(kAuxSubrectNames) / sizeof(kAuxSubrectNames[0]); ++index)
{
static std::string nameX, nameY;
nameX.assign(kAuxSubrectNames[index]).append(".Subrect.Base.X");
nameY.assign(kAuxSubrectNames[index]).append(".Subrect.Base.Y");
p.auxSubrect[index][0] = get_uint_or(parameters, nameX.c_str(), 0);
p.auxSubrect[index][1] = get_uint_or(parameters, nameY.c_str(), 0);
}
for (size_t index = 0; index < sizeof(kAuxMatrices) / sizeof(kAuxMatrices[0]); ++index)
{
// A matrix is registered as a pointer to float[16] (nvsdk: the SDK's helpers
// set these with SetVoidPointer), never as sixteen separate scalars.
void* value = nullptr;
p.auxMatrixSet[index] =
d4r_ngx_get_void(parameters, kAuxMatrices[index], &value) == NGX_SUCCESS && value != nullptr;
if (p.auxMatrixSet[index])
std::memcpy(p.auxMatrix[index], value, sizeof(p.auxMatrix[index]));
}
}
if (engine_requested() && engine_evaluate(*feature, list, color, depth, motion, output, exposure, p, frame))
return NGX_SUCCESS;
if (feature->cudaCreationDeferred)
{
logf("engine backend: creating CUDA fallback for feature %u at frame %u", feature->handle.Id, frame);
const NgxResult created = create_cuda_feature(*feature);
if (created != NGX_SUCCESS) return created;
p.reset = 1; // the freshly created CUDA feature has none of the Vulkan engine's history
logf("engine backend: CUDA fallback initialized; temporal history reset");
}
const int slotIndex = static_cast<int>(frame % kSlots);
InputSlot& slot = feature->inputs[slotIndex];
// Throttle the game to DLSS speed when the pipeline still owns this slot
// or already holds the maximum number of frames.
static const int maxInFlight = static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) > 0
? static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) : 3;
// Split frames reuse frame N's result slot for frame N + kOutputSlots.
const int cap = feature->split && maxInFlight > kOutputSlots - 1 ? kOutputSlots - 1 : maxInFlight;
const auto waitStart = std::chrono::steady_clock::now();
while (slot.busy.load() || feature->inFlight.load() >= cap)
{
if (std::chrono::steady_clock::now() - waitStart > std::chrono::seconds(2))
{
logf("frame %u: input slot %d busy or %d frames in flight; skipping evaluation", frame, slotIndex, feature->inFlight.load());
return NGX_SUCCESS;
}
d4r_sleep_us(200);
}
if (timing.enabled)
timing.slotWait = profile_ms(waitStart, ProfileClock::now());
if (g_vk.lifetime == nullptr ||
FAILED(g_vk.lifetime->RetainExternalResources(list, feature->resources)))
{
logf("frame %u: unable to retain command-list resources; evaluation not recorded", frame);
return NGX_FAIL_PLATFORM_ERROR;
}
// VRAM interop when every resource qualifies (decided once per feature).
ID3D12Resource* const inputs[4] = {color, depth, motion, exposure};
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
const int inputCount = exposure != nullptr ? 4 : 3;
VramCopy vramInputs[4], vramOutput;
// Alpha readback uses the host result ring; color must use the same pipeline so both
// outputs can present one produced frame. The existing cutover below drains VRAM work.
const bool alphaOutputRequested =
feature->rayReconstruction && get_resource(parameters, "DLSSD.OutputAlpha") != nullptr;
if (!alphaOutputRequested && (!feature->vramDecided || feature->vram) && vram_interop_available())
{
p.vram = describe_vram_copy(output, Plane::Color, vramOutput, true);
for (int index = 0; index < inputCount && p.vram; ++index)
p.vram = describe_vram_copy(inputs[index], planes[index], vramInputs[index]);
}
if (!feature->vramDecided)
{
feature->vramDecided = true;
feature->vram = p.vram;
feature->split = p.vram && env_uint("D4R_SHIM_SPLIT_FRAME", 0) != 0 && g_vk.split != nullptr &&
(feature->splitSemaphore != VK_NULL_HANDLE || create_split_semaphore(*feature));
feature->linearInputs = feature->split && env_uint("D4R_SHIM_LINEAR_INPUTS", 0) != 0 &&
g.cu.registerLinearTexture != nullptr;
if (feature->linearInputs)
logf("linear inputs: NGX samples the interop buffers directly");
logf("VRAM interop %s for this feature%s%s", p.vram ? "on" : "off",
feature->split ? ", presenting each frame's own result (split frames)" : "",
p.vram ? ", waiting for input readiness with CPU sleeps" : "");
}
else if (feature->vram && !p.vram)
{
logf("frame %u: resources no longer qualify for VRAM interop; staging through host memory from now on", frame);
drain_pipeline();
{
std::lock_guard<std::mutex> lock(feature->splitMutex);
feature->split = false;
}
feature->vram = false;
feature->linearInputs = false;
feature->latestOutput = -1;
feature->outputFormat = DXGI_FORMAT_UNKNOWN; // recreates the output staging below
}
const bool verify = p.vram && env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
if (p.vram)
{
for (int index = 0; index < inputCount; ++index)
{
HostPlane& geometry = slot.host[index]; // geometry only; the bytes stay in VRAM
geometry.width = vramInputs[index].width;
geometry.height = vramInputs[index].height;
geometry.rowBytes = canonical_texel_bytes(planes[index]) * geometry.width;
if (feature->linearInputs)
geometry.rowBytes = (geometry.rowBytes + 255) & ~static_cast<size_t>(255); // texture pitch alignment
if (!ensure_vram_buffer(*feature, slot.vram[index], geometry.size()))
return NGX_FAIL_PLATFORM_ERROR;
}
// M's downsample kernel has only formatted stores, which the native store can pack (kernels/tex/tex_common.h).
p.packedOutput = feature->split && feature->outputDirectAllowed && feature->preset == 13 && vramOutput.convert &&
outputDesc.Format == DXGI_FORMAT_R10G10B10A2_UNORM && vramOutput.width % 2 == 0 &&
env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0 && env_uint("D4R_SHIM_OUTPUT_PACKED", 0) != 0 &&
feature->packedReady.load();
static bool packedLogged = false;
if (p.packedOutput && !packedLogged)
{
packedLogged = true;
logf("frame %u: packed output (the output kernel stores R10G10B10A2 texels; no conversion blit)", frame);
}
InPlaceTexture outputTexture;
const size_t outputTexelBytes = p.packedOutput ? 4 : 8;
if (feature->split && feature->outputDirectAllowed && (!vramOutput.convert || p.packedOutput) &&
env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0 &&
describe_in_place(output, outputTexture) && outputTexture.rowPitch % 8 == 0 &&
outputTexture.rowPitch / 8 <= 0xffffu &&
outputTexture.rowPitch >= static_cast<size_t>(vramOutput.width) * outputTexelBytes)
{
p.outputInPlace = outputTexture.texels;
p.outputInPlacePitch = static_cast<uint32_t>(outputTexture.rowPitch);
}
static bool outputInPlaceLogged = false;
if (p.outputInPlace != 0 && !outputInPlaceLogged)
{
outputInPlaceLogged = true;
logf("frame %u: output in place (the output kernel writes the game's %s texture; no game-side copy)", frame,
p.packedOutput ? "R10G10B10A2" : "RGBA16F");
}
static bool inPlaceLogged = false;
if (g_vk.linear != nullptr && !inPlaceLogged)
{
inPlaceLogged = true;
if (p.outputInPlace == 0)
{
UINT64 memory = 0, size = 0, offset = 0, pitch = 0;
const HRESULT linear = g_vk.linear->GetVulkanLinearImageInfo(output, &memory, &size, &offset, &pitch);
logf("in-place: output copied on the first frame (split %d, direct output allowed %d, format %d, needs conversion %d, "
"linear exportable texture %d, row pitch %llu)", static_cast<int>(feature->split), feature->outputDirectAllowed,
static_cast<int>(outputDesc.Format), vramOutput.convert, SUCCEEDED(linear),
static_cast<unsigned long long>(pitch));
}
}
if (vramInputs[0].convert && !ensure_conversion_image(*feature, feature->colorConversion,
vramInputs[0].width, vramInputs[0].height,
VK_FORMAT_R16G16B16A16_SFLOAT))
return NGX_FAIL_PLATFORM_ERROR;
if (vramInputs[1].convert && !ensure_conversion_image(*feature, feature->depthConversion,
vramInputs[1].width, vramInputs[1].height,
VK_FORMAT_R32_SFLOAT))
return NGX_FAIL_PLATFORM_ERROR;
if (vramInputs[2].convert && !ensure_conversion_image(*feature, feature->motionConversion,
vramInputs[2].width, vramInputs[2].height,
VK_FORMAT_R16G16_SFLOAT))
return NGX_FAIL_PLATFORM_ERROR;
if (inputCount == 4 && vramInputs[3].convert &&
!ensure_conversion_image(*feature, feature->exposureConversion,
vramInputs[3].width, vramInputs[3].height, VK_FORMAT_R32_SFLOAT))
return NGX_FAIL_PLATFORM_ERROR;
if (vramOutput.convert && !ensure_conversion_image(*feature, feature->outputConversion,
vramOutput.width, vramOutput.height,
VK_FORMAT_R16G16B16A16_SFLOAT))
return NGX_FAIL_PLATFORM_ERROR;
}
if ((!p.vram || verify) &&
(!ensure_staging(slot.color, color, D3D12_HEAP_TYPE_READBACK) ||
!ensure_staging(slot.depth, depth, D3D12_HEAP_TYPE_READBACK) ||
!ensure_staging(slot.motion, motion, D3D12_HEAP_TYPE_READBACK) ||
(exposure != nullptr && !ensure_staging(slot.exposure, exposure, D3D12_HEAP_TYPE_READBACK))))
return NGX_FAIL_PLATFORM_ERROR;
if (outputDesc.Format != feature->outputFormat || outputDesc.Width != feature->outputWidth ||
outputDesc.Height != feature->outputHeight)
{
// A new output size invalidates earlier results; let queued work finish
// before the output buffers go away.
drain_pipeline();
feature->latestOutput = -1;
for (OutputSlot& outputSlot : feature->outputs)
{
outputSlot.staging.retire();
if (p.vram ? !ensure_vram_buffer(*feature, outputSlot.vram, static_cast<size_t>(outputDesc.Width) * 8 *
outputDesc.Height)
: !ensure_staging(outputSlot.staging, output, D3D12_HEAP_TYPE_UPLOAD))
return NGX_FAIL_PLATFORM_ERROR;
}
feature->outputFormat = outputDesc.Format;
feature->outputWidth = static_cast<UINT>(outputDesc.Width);
feature->outputHeight = outputDesc.Height;
}
const auto inputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_INPUT_STATE", 0x40));
const auto depthState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_DEPTH_STATE", 0x40));
const auto outputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_OUTPUT_STATE", 0x8));
const auto inputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
if (p.vram)
{
const D3D12_RESOURCE_STATES states[4] = {inputState, depthState, inputState, inputState};
for (int index = 0; index < inputCount; ++index)
transition(list, inputs[index], states[index], D3D12_RESOURCE_STATE_COPY_SOURCE);
const bool recorded = record_vram_inputs(*feature, list, slot, vramInputs, inputCount);
for (int index = 0; index < inputCount; ++index)
transition(list, inputs[index], D3D12_RESOURCE_STATE_COPY_SOURCE, states[index]);
if (!recorded)
{
logf("frame %u: BeginVkCommandBufferInterop failed", frame);
return NGX_FAIL_PLATFORM_ERROR;
}
}
if (!p.vram || verify)
{
copy_to_staging(list, color, slot.color, inputState);
copy_to_staging(list, depth, slot.depth, depthState);
copy_to_staging(list, motion, slot.motion, inputState);
if (exposure != nullptr)
copy_to_staging(list, exposure, slot.exposure, inputState);
}
// Frame marker, written once the copies above have completed.
ID3D12GraphicsCommandList2* list2 = nullptr;
if (FAILED(list->QueryInterface(__uuidof(ID3D12GraphicsCommandList2), reinterpret_cast<void**>(&list2))))
{
logf("ID3D12GraphicsCommandList2 unavailable; cannot place frame marker");
return NGX_FAIL_PLATFORM_ERROR;
}
// Ray Reconstruction's auxiliary surfaces go into their own readback buffers here, still
// ahead of the frame marker so the worker can read them the moment that marker lands.
if (feature->rayReconstruction)
{
if (!capture_alpha_output(*feature, parameters, slotIndex, frame) ||
!gather_aux_inputs(*feature, list, parameters, slotIndex, inputState, frame))
{
// capture_alpha_output may already have taken a reference on the caller's texture,
// and this frame is never queued, so nothing else would release it.
InputSlot& failed = feature->inputs[slotIndex];
if (failed.alphaResource != nullptr)
{
failed.alphaResource->Release();
failed.alphaResource = nullptr;
failed.alphaRequested = false;
}
list2->Release();
return NGX_FAIL_PLATFORM_ERROR;
}
}
if (timing.enabled)
timing.inputRecord = profile_ms(inputRecordStart, ProfileClock::now());
const D3D12_GPU_VIRTUAL_ADDRESS markerAddress = feature->marker->GetGPUVirtualAddress();
D3D12_WRITEBUFFERIMMEDIATE_PARAMETER markers[] = {
{markerAddress + sizeof(uint32_t) * (1 + slotIndex), frame},
{markerAddress, frame},
};
D3D12_WRITEBUFFERIMMEDIATE_MODE modes[] = {
D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT, D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT,
};
list2->WriteBufferImmediate(2, markers, modes);
list2->Release();
// Present the most recent finished DLSS result.
int latest;
uint32_t colorFrame = 0; // the frame whose colour is presented below, 0 if none
{
std::lock_guard<std::mutex> lock(feature->outputMutex);
latest = feature->latestOutput.load();
if (latest >= 0)
{
feature->outputs[latest].lastReadFrame = frame;
colorFrame = feature->outputs[latest].producedFrame.load();
if (timing.enabled)
timing.presentedFrame = colorFrame;
}
}
const auto outputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
if (p.vram && feature->split)
p.split = SUCCEEDED(g_vk.split->SplitCommandListForExternalWait(
list, reinterpret_cast<UINT64>(feature->splitSemaphore), frame));
if (!p.split)
{
p.outputInPlace = 0; // only a split frame waits for the result before the game reads its texture
p.packedOutput = false;
}
if (p.split)
{
// Runs in the second half of the list, once frame's result is in place. Split frames
// present their own result rather than the newest one, so the alpha must pair with
// that one instead.
const int target = static_cast<int>(frame % kOutputSlots);
{
std::lock_guard<std::mutex> lock(feature->outputMutex);
colorFrame = feature->outputs[target].producedFrame.load();
}
if (p.outputInPlace == 0)
{
if (p.packedOutput)
vramOutput.convert = false; // the buffer already holds the texture's own texels
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
if (!record_vram_output(list, feature->outputs[target].vram, vramOutput, feature->outputConversion))
logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
}
if (timing.enabled)
timing.presentedFrame = frame;
}
else if (latest >= 0 && p.vram)
{
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
if (!record_vram_output(list, feature->outputs[latest].vram, vramOutput, feature->outputConversion))
logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
}
else if (latest >= 0)
{
OutputSlot& outputSlot = feature->outputs[latest];
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
D3D12_TEXTURE_COPY_LOCATION destination = {};
destination.pResource = output;
destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
destination.SubresourceIndex = 0;
D3D12_TEXTURE_COPY_LOCATION source = {};
source.pResource = outputSlot.staging.buffer;
source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
source.PlacedFootprint = outputSlot.staging.layout;
list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
}
if (feature->rayReconstruction)
record_alpha_copyback(*feature, list, slot, outputState, colorFrame);
if (timing.enabled)
{
timing.outputRecord = profile_ms(outputRecordStart, ProfileClock::now());
timing.queueDepth = g.prep.pending() + g.worker.pending() + g.finish.pending() + 1;
timing.queuedAt = ProfileClock::now();
timing.gameCall = profile_ms(timing.gameStart, timing.queuedAt);
}
slot.busy = true;
feature->inFlight.fetch_add(1);
if (feature->split)
{
std::lock_guard<std::mutex> lock(feature->splitMutex);
feature->splitCompletion.admit(frame);
}
g.prep.post([feature, slotIndex, frame, p, timing] { prepare_inputs(feature, slotIndex, frame, p, timing); });
if (frame <= 3 || frame % 600 == 0 || GetEnvironmentVariableA("D4R_SHIM_TRACE_FRAMES", nullptr, 0) != 0)
logf("evaluate frame %u queued (slot %d, showing result %d, jitter %.6f,%.6f, reset %d, "
"mvScale %.3f,%.3f, preExposure %.3f, exposureScale %.3f, frameTime %.3f, sharpness %.3f)",
frame, slotIndex, latest, p.jitterX, p.jitterY, p.reset, p.mvScaleX, p.mvScaleY,
p.preExposure, p.exposureScale, p.frameTime, p.sharpness);
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
void* parameters, void*)
{
return evaluate(list, handle, parameters);
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature_C(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
void* parameters, void*)
{
return evaluate(list, handle, parameters);
}
static void release_feature(Feature* feature)
{
// A queued GPU-side input wait for a frame that never ran would stall the network's stream: release it
// before the drain below waits for the worker (the worker also releases waits it sees stuck).
if (feature->engineNet != nullptr)
{
const uint32_t waiting = feature->engineWaitFrame.load();
if (waiting != 0)
release_engine_input_wait(*feature, waiting, "the feature is being released", false);
}
// Wait for outstanding evaluations, then free CUDA resources on the worker.
drain_pipeline();
g.worker.call([feature] {
if (feature->cudaHandle != nullptr)
g.ngx.releaseFeature(feature->cudaHandle);
if (feature->cudaParams != nullptr)
g.ngx.destroyParameters(feature->cudaParams);
// Every denoiser input dies with its feature, drains to nothing first.
for (AuxSurface& surface : feature->aux)
destroy_aux_surface(surface);
feature->aux.clear();
if (feature->alphaImage.object != 0)
g.cu.surfObjectDestroy(feature->alphaImage.object);
if (feature->alphaImage.array != nullptr)
g.cu.arrayDestroy(feature->alphaImage.array);
release_host(feature->alphaHost);
for (AlphaSlot& slot : feature->alpha)
slot.staging.release();
for (InputSlot& slot : feature->inputs)
if (slot.alphaResource != nullptr)
slot.alphaResource->Release();
for (InputSlot& slot : feature->inputs)
{
for (CudaObject& texture : slot.linearTexture)
if (texture != 0)
g.cu.texObjectDestroy(texture), texture = 0;
if (slot.dilatedTexture != 0)
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
if (slot.dilatedMotion != 0)
g.cu.memFree(slot.dilatedMotion), slot.dilatedMotion = 0;
}
for (CudaImage* image : {&feature->color, &feature->depth, &feature->motion, &feature->exposure})
{
if (image->object != 0)
g.cu.texObjectDestroy(image->object);
if (image->array != nullptr)
g.cu.arrayDestroy(image->array);
}
if (feature->output.object != 0)
g.cu.surfObjectDestroy(feature->output.object);
if (feature->output.array != nullptr)
g.cu.arrayDestroy(feature->output.array);
if (feature->profileStart != nullptr)
g.cu.eventDestroy(feature->profileStart);
if (feature->profileEnd != nullptr)
g.cu.eventDestroy(feature->profileEnd);
if (feature->outputReadyEvent != nullptr)
g.cu.eventDestroy(feature->outputReadyEvent);
if (feature->scratch != 0)
g.cu.memFree(feature->scratch);
for (CudaDevicePtr buffer : feature->planeLinear)
if (buffer != 0)
g.cu.memFree(buffer);
if (feature->outputLinear != 0)
g.cu.memFree(feature->outputLinear);
for (InputSlot& slot : feature->inputs)
for (HostPlane& host : slot.host)
release_host(host);
for (HostPlane& host : feature->outputHost)
release_host(host);
return 0;
});
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
g_features.erase(std::remove(g_features.begin(), g_features.end(), feature), g_features.end());
}
// Recorded lists retain this owner through their allocator. Reset/discard
// or GPU completion followed by allocator destruction releases those refs.
if (feature->resources != nullptr)
feature->resources->Release();
else
{
release_engine_network(*feature);
delete feature;
}
}
static void destroy_recorded_resources(Feature* feature)
{
for (InputSlot& slot : feature->inputs)
for (Staging* staging : {&slot.color, &slot.depth, &slot.motion, &slot.exposure})
staging->release();
for (OutputSlot& slot : feature->outputs)
slot.staging.release();
release_engine_network(*feature);
const bool engineSemaphore = !feature->vramDecided && feature->splitSemaphore != VK_NULL_HANDLE;
if (g_vk.ready && feature->vramDecided)
release_vram(*feature);
else if (engineSemaphore) // created for the engine's HIP network; release_vram destroys it otherwise
g_vk.destroySemaphore(g_vk.device, feature->splitSemaphore, nullptr);
if (feature->marker != nullptr)
feature->marker->Release();
logf("feature %u recorded resources retired after allocator completion", feature->handle.Id);
delete feature;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_ReleaseFeature(NgxHandle* handle)
{
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
Feature* feature = nullptr;
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
for (Feature* candidate : g_features)
if (&candidate->handle == handle && !candidate->nativeVulkan && !candidate->retiring.exchange(true))
{
feature = candidate;
break;
}
}
if (feature == nullptr || feature->nativeVulkan)
return NGX_FAIL_INVALID_PARAMETER;
logf("NVSDK_NGX_D3D12_ReleaseFeature(id=%u) after %u completed frames", feature->handle.Id,
feature->completedFrames.load());
release_feature(feature);
return NGX_SUCCESS;
}
static void finish_shutdown()
{
std::lock_guard<std::mutex> lock(g.mutex);
if (!g.shutdownPending || g.resourceOwners.load() != 0)
return;
if (g_vk.ready)
free_vram_pool();
if (g.ngxInitialized)
{
drain_pipeline();
g.worker.call([] { return g.ngx.shutdown(); });
}
// No separate denoiser shutdown: the core loaded it and the core's shutdown releases it.
g.ngxInitialized = false;
g.shutdownPending = false;
if (g.device != nullptr)
{
g.device->Release();
g.device = nullptr;
}
logf("NGX shutdown completed after recorded resources retired");
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown()
{
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
logf("NVSDK_NGX_D3D12_Shutdown");
std::vector<Feature*> features;
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
features = g_features;
for (Feature* feature : features)
if (feature->nativeVulkan)
return NGX_FAIL_INVALID_PARAMETER;
for (Feature* feature : features)
feature->retiring = true;
}
for (Feature* feature : features)
release_feature(feature);
{
std::lock_guard<std::mutex> lock(g.mutex);
if (!g.started)
return NGX_SUCCESS;
g.shutdownPending = true;
}
// Drain already queued cleanup before returning. If command allocators
// still own resources, finish_shutdown leaves NGX alive until their last
// reference retires; otherwise shutdown completes synchronously.
g.cleanup.call([] { finish_shutdown(); return 0; });
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown1(ID3D12Device*)
{
return NVSDK_NGX_D3D12_Shutdown();
}
D4R_EXPORT NgxResult NVSDK_NGX_UpdateFeature(const void*, unsigned int)
{
return NGX_SUCCESS;
}
#include "d4r_vulkan_frontend.inl"
BOOL WINAPI DllMain(HINSTANCE instance, DWORD reason, LPVOID)
{
if (reason == DLL_PROCESS_ATTACH)
{
g_selfModule = instance;
DisableThreadLibraryCalls(instance);
}
else if (reason == DLL_PROCESS_DETACH)
native_vk_process_exit();
return TRUE;
}
// The DLL entry point (linked with --entry). At process termination (reserved != NULL) ExitProcess
// has already killed the worker and waiter threads, possibly inside winpthreads' global condition
// variable spinlock, so the CRT's static destructors (mutexes, condition variables) can spin forever
// and the game never exits. Release the GPU waits and skip all CRT teardown then, as Windows advises;
// the system reclaims everything. Every other notification goes through the CRT as usual.
extern "C" BOOL WINAPI DllMainCRTStartup(HANDLE, DWORD, LPVOID);
extern "C" BOOL WINAPI d4r_dll_entry(HANDLE instance, DWORD reason, LPVOID reserved)
{
if (reason == DLL_PROCESS_DETACH && reserved != nullptr)
{
native_vk_process_exit();
return TRUE;
}
return DllMainCRTStartup(instance, reason, reserved);
}
Reference in new issue View Git Blame Copy Permalink