Tk-Glitch
40e0094c19
Alan Wake 2 uses the format for depth when enabling Ray Reconstruction. The CPU path converts half to float in convert_row_in. The VRAM interop path blits R16F/R16UNORM depth to an R32F conversion image before copying into the canonical buffer. Signed-off-by: Tk-Glitch <[email protected]>
5241 lines
231 KiB
C++
5241 lines
231 KiB
C++
// d4r_nvngx.dll: an NGX core replacement that implements the D3D12 DLSS
|
|
// entry points on top of the official NGX core's CUDA path (which runs on AMD
|
|
// through the Wine nvcuda bridge and ZLUDA). OptiScaler loads it through its
|
|
// NvngxPath setting.
|
|
//
|
|
// A D3D12 EvaluateFeature call only records into the game's still-open command
|
|
// list, while DLSS runs on a separate CUDA context. Each evaluation therefore:
|
|
// 1. records copies of the inputs into readback buffers plus a GPU-written
|
|
// frame marker (WriteBufferImmediate) on the game's command list,
|
|
// 2. records a copy of the most recent finished DLSS result from an upload
|
|
// buffer into the output texture,
|
|
// 3. queues a CUDA job that waits for the marker, uploads the inputs into
|
|
// CUDA arrays, runs the official DLSS evaluation, and stores the result
|
|
// into an upload buffer for a later frame.
|
|
// The output is therefore one or more frames behind the game's other passes.
|
|
//
|
|
// Environment:
|
|
// D4R_NGX_CORE Windows path of the official _nvngx.dll (required)
|
|
// D4R_NGX_FEATURE_DIR directory holding nvngx_dlss.dll (optional, added to
|
|
// the feature search paths)
|
|
// D4R_SHIM_LOG log file path (default: d4r_nvngx.log next to the DLL)
|
|
// D4R_PROFILE=1 log per-frame CPU stages and default-stream GPU event time
|
|
// D4R_SHIM_OUTPUT_HASH=1 log an FNV-1a hash of every DLSS output as produced
|
|
// D4R_SHIM_OUTPUT_DUMP_DIR=<Windows path> save selected raw RGBA16F DLSS frames
|
|
// D4R_SHIM_OUTPUT_DUMP_START / _COUNT / _EVERY select frames (defaults 1/1/1)
|
|
// D4R_SHIM_INPUT_DUMP_DIR=<Windows path> save selected canonical input planes
|
|
// (use D4R_SHIM_VRAM_VERIFY=1 with VRAM interop)
|
|
// D4R_SHIM_CAPTURE_TRIGGER=<Windows path> select the next frame when this file appears
|
|
// (D4R_SHIM_CAPTURE_COUNT=N: that frame and the N-1 after it;
|
|
// D4R_SHIM_CAPTURE_MEMORY=1 buffers them in memory, written afterwards)
|
|
// D4R_SHIM_INPUT_STATE / D4R_SHIM_DEPTH_STATE / D4R_SHIM_OUTPUT_STATE
|
|
// D3D12 resource states the inputs/output are in when
|
|
// EvaluateFeature is called (defaults 0x40, 0x40, 0x8)
|
|
// D4R_SHIM_VRAM_INTEROP=1 keep inputs and output in VRAM (see "VRAM interop")
|
|
// D4R_SHIM_SPLIT_FRAME=1 with VRAM interop and the d4r vkd3d-proton patch,
|
|
// present each frame's own DLSS result (see "Split frames")
|
|
#define WIDL_EXPLICIT_AGGREGATE_RETURNS
|
|
#include <windows.h>
|
|
#include <d3d12.h>
|
|
#include <vulkan/vulkan_core.h>
|
|
#include "d4r_motion_dilation.h"
|
|
|
|
#include <algorithm>
|
|
#include <bit>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <condition_variable>
|
|
#include <cstdarg>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <deque>
|
|
#include <functional>
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <string>
|
|
#include <thread>
|
|
#include <vector>
|
|
#include "d4r_event_wait.h"
|
|
#include "d4r_frame_completion.h"
|
|
#include "d4r_win32_wait.h"
|
|
#include "d4r_vkd3d_interop.h"
|
|
|
|
// --- NGX types (declared locally: the core's exports differ from the SDK) --
|
|
|
|
using NgxResult = unsigned int;
|
|
constexpr NgxResult NGX_SUCCESS = 0x1;
|
|
constexpr NgxResult NGX_FAIL = 0xBAD00000;
|
|
constexpr NgxResult NGX_FAIL_FEATURE_NOT_SUPPORTED = NGX_FAIL | 1;
|
|
constexpr NgxResult NGX_FAIL_PLATFORM_ERROR = NGX_FAIL | 2;
|
|
constexpr NgxResult NGX_FAIL_INVALID_PARAMETER = NGX_FAIL | 5;
|
|
constexpr NgxResult NGX_FAIL_NOT_INITIALIZED = NGX_FAIL | 7;
|
|
constexpr NgxResult NGX_FAIL_UNSUPPORTED_FORMAT = NGX_FAIL | 14;
|
|
constexpr unsigned int NGX_FEATURE_SUPER_SAMPLING = 1;
|
|
|
|
struct NgxHandle
|
|
{
|
|
unsigned int Id;
|
|
};
|
|
|
|
struct NgxPathListInfo
|
|
{
|
|
const wchar_t* const* Path;
|
|
unsigned int Length;
|
|
};
|
|
|
|
struct NgxLoggingInfo
|
|
{
|
|
void* LoggingCallback;
|
|
int MinimumLoggingLevel;
|
|
bool DisableOtherLoggingSinks;
|
|
};
|
|
|
|
struct NgxFeatureCommonInfo
|
|
{
|
|
NgxPathListInfo PathListInfo;
|
|
void* InternalData;
|
|
NgxLoggingInfo LoggingInfo;
|
|
};
|
|
|
|
struct NgxFeatureRequirement
|
|
{
|
|
unsigned int FeatureSupported;
|
|
unsigned int MinHWArchitecture;
|
|
char MinOSVersion[255];
|
|
};
|
|
|
|
// MSVC-ABI parameter accessors (d4r_ngx_param_msvc.cpp, built with clang-cl).
|
|
extern "C"
|
|
{
|
|
void d4r_ngx_set_ull(void* parameters, const char* name, unsigned long long value);
|
|
void d4r_ngx_set_float(void* parameters, const char* name, float value);
|
|
void d4r_ngx_set_uint(void* parameters, const char* name, unsigned int value);
|
|
void d4r_ngx_set_int(void* parameters, const char* name, int value);
|
|
void d4r_ngx_set_void(void* parameters, const char* name, void* value);
|
|
NgxResult d4r_ngx_get_ull(void* parameters, const char* name, unsigned long long* value);
|
|
NgxResult d4r_ngx_get_float(void* parameters, const char* name, float* value);
|
|
NgxResult d4r_ngx_get_uint(void* parameters, const char* name, unsigned int* value);
|
|
NgxResult d4r_ngx_get_int(void* parameters, const char* name, int* value);
|
|
NgxResult d4r_ngx_get_void(void* parameters, const char* name, void** value);
|
|
NgxResult d4r_ngx_get_d3d12_resource(void* parameters, const char* name, ID3D12Resource** value);
|
|
void* d4r_ngx_parameters_create();
|
|
void d4r_ngx_parameters_destroy(void* parameters);
|
|
}
|
|
|
|
// --- CUDA driver API (through nvcuda.dll, the Wine bridge to ZLUDA) --------
|
|
|
|
using CudaDevicePtr = unsigned long long;
|
|
using CudaArray = void*;
|
|
using CudaObject = unsigned long long;
|
|
using CudaEvent = void*;
|
|
|
|
struct CudaArrayDescriptor
|
|
{
|
|
size_t Width;
|
|
size_t Height;
|
|
uint32_t Format;
|
|
uint32_t NumChannels;
|
|
};
|
|
|
|
struct CudaArray3DDescriptor
|
|
{
|
|
size_t Width;
|
|
size_t Height;
|
|
size_t Depth;
|
|
uint32_t Format;
|
|
uint32_t NumChannels;
|
|
uint32_t Flags;
|
|
};
|
|
|
|
struct CudaMemcpy2D
|
|
{
|
|
size_t srcXInBytes;
|
|
size_t srcY;
|
|
uint32_t srcMemoryType;
|
|
uint32_t srcAlignment;
|
|
const void* srcHost;
|
|
CudaDevicePtr srcDevice;
|
|
CudaArray srcArray;
|
|
size_t srcPitch;
|
|
size_t dstXInBytes;
|
|
size_t dstY;
|
|
uint32_t dstMemoryType;
|
|
uint32_t dstAlignment;
|
|
void* dstHost;
|
|
CudaDevicePtr dstDevice;
|
|
CudaArray dstArray;
|
|
size_t dstPitch;
|
|
size_t WidthInBytes;
|
|
size_t Height;
|
|
};
|
|
static_assert(sizeof(CudaMemcpy2D) == 128);
|
|
|
|
struct CudaResourceDesc
|
|
{
|
|
uint32_t resType;
|
|
uint32_t alignment;
|
|
union
|
|
{
|
|
struct
|
|
{
|
|
CudaArray hArray;
|
|
} array;
|
|
int reserved[32];
|
|
} res;
|
|
uint32_t flags;
|
|
uint32_t reserved;
|
|
};
|
|
static_assert(sizeof(CudaResourceDesc) == 144);
|
|
|
|
struct CudaTextureDesc
|
|
{
|
|
uint32_t addressMode[3];
|
|
uint32_t filterMode;
|
|
uint32_t flags;
|
|
uint32_t maxAnisotropy;
|
|
uint32_t mipmapFilterMode;
|
|
float mipmapLevelBias;
|
|
float minMipmapLevelClamp;
|
|
float maxMipmapLevelClamp;
|
|
float borderColor[4];
|
|
int32_t reserved[12];
|
|
};
|
|
static_assert(sizeof(CudaTextureDesc) == 104);
|
|
|
|
constexpr uint32_t CUDA_FORMAT_HALF = 0x10;
|
|
constexpr uint32_t CUDA_FORMAT_FLOAT = 0x20;
|
|
constexpr uint32_t CUDA_MEMORY_HOST = 1;
|
|
constexpr uint32_t CUDA_MEMORY_DEVICE = 2;
|
|
constexpr uint32_t CUDA_MEMORY_ARRAY = 3;
|
|
|
|
struct CudaApi
|
|
{
|
|
int(WINAPI* memAlloc)(CudaDevicePtr*, size_t) = nullptr;
|
|
int(WINAPI* memFree)(CudaDevicePtr) = nullptr;
|
|
int(WINAPI* arrayCreate)(CudaArray*, const CudaArrayDescriptor*) = nullptr;
|
|
int(WINAPI* array3DCreate)(CudaArray*, const CudaArray3DDescriptor*) = nullptr;
|
|
int(WINAPI* arrayDestroy)(CudaArray) = nullptr;
|
|
int(WINAPI* memcpy2D)(const CudaMemcpy2D*) = nullptr;
|
|
int(WINAPI* texObjectCreate)(CudaObject*, const CudaResourceDesc*, const CudaTextureDesc*, const void*) = nullptr;
|
|
int(WINAPI* texObjectDestroy)(CudaObject) = nullptr;
|
|
int(WINAPI* surfObjectCreate)(CudaObject*, const CudaResourceDesc*) = nullptr;
|
|
int(WINAPI* surfObjectDestroy)(CudaObject) = nullptr;
|
|
int(WINAPI* ctxSynchronize)() = nullptr;
|
|
int(WINAPI* eventCreate)(CudaEvent*, unsigned int) = nullptr;
|
|
int(WINAPI* eventRecord)(CudaEvent, void*) = nullptr;
|
|
int(WINAPI* eventQuery)(CudaEvent) = nullptr;
|
|
int(WINAPI* eventSynchronize)(CudaEvent) = nullptr;
|
|
int(WINAPI* eventElapsedTime)(float*, CudaEvent, CudaEvent) = nullptr;
|
|
int(WINAPI* eventDestroy)(CudaEvent) = nullptr;
|
|
// Optional: overlapped staging (see async_copies()).
|
|
int(WINAPI* memAllocHost)(void**, size_t) = nullptr;
|
|
int(WINAPI* memFreeHost)(void*) = nullptr;
|
|
int(WINAPI* memcpyHtoDAsync)(CudaDevicePtr, const void*, size_t, void*) = nullptr;
|
|
int(WINAPI* memcpyDtoHAsync)(void*, CudaDevicePtr, size_t, void*) = nullptr;
|
|
int(WINAPI* streamCreate)(void**, unsigned int) = nullptr;
|
|
int(WINAPI* streamSynchronize)(void*) = nullptr;
|
|
// Optional (d4r nvcuda bridge): device <-> array copies queued on the null stream.
|
|
int(WINAPI* memcpy2DAsync)(const CudaMemcpy2D*, void*) = nullptr;
|
|
// Optional (d4r nvcuda bridge): null-stream wait for a device u32, and a write from another stream.
|
|
int(WINAPI* streamWaitValue32)(CudaDevicePtr, uint32_t) = nullptr;
|
|
int(WINAPI* writeValue32)(CudaDevicePtr, uint32_t) = nullptr;
|
|
// Optional (d4r nvcuda bridge): surfaces of an array store to linear memory instead (d4r native kernels).
|
|
int(WINAPI* setArrayRedirect)(CudaArray, CudaDevicePtr, uint32_t) = nullptr;
|
|
int(WINAPI* outputKernelNative)() = nullptr;
|
|
// Optional (d4r nvcuda bridge): report a pitch-linear texture to NGX as an array of that size/format.
|
|
int(WINAPI* registerLinearTexture)(CudaObject, size_t, size_t, uint32_t, uint32_t) = nullptr;
|
|
int(WINAPI* dilateMotion)(const D4rMotionDilationParams*) = nullptr;
|
|
};
|
|
|
|
// --- official NGX core CUDA API --------------------------------------------
|
|
|
|
struct CoreApi
|
|
{
|
|
NgxResult (*init)(unsigned long long, const wchar_t*, const NgxFeatureCommonInfo*, unsigned int) = nullptr;
|
|
NgxResult (*initProjectId)(const char*, int, const char*, const wchar_t*, unsigned int,
|
|
const NgxFeatureCommonInfo*) = nullptr;
|
|
NgxResult (*shutdown)() = nullptr;
|
|
NgxResult (*getParameters)(void**) = nullptr;
|
|
NgxResult (*allocateParameters)(void**) = nullptr;
|
|
NgxResult (*getCapabilityParameters)(void**) = nullptr;
|
|
NgxResult (*destroyParameters)(void*) = nullptr;
|
|
NgxResult (*createFeature)(unsigned int, void*, NgxHandle**) = nullptr;
|
|
NgxResult (*releaseFeature)(NgxHandle*) = nullptr;
|
|
NgxResult (*evaluateFeature)(const NgxHandle*, void*, void*) = nullptr;
|
|
};
|
|
|
|
// --- logging -----------------------------------------------------------------
|
|
|
|
static std::mutex g_logMutex;
|
|
static FILE* g_log = nullptr;
|
|
static HMODULE g_selfModule = nullptr;
|
|
|
|
// --- portable install ----------------------------------------------------------
|
|
// The drag-in release keeps this DLL (as nvngx.dll, explicitly selected by OptiScaler's NvngxPath),
|
|
// the nvcuda bridge, ZLUDA and the native kernels in an d4r folder next to the game's executable,
|
|
// with that game's settings in d4r\d4r.ini. The file's settings become the environment variables the
|
|
// developer launcher sets (config/d4r.ini.default documents them), and a variable that is already set,
|
|
// for example in the game's launch options, wins. Settings for the Linux side (ZLUDA, the bridge) are
|
|
// handed to the bridge right after it is loaded, before its first CUDA call. Without an d4r.ini next
|
|
// to the DLL nothing changes.
|
|
|
|
struct PortableInstall
|
|
{
|
|
bool active = false;
|
|
std::wstring dir; // the d4r folder
|
|
std::string unixDir; // the same folder as a Linux path
|
|
std::vector<std::pair<std::string, std::string>> unixEnv; // for the bridge's d4rSetEnv
|
|
std::vector<std::string> notes; // logged once the log is open
|
|
};
|
|
static PortableInstall g_portable;
|
|
static std::once_flag g_portableOnce;
|
|
static bool g_logFresh = false; // a portable install starts a new log for every launch
|
|
|
|
struct IniEntry
|
|
{
|
|
std::string section, key, value;
|
|
};
|
|
|
|
static std::string ascii_lower(std::string text)
|
|
{
|
|
for (char& c : text)
|
|
c = static_cast<char>(tolower(static_cast<unsigned char>(c)));
|
|
return text;
|
|
}
|
|
|
|
static std::string trim(const std::string& text)
|
|
{
|
|
const size_t first = text.find_first_not_of(" \t\r\n");
|
|
if (first == std::string::npos)
|
|
return std::string();
|
|
return text.substr(first, text.find_last_not_of(" \t\r\n") - first + 1);
|
|
}
|
|
|
|
// INI in the configparser dialect d4r_config.py reads: [Section], key = value, full-line comments
|
|
// with ; or #, inline comments after whitespace.
|
|
static std::vector<IniEntry> parse_ini(const std::string& text)
|
|
{
|
|
std::vector<IniEntry> entries;
|
|
std::string section;
|
|
size_t start = text.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0;
|
|
while (start < text.size())
|
|
{
|
|
size_t end = text.find('\n', start);
|
|
if (end == std::string::npos)
|
|
end = text.size();
|
|
const std::string line = trim(text.substr(start, end - start));
|
|
start = end + 1;
|
|
if (line.empty() || line[0] == ';' || line[0] == '#')
|
|
continue;
|
|
if (line[0] == '[')
|
|
{
|
|
section = ascii_lower(trim(line.substr(1, line.find(']') - 1)));
|
|
continue;
|
|
}
|
|
const size_t equals = line.find('=');
|
|
if (equals == std::string::npos)
|
|
continue;
|
|
std::string value = line.substr(equals + 1);
|
|
for (size_t i = 1; i < value.size(); ++i)
|
|
if ((value[i] == ';' || value[i] == '#') && (value[i - 1] == ' ' || value[i - 1] == '\t'))
|
|
{
|
|
value.resize(i);
|
|
break;
|
|
}
|
|
entries.push_back({section, trim(line.substr(0, equals)), trim(value)});
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
// The last value for [section] key; "" when it is missing, empty or "auto".
|
|
static std::string ini_value(const std::vector<IniEntry>& ini, const char* section, const char* key)
|
|
{
|
|
std::string result;
|
|
for (const IniEntry& entry : ini)
|
|
if (entry.section == section && ascii_lower(entry.key) == ascii_lower(key))
|
|
result = entry.value;
|
|
return ascii_lower(result) == "auto" ? std::string() : result;
|
|
}
|
|
|
|
// 1 or 0 for a boolean setting, fallback when it is unset or not a boolean.
|
|
static int ini_flag(const std::vector<IniEntry>& ini, const char* section, const char* key, int fallback)
|
|
{
|
|
const std::string value = ascii_lower(ini_value(ini, section, key));
|
|
if (value.empty())
|
|
return fallback;
|
|
if (value == "1" || value == "true" || value == "yes" || value == "on")
|
|
return 1;
|
|
if (value == "0" || value == "false" || value == "no" || value == "off")
|
|
return 0;
|
|
g_portable.notes.push_back("d4r.ini: [" + std::string(section) + "] " + key + " must be true or false, not '" +
|
|
value + "'; using " + (fallback ? "true" : "false"));
|
|
return fallback;
|
|
}
|
|
|
|
static bool read_whole_file(const std::wstring& path, std::string& text)
|
|
{
|
|
HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
|
|
FILE_ATTRIBUTE_NORMAL, nullptr);
|
|
if (file == INVALID_HANDLE_VALUE)
|
|
return false;
|
|
LARGE_INTEGER size = {};
|
|
bool ok = GetFileSizeEx(file, &size) && size.QuadPart < (1 << 20);
|
|
if (ok)
|
|
{
|
|
text.resize(static_cast<size_t>(size.QuadPart));
|
|
DWORD read = 0;
|
|
ok = text.empty() || (ReadFile(file, text.data(), static_cast<DWORD>(text.size()), &read, nullptr) &&
|
|
read == text.size());
|
|
}
|
|
CloseHandle(file);
|
|
return ok;
|
|
}
|
|
|
|
static bool file_exists(const std::wstring& path)
|
|
{
|
|
const DWORD attributes = GetFileAttributesW(path.c_str());
|
|
return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
|
|
}
|
|
|
|
// Wine's kernel32 maps a Windows path to the Linux one.
|
|
static std::string unix_path(const std::wstring& path)
|
|
{
|
|
using UnixFileNameFn = char*(CDECL*)(const wchar_t*);
|
|
const auto function = reinterpret_cast<UnixFileNameFn>(
|
|
reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "wine_get_unix_file_name")));
|
|
char* result = function != nullptr ? function(path.c_str()) : nullptr;
|
|
if (result == nullptr)
|
|
return std::string();
|
|
std::string text(result);
|
|
HeapFree(GetProcessHeap(), 0, result);
|
|
return text;
|
|
}
|
|
|
|
static std::wstring widen(const std::string& text)
|
|
{
|
|
std::wstring wide(text.size() + 1, L'\0');
|
|
const int length = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), -1, wide.data(), static_cast<int>(wide.size()));
|
|
wide.resize(length > 0 ? static_cast<size_t>(length - 1) : 0);
|
|
return wide;
|
|
}
|
|
|
|
// A Windows path from d4r.ini: absolute (C:\..., \\...) or relative to the d4r folder.
|
|
static std::wstring portable_path(const std::string& value)
|
|
{
|
|
const std::wstring path = widen(value);
|
|
if ((path.size() > 2 && path[1] == L':') || (path.size() > 1 && path[0] == L'\\' && path[1] == L'\\'))
|
|
return path;
|
|
return g_portable.dir + L"\\" + path;
|
|
}
|
|
|
|
static void portable_set(const char* name, const std::string& value)
|
|
{
|
|
if (GetEnvironmentVariableA(name, nullptr, 0) == 0)
|
|
SetEnvironmentVariableA(name, value.c_str());
|
|
}
|
|
|
|
static void portable_set(const wchar_t* name, const std::wstring& value)
|
|
{
|
|
if (GetEnvironmentVariableW(name, nullptr, 0) == 0)
|
|
SetEnvironmentVariableW(name, value.c_str());
|
|
}
|
|
|
|
static void portable_set_unix(const char* name, const std::string& value)
|
|
{
|
|
g_portable.unixEnv.emplace_back(name, value);
|
|
}
|
|
|
|
// [DLSS] Model: a preset letter, a friendly name or a render-preset number; "" leaves the choice to the game.
|
|
static std::string preset_number(const std::string& model)
|
|
{
|
|
std::string name;
|
|
for (char c : model)
|
|
name += static_cast<char>(toupper(static_cast<unsigned char>(c)));
|
|
if (name.empty() || name == "GAME" || name == "DEFAULT")
|
|
return std::string();
|
|
if (name.find_first_not_of("0123456789") == std::string::npos)
|
|
return name;
|
|
static const std::pair<const char*, const char*> aliases[] = {
|
|
{"CNN", "E"}, {"DLSS3", "E"}, {"DLSS4", "K"}, {"DLSS4.5", "M"}, {"TRANSFORMER", "K"}};
|
|
for (const auto& [alias, letter] : aliases)
|
|
if (name == alias)
|
|
name = letter;
|
|
static const std::pair<const char*, int> letters[] = {{"A", 1}, {"B", 2}, {"C", 3}, {"D", 4}, {"E", 5}, {"F", 6},
|
|
{"G", 7}, {"J", 10}, {"K", 11}, {"L", 12}, {"M", 13}};
|
|
for (const auto& [letter, number] : letters)
|
|
if (name == letter)
|
|
return std::to_string(number);
|
|
g_portable.notes.push_back("d4r.ini: [DLSS] Model '" + model + "' is not a preset (use K, E, L, M or a number); "
|
|
"leaving the preset to the game");
|
|
return std::string();
|
|
}
|
|
|
|
static void load_portable_config()
|
|
{
|
|
wchar_t module[MAX_PATH] = {};
|
|
const DWORD length = GetModuleFileNameW(g_selfModule, module, MAX_PATH);
|
|
if (length == 0 || length >= MAX_PATH)
|
|
return;
|
|
std::wstring dir(module, length);
|
|
dir.resize(dir.find_last_of(L"\\/"));
|
|
g_portable.dir = dir; // also without d4r.ini: the default log goes here
|
|
std::string text;
|
|
if (!read_whole_file(dir + L"\\d4r.ini", text))
|
|
return; // developer layout: the launcher's environment has everything
|
|
const std::vector<IniEntry> ini = parse_ini(text);
|
|
g_portable.active = true;
|
|
g_portable.unixDir = unix_path(dir);
|
|
if (g_portable.unixDir.empty())
|
|
g_portable.notes.push_back("cannot map the d4r folder to a Linux path (not running under Wine?)");
|
|
|
|
// Files of the install; NGX looks for nvngx_dlss.dll next to this DLL.
|
|
portable_set(L"D4R_NVCUDA_BRIDGE", dir + L"\\nvcuda.dll");
|
|
const std::string core = ini_value(ini, "paths", "NgxCore");
|
|
portable_set(L"D4R_NGX_CORE", core.empty() ? dir + L"\\ngx\\_nvngx.dll" : portable_path(core));
|
|
portable_set(L"D4R_NGX_FEATURE_DIR", dir);
|
|
const std::string log = ini_value(ini, "debug", "Log");
|
|
g_logFresh = log.empty();
|
|
portable_set(L"D4R_SHIM_LOG", log.empty() ? dir + L"\\d4r_nvngx.log" : portable_path(log));
|
|
portable_set("D4R_CUDA_CAPTURE", "0"); // the bridge would otherwise save every DLSS module it loads
|
|
|
|
const std::string zluda = ini_value(ini, "paths", "ZludaDir");
|
|
portable_set_unix("D4R_ZLUDA_LIBCUDA", (zluda.empty() ? g_portable.unixDir + "/zluda" : zluda) + "/libcuda.so");
|
|
// The release's bundled ROCm runtime unless d4r.ini names another
|
|
const std::string rocm = ini_value(ini, "paths", "RocmDir");
|
|
if (!rocm.empty())
|
|
portable_set_unix("D4R_ROCM_DIR", rocm);
|
|
else if (!g_portable.unixDir.empty() && GetFileAttributesW((dir + L"\\rocm\\lib").c_str()) != INVALID_FILE_ATTRIBUTES)
|
|
portable_set_unix("D4R_ROCM_DIR", g_portable.unixDir + "/rocm");
|
|
const std::string cache = ini_value(ini, "paths", "CacheDir");
|
|
portable_set_unix("D4R_ZLUDA_CACHE_HOME", cache.empty() ? "~/.cache/d4r" : cache);
|
|
|
|
std::string model = ini_value(ini, "dlss", "Model");
|
|
if (ascii_lower(model) != "game")
|
|
model = preset_number(model.empty() ? "K" : model);
|
|
else
|
|
model.clear();
|
|
if (!model.empty())
|
|
portable_set("D4R_DLSS_PRESET", model);
|
|
|
|
const std::string age = ini_value(ini, "latency", "FrameAge");
|
|
if (age.empty() || age == "0")
|
|
{
|
|
portable_set("D4R_SHIM_SPLIT_FRAME", "1");
|
|
portable_set("D4R_SHIM_MAX_IN_FLIGHT", "3");
|
|
}
|
|
else if (age == "1" || age == "2" || age == "3")
|
|
{
|
|
portable_set("D4R_SHIM_SPLIT_FRAME", "0");
|
|
portable_set("D4R_SHIM_MAX_IN_FLIGHT", age);
|
|
}
|
|
else
|
|
g_portable.notes.push_back("d4r.ini: [Latency] FrameAge must be 0-3, not '" + age + "'");
|
|
|
|
// Native kernels: d4r\kernels, each used only when DLSS's PTX for it matches (see the bridge).
|
|
const std::string native = ascii_lower(ini_value(ini, "kernels", "NativeKernels"));
|
|
// The bridge selects accuracy variants and enforces their compiler/sync policy before cuInit.
|
|
portable_set_unix("D4R_PREFER_ACCURACY", ini_flag(ini, "kernels", "PreferAccuracy", 0) ? "1" : "0");
|
|
portable_set_unix("D4R_NATIVE_SWIN_ENCODERS", ini_flag(ini, "kernels", "NativeSwinEncoders", 1) ? "1" : "0");
|
|
const bool nativeOn = native.empty() || native == "on" || native == "true" || native == "1" || native == "fast";
|
|
if (!nativeOn && native != "off" && native != "false" && native != "0")
|
|
g_portable.notes.push_back("d4r.ini: [Kernels] NativeKernels must be on or off, not '" + native + "'");
|
|
if (nativeOn && !g_portable.unixDir.empty())
|
|
portable_set_unix("D4R_ZLUDA_NATIVE_DIR", g_portable.unixDir + "/kernels");
|
|
portable_set_unix("D4R_ZLUDA_WMMA", ini_flag(ini, "kernels", "Wmma", 1) ? "1" : "0");
|
|
portable_set_unix("D4R_ZLUDA_WMMA_FP8", ini_flag(ini, "kernels", "Fp8Wmma", 1) ? "1" : "0");
|
|
// RDNA4's native FP8 WMMA; ZLUDA and the bridge ignore it on other GPUs
|
|
portable_set_unix("D4R_ZLUDA_WMMA_FP8_NATIVE", ini_flag(ini, "kernels", "NativeFp8", 1) ? "1" : "0");
|
|
portable_set_unix("D4R_ZLUDA_IGNORE_DENORMAL", ini_flag(ini, "kernels", "IgnoreDenormals", 1) ? "1" : "0");
|
|
const std::string maxBlock = ini_value(ini, "kernels", "ImplicitMaxBlock");
|
|
portable_set_unix("D4R_ZLUDA_IMPLICIT_MAX_BLOCK", maxBlock.empty() ? "256" : maxBlock);
|
|
|
|
static const struct
|
|
{
|
|
const char* key;
|
|
const char* variable;
|
|
int fallback;
|
|
} interop[] = {{"VramInterop", "D4R_SHIM_VRAM_INTEROP", 1}, {"InputSync", "D4R_SHIM_INPUT_SYNC", 1},
|
|
{"LinearInputs", "D4R_SHIM_LINEAR_INPUTS", 1},
|
|
{"EvalSync", "D4R_SHIM_EVAL_SYNC", 0}};
|
|
for (const auto& setting : interop)
|
|
portable_set(setting.variable, ini_flag(ini, "interop", setting.key, setting.fallback) ? "1" : "0");
|
|
portable_set_unix("D4R_ELIDE_NGX_SYNC", ini_flag(ini, "interop", "ElideNgxSync", 1) ? "1" : "0");
|
|
const std::string poll = ini_value(ini, "interop", "MarkerPollUs");
|
|
portable_set("D4R_SHIM_MARKER_POLL_US", poll.empty() ? "200" : poll);
|
|
// Direct output takes effect only while NGX runs the native output kernel (the bridge reports it); the
|
|
// release cannot ship that kernel, which is built from NVIDIA's PTX.
|
|
portable_set("D4R_SHIM_OUTPUT_DIRECT", ini_flag(ini, "interop", "DirectOutput", nativeOn) ? "1" : "0");
|
|
|
|
portable_set("D4R_SHIM_WATERMARK", ini_flag(ini, "dlss", "ShowWatermark", 0) ? "1" : "0");
|
|
const std::string dilation = ini_value(ini, "interop", "MotionVectorDilation");
|
|
if (dilation.empty() || dilation == "0" || dilation == "1" || dilation == "2")
|
|
portable_set("D4R_MOTION_DILATION", dilation.empty() ? "0" : dilation);
|
|
else
|
|
g_portable.notes.push_back("MotionVectorDilation must be 0, 1 or 2; using original motion vectors");
|
|
if (ini_flag(ini, "debug", "Profile", 0))
|
|
portable_set("D4R_PROFILE", "1");
|
|
if (const std::string level = ini_value(ini, "debug", "NgxLogLevel"); !level.empty())
|
|
portable_set("D4R_NGX_LOG_LEVEL", level);
|
|
|
|
// [Env]: any other variable, for both sides
|
|
for (const IniEntry& entry : ini)
|
|
if (entry.section == "env" && !entry.key.empty())
|
|
{
|
|
portable_set(entry.key.c_str(), entry.value);
|
|
portable_set_unix(entry.key.c_str(), entry.value);
|
|
}
|
|
|
|
if (!file_exists(dir + L"\\nvngx_dlss.dll"))
|
|
g_portable.notes.push_back("nvngx_dlss.dll is missing: copy NVIDIA's DLSS library (310.7 or 310.9 recommended) "
|
|
"into the d4r folder");
|
|
wchar_t corePath[MAX_PATH] = {};
|
|
const DWORD coreLength = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
|
|
if (coreLength > 0 && coreLength < MAX_PATH && !file_exists(corePath))
|
|
g_portable.notes.push_back("the NGX core is missing: copy NVIDIA's _nvngx.dll into d4r\\ngx");
|
|
}
|
|
|
|
static void ensure_portable_config()
|
|
{
|
|
std::call_once(g_portableOnce, load_portable_config);
|
|
}
|
|
|
|
static void log_open()
|
|
{
|
|
ensure_portable_config();
|
|
wchar_t path[MAX_PATH] = {};
|
|
DWORD length = GetEnvironmentVariableW(L"D4R_SHIM_LOG", path, MAX_PATH);
|
|
if (length > 0 && length < MAX_PATH)
|
|
g_log = _wfopen(path, g_logFresh ? L"w" : L"a");
|
|
else if (!g_portable.dir.empty()) // no log when the DLL's folder is unknown
|
|
g_log = _wfopen((g_portable.dir + L"\\d4r_nvngx.log").c_str(), g_logFresh ? L"w" : L"a");
|
|
}
|
|
|
|
static void logf(const char* format, ...)
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_logMutex);
|
|
if (g_log == nullptr)
|
|
log_open();
|
|
if (g_log == nullptr)
|
|
return;
|
|
std::fprintf(g_log, "[%10.3f] ", GetTickCount64() / 1000.0);
|
|
va_list arguments;
|
|
va_start(arguments, format);
|
|
std::vfprintf(g_log, format, arguments);
|
|
va_end(arguments);
|
|
std::fputc('\n', g_log);
|
|
std::fflush(g_log);
|
|
}
|
|
|
|
// NGX core and feature diagnostics, routed into the shim log.
|
|
static void ngx_log_callback(const char* message, int level, unsigned int component)
|
|
{
|
|
if (message == nullptr)
|
|
return;
|
|
std::string line(message);
|
|
while (!line.empty() && (line.back() == '\n' || line.back() == '\r'))
|
|
line.pop_back();
|
|
logf("NGX[%d/%u] %s", level, component, line.c_str());
|
|
}
|
|
|
|
static unsigned int env_uint(const char* name, unsigned int fallback)
|
|
{
|
|
char value[64] = {};
|
|
DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
|
|
if (length == 0 || length >= sizeof(value) || value[0] < '0' || value[0] > '9')
|
|
return fallback; // also signs and text: strtoul turns "-1" into 4294967295
|
|
return static_cast<unsigned int>(std::strtoul(value, nullptr, 0));
|
|
}
|
|
|
|
static std::string env_string(const char* name)
|
|
{
|
|
char value[1024] = {};
|
|
const DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
|
|
return length > 0 && length < sizeof(value) ? std::string(value, length) : std::string();
|
|
}
|
|
|
|
static bool profile_enabled()
|
|
{
|
|
static const bool enabled = env_uint("D4R_PROFILE", 0) != 0;
|
|
return enabled;
|
|
}
|
|
|
|
using ProfileClock = std::chrono::steady_clock;
|
|
static double profile_ms(ProfileClock::time_point start, ProfileClock::time_point end)
|
|
{
|
|
return std::chrono::duration<double, std::milli>(end - start).count();
|
|
}
|
|
|
|
static double profile_thread_cpu_ms()
|
|
{
|
|
FILETIME created = {}, exited = {}, kernel = {}, user = {};
|
|
if (!GetThreadTimes(GetCurrentThread(), &created, &exited, &kernel, &user))
|
|
return -1.0;
|
|
const uint64_t kernelTicks = (static_cast<uint64_t>(kernel.dwHighDateTime) << 32) | kernel.dwLowDateTime;
|
|
const uint64_t userTicks = (static_cast<uint64_t>(user.dwHighDateTime) << 32) | user.dwLowDateTime;
|
|
return static_cast<double>(kernelTicks + userTicks) / 10000.0;
|
|
}
|
|
|
|
// --- CUDA worker thread --------------------------------------------------------
|
|
// Every NGX-core and CUDA call runs on this thread so the ZLUDA context that
|
|
// the nvcuda bridge creates on first use stays current.
|
|
|
|
class Worker
|
|
{
|
|
public:
|
|
void start()
|
|
{
|
|
// Detached: the process may exit without shutting NGX down.
|
|
std::thread([this] { run(); }).detach();
|
|
}
|
|
|
|
void post(std::function<void()> job)
|
|
{
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
jobs_.push_back(std::move(job));
|
|
}
|
|
condition_.notify_one();
|
|
}
|
|
|
|
size_t pending()
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
return jobs_.size();
|
|
}
|
|
|
|
template <typename Function> auto call(Function function) -> decltype(function())
|
|
{
|
|
using Result = decltype(function());
|
|
auto task = std::make_shared<std::packaged_task<Result()>>(std::move(function));
|
|
auto future = task->get_future();
|
|
post([task] { (*task)(); });
|
|
return future.get();
|
|
}
|
|
|
|
private:
|
|
void run()
|
|
{
|
|
for (;;)
|
|
{
|
|
std::function<void()> job;
|
|
{
|
|
std::unique_lock<std::mutex> lock(mutex_);
|
|
condition_.wait(lock, [this] { return !jobs_.empty(); });
|
|
job = std::move(jobs_.front());
|
|
jobs_.pop_front();
|
|
}
|
|
job();
|
|
}
|
|
}
|
|
|
|
std::mutex mutex_;
|
|
std::condition_variable condition_;
|
|
std::deque<std::function<void()>> jobs_;
|
|
};
|
|
|
|
// --- half/float and packed format conversions --------------------------------
|
|
|
|
static uint16_t float_to_half(float value)
|
|
{
|
|
uint32_t bits;
|
|
std::memcpy(&bits, &value, sizeof(bits));
|
|
const uint32_t sign = (bits >> 16) & 0x8000u;
|
|
const uint32_t exponent = (bits >> 23) & 0xffu;
|
|
uint32_t mantissa = bits & 0x7fffffu;
|
|
if (exponent == 0xff)
|
|
return static_cast<uint16_t>(sign | 0x7c00u | (mantissa ? 0x200u : 0u));
|
|
int halfExponent = static_cast<int>(exponent) - 112;
|
|
if (halfExponent >= 31)
|
|
return static_cast<uint16_t>(sign | 0x7c00u);
|
|
uint32_t shift = 13;
|
|
if (halfExponent <= 0)
|
|
{
|
|
if (halfExponent < -10)
|
|
return static_cast<uint16_t>(sign);
|
|
mantissa |= 0x800000u;
|
|
shift = static_cast<uint32_t>(14 - halfExponent);
|
|
halfExponent = 0;
|
|
}
|
|
const uint32_t halfMantissa = mantissa >> shift;
|
|
const uint32_t remainder = mantissa & ((1u << shift) - 1u);
|
|
const uint32_t halfway = 1u << (shift - 1u);
|
|
uint32_t result = sign | (static_cast<uint32_t>(halfExponent) << 10);
|
|
result += halfMantissa & (halfExponent == 0 ? 0x7ffu : 0x3ffu);
|
|
if (remainder > halfway || (remainder == halfway && (halfMantissa & 1u)))
|
|
++result;
|
|
return static_cast<uint16_t>(result);
|
|
}
|
|
|
|
static float half_to_float(uint16_t bits)
|
|
{
|
|
const uint32_t sign = static_cast<uint32_t>(bits & 0x8000u) << 16;
|
|
const uint32_t exponent = (bits >> 10) & 0x1fu;
|
|
const uint32_t mantissa = bits & 0x3ffu;
|
|
uint32_t result;
|
|
if (exponent == 0)
|
|
{
|
|
float magnitude = static_cast<float>(mantissa) * (1.0f / 16777216.0f);
|
|
std::memcpy(&result, &magnitude, sizeof(result));
|
|
result |= sign;
|
|
}
|
|
else if (exponent == 0x1f)
|
|
result = sign | 0x7f800000u | (mantissa << 13);
|
|
else
|
|
result = sign | ((exponent + 112u) << 23) | (mantissa << 13);
|
|
float value;
|
|
std::memcpy(&value, &result, sizeof(value));
|
|
return value;
|
|
}
|
|
|
|
// Unsigned small floats (R11G11B10): 5-bit exponent, 6 or 5 bit mantissa.
|
|
static float small_float_to_float(uint32_t bits, int mantissaBits)
|
|
{
|
|
const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
|
|
const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
|
|
if (exponent == 0)
|
|
return std::ldexp(static_cast<float>(mantissa), -14 - mantissaBits);
|
|
if (exponent == 0x1f)
|
|
return mantissa ? NAN : INFINITY;
|
|
return std::ldexp(1.0f + static_cast<float>(mantissa) / static_cast<float>(1u << mantissaBits),
|
|
static_cast<int>(exponent) - 15);
|
|
}
|
|
|
|
static uint32_t float_to_small_float(float value, int mantissaBits)
|
|
{
|
|
if (!(value > 0.0f))
|
|
return 0; // negatives and NaN clamp to zero
|
|
const float maximum = std::ldexp(2.0f - std::ldexp(1.0f, -mantissaBits), 15);
|
|
if (value >= maximum)
|
|
value = maximum;
|
|
int exponent;
|
|
const float fraction = std::frexp(value, &exponent); // value = fraction * 2^exponent
|
|
int biased = exponent - 1 + 15;
|
|
if (biased <= 0)
|
|
{
|
|
const uint32_t mantissa = static_cast<uint32_t>(std::lround(std::ldexp(value, 14 + mantissaBits)));
|
|
return mantissa;
|
|
}
|
|
uint32_t mantissa = static_cast<uint32_t>(std::lround((fraction * 2.0f - 1.0f) * static_cast<float>(1u << mantissaBits)));
|
|
if (mantissa >= (1u << mantissaBits))
|
|
{
|
|
mantissa = 0;
|
|
++biased;
|
|
}
|
|
if (biased >= 31)
|
|
return (30u << mantissaBits) | ((1u << mantissaBits) - 1u);
|
|
return (static_cast<uint32_t>(biased) << mantissaBits) | mantissa;
|
|
}
|
|
|
|
// Canonical CUDA-side layouts: color RGBA16F, depth R32F, motion RG16F,
|
|
// exposure R32F, output RGBA16F.
|
|
enum class Plane
|
|
{
|
|
Color,
|
|
Depth,
|
|
Motion,
|
|
Exposure
|
|
};
|
|
|
|
static bool supported_input(Plane plane, DXGI_FORMAT format)
|
|
{
|
|
switch (plane)
|
|
{
|
|
case Plane::Color:
|
|
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
|
|
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
|
|
format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
|
|
format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB ||
|
|
format == DXGI_FORMAT_R8G8B8A8_TYPELESS || format == DXGI_FORMAT_B8G8R8A8_UNORM ||
|
|
format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB || format == DXGI_FORMAT_B8G8R8A8_TYPELESS ||
|
|
format == DXGI_FORMAT_R9G9B9E5_SHAREDEXP;
|
|
case Plane::Depth:
|
|
return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
|
|
format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
|
|
format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS ||
|
|
format == DXGI_FORMAT_D24_UNORM_S8_UINT || format == DXGI_FORMAT_R24G8_TYPELESS ||
|
|
format == DXGI_FORMAT_R24_UNORM_X8_TYPELESS || format == DXGI_FORMAT_D16_UNORM ||
|
|
format == DXGI_FORMAT_R16_UNORM || format == DXGI_FORMAT_R16_TYPELESS ||
|
|
format == DXGI_FORMAT_R16_FLOAT;
|
|
case Plane::Motion:
|
|
return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS ||
|
|
format == DXGI_FORMAT_R32G32_FLOAT || format == DXGI_FORMAT_R32G32_TYPELESS ||
|
|
format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
|
|
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R8G8B8A8_UNORM;
|
|
case Plane::Exposure:
|
|
return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS ||
|
|
format == DXGI_FORMAT_R16_FLOAT || format == DXGI_FORMAT_R16_TYPELESS ||
|
|
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R16G16B16A16_FLOAT;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static bool supported_output(DXGI_FORMAT format)
|
|
{
|
|
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
|
|
format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
|
|
format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
|
|
format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_TYPELESS ||
|
|
format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_TYPELESS ||
|
|
format == DXGI_FORMAT_R9G9B9E5_SHAREDEXP;
|
|
}
|
|
|
|
static float unorm(uint32_t value, int bits)
|
|
{
|
|
return static_cast<float>(value) / static_cast<float>((1u << bits) - 1u);
|
|
}
|
|
|
|
static uint32_t to_unorm(float value, int bits)
|
|
{
|
|
const float clamped = value < 0.0f ? 0.0f : (value > 1.0f ? 1.0f : value);
|
|
return static_cast<uint32_t>(std::lround(clamped * static_cast<float>((1u << bits) - 1u)));
|
|
}
|
|
|
|
// Converts one row of a D3D12 texel format into the canonical layout.
|
|
static void convert_row_in(Plane plane, DXGI_FORMAT format, const uint8_t* source, void* destination, UINT width)
|
|
{
|
|
uint16_t* half = static_cast<uint16_t*>(destination);
|
|
float* single = static_cast<float*>(destination);
|
|
for (UINT x = 0; x < width; ++x)
|
|
{
|
|
switch (plane)
|
|
{
|
|
case Plane::Color:
|
|
{
|
|
float rgba[4] = {0.0f, 0.0f, 0.0f, 1.0f};
|
|
switch (format)
|
|
{
|
|
case DXGI_FORMAT_R16G16B16A16_FLOAT:
|
|
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
|
|
std::memcpy(half + x * 4, source + x * 8, 8);
|
|
continue;
|
|
case DXGI_FORMAT_R32G32B32A32_FLOAT:
|
|
std::memcpy(rgba, source + x * 16, 16);
|
|
break;
|
|
case DXGI_FORMAT_R11G11B10_FLOAT:
|
|
{
|
|
uint32_t packed;
|
|
std::memcpy(&packed, source + x * 4, 4);
|
|
rgba[0] = small_float_to_float(packed & 0x7ffu, 6);
|
|
rgba[1] = small_float_to_float((packed >> 11) & 0x7ffu, 6);
|
|
rgba[2] = small_float_to_float((packed >> 22) & 0x3ffu, 5);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_R10G10B10A2_UNORM:
|
|
case DXGI_FORMAT_R10G10B10A2_TYPELESS:
|
|
{
|
|
uint32_t packed;
|
|
std::memcpy(&packed, source + x * 4, 4);
|
|
rgba[0] = unorm(packed & 0x3ffu, 10);
|
|
rgba[1] = unorm((packed >> 10) & 0x3ffu, 10);
|
|
rgba[2] = unorm((packed >> 20) & 0x3ffu, 10);
|
|
rgba[3] = unorm(packed >> 30, 2);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_B8G8R8A8_UNORM:
|
|
case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB:
|
|
case DXGI_FORMAT_B8G8R8A8_TYPELESS:
|
|
rgba[0] = unorm(source[x * 4 + 2], 8);
|
|
rgba[1] = unorm(source[x * 4 + 1], 8);
|
|
rgba[2] = unorm(source[x * 4 + 0], 8);
|
|
rgba[3] = unorm(source[x * 4 + 3], 8);
|
|
break;
|
|
case DXGI_FORMAT_R9G9B9E5_SHAREDEXP:
|
|
{
|
|
// Microsoft spec: c = (p >> offset) & 0x1FF, E = (p >> 27) & 0x1F,
|
|
// v = c * 2^(E - 24) for every E: there is no implicit leading bit.
|
|
uint32_t packed;
|
|
std::memcpy(&packed, source + x * 4, 4);
|
|
uint32_t e = (packed >> 27) & 0x1Fu;
|
|
float scale = std::ldexp(1.0f, (int)e - 24);
|
|
rgba[0] = static_cast<float>((packed >> 0) & 0x1FFu) * scale;
|
|
rgba[1] = static_cast<float>((packed >> 9) & 0x1FFu) * scale;
|
|
rgba[2] = static_cast<float>((packed >> 18) & 0x1FFu) * scale;
|
|
rgba[3] = 1.0f;
|
|
break;
|
|
}
|
|
default: // RGBA8 variants
|
|
for (int c = 0; c < 4; ++c)
|
|
rgba[c] = unorm(source[x * 4 + c], 8);
|
|
break;
|
|
}
|
|
for (int c = 0; c < 4; ++c)
|
|
half[x * 4 + c] = float_to_half(rgba[c]);
|
|
break;
|
|
}
|
|
case Plane::Depth:
|
|
{
|
|
switch (format)
|
|
{
|
|
case DXGI_FORMAT_D24_UNORM_S8_UINT:
|
|
case DXGI_FORMAT_R24G8_TYPELESS:
|
|
case DXGI_FORMAT_R24_UNORM_X8_TYPELESS:
|
|
{
|
|
uint32_t packed;
|
|
std::memcpy(&packed, source + x * 4, 4);
|
|
single[x] = unorm(packed & 0xffffffu, 24);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_D16_UNORM:
|
|
case DXGI_FORMAT_R16_UNORM:
|
|
case DXGI_FORMAT_R16_TYPELESS:
|
|
{
|
|
uint16_t value;
|
|
std::memcpy(&value, source + x * 2, 2);
|
|
single[x] = unorm(value, 16);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_R16_FLOAT:
|
|
{
|
|
uint16_t value;
|
|
std::memcpy(&value, source + x * 2, 2);
|
|
single[x] = half_to_float(value);
|
|
break;
|
|
}
|
|
default: // 32-bit float depth (the depth plane of D32S8 copies as 4 bytes)
|
|
std::memcpy(single + x, source + x * 4, 4);
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
case Plane::Motion:
|
|
switch (format)
|
|
{
|
|
case DXGI_FORMAT_R16G16_FLOAT:
|
|
case DXGI_FORMAT_R16G16_TYPELESS:
|
|
std::memcpy(half + x * 2, source + x * 4, 4);
|
|
break;
|
|
case DXGI_FORMAT_R16G16B16A16_FLOAT:
|
|
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
|
|
std::memcpy(half + x * 2, source + x * 8, 4);
|
|
break;
|
|
case DXGI_FORMAT_R32G32B32A32_FLOAT:
|
|
case DXGI_FORMAT_R32G32_FLOAT:
|
|
case DXGI_FORMAT_R32G32_TYPELESS:
|
|
{
|
|
const size_t stride = format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 8;
|
|
float mv[2];
|
|
std::memcpy(mv, source + x * stride, 8);
|
|
half[x * 2] = float_to_half(mv[0]);
|
|
half[x * 2 + 1] = float_to_half(mv[1]);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_R8G8B8A8_UNORM:
|
|
{
|
|
const uint8_t* rgba = source + x * 4;
|
|
half[x * 2] = float_to_half(static_cast<float>(rgba[0]) / 255.0f);
|
|
half[x * 2 + 1] = float_to_half(static_cast<float>(rgba[1]) / 255.0f);
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
break;
|
|
case Plane::Exposure:
|
|
switch (format)
|
|
{
|
|
case DXGI_FORMAT_R16_FLOAT:
|
|
case DXGI_FORMAT_R16_TYPELESS:
|
|
case DXGI_FORMAT_R16G16B16A16_FLOAT:
|
|
{
|
|
uint16_t value;
|
|
std::memcpy(&value, source + x * (format == DXGI_FORMAT_R16G16B16A16_FLOAT ? 8 : 2), 2);
|
|
single[x] = half_to_float(value);
|
|
break;
|
|
}
|
|
default:
|
|
std::memcpy(single + x, source + x * (format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 4), 4);
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Formats whose texels are bit-identical to the canonical CUDA layout (the
|
|
// cases convert_row_in handles as a plain copy), so the mapped staging rows can
|
|
// be copied straight into the CUDA array with their D3D12 row pitch.
|
|
static bool canonical_input(Plane plane, DXGI_FORMAT format)
|
|
{
|
|
switch (plane)
|
|
{
|
|
case Plane::Color:
|
|
return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS;
|
|
case Plane::Depth:
|
|
return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
|
|
format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
|
|
format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS;
|
|
case Plane::Motion:
|
|
return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS;
|
|
case Plane::Exposure:
|
|
return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// R11G11B10 floats share binary16's 5-bit exponent and bias, so each channel
|
|
// widens to half exactly by shifting its mantissa; NaN becomes the quiet NaN
|
|
// float_to_half produces. Matches convert_row_in bit for bit, without libm.
|
|
static uint16_t small_float_to_half(uint32_t bits, int mantissaBits)
|
|
{
|
|
const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
|
|
const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
|
|
if (exponent == 0x1f && mantissa != 0)
|
|
return 0x7e00u;
|
|
return static_cast<uint16_t>((exponent << 10) | (mantissa << (10 - mantissaBits)));
|
|
}
|
|
|
|
static void convert_r11g11b10_row(const uint8_t* source, uint16_t* half, UINT width)
|
|
{
|
|
for (UINT x = 0; x < width; ++x)
|
|
{
|
|
uint32_t packed;
|
|
std::memcpy(&packed, source + x * 4, 4);
|
|
half[x * 4 + 0] = small_float_to_half(packed & 0x7ffu, 6);
|
|
half[x * 4 + 1] = small_float_to_half((packed >> 11) & 0x7ffu, 6);
|
|
half[x * 4 + 2] = small_float_to_half((packed >> 22) & 0x3ffu, 5);
|
|
half[x * 4 + 3] = 0x3c00u; // 1.0
|
|
}
|
|
}
|
|
|
|
// Packs one RGBA16F DLSS texel into the 9-9-9-5 shared-exponent layout.
|
|
// Microsoft spec: v = c * 2^(E - 24), c in [0, 511]. The exponent is chosen as
|
|
// the smallest E such that all channels fit in 9 bits, then each channel is
|
|
// rounded to the nearest representable value (round-half-even, matching
|
|
// D3D's conversion semantics). Values above the format's max (65408.0) are
|
|
// clamped, like D3D12 does.
|
|
static void rgbe_pack(const float rgba[4], uint8_t* destination)
|
|
{
|
|
const float MAX_VALUE = 65408.0f; // 511 * 2^(31 - 24)
|
|
float r = rgba[0] < 0.0f ? 0.0f : (rgba[0] > MAX_VALUE ? MAX_VALUE : rgba[0]);
|
|
float g = rgba[1] < 0.0f ? 0.0f : (rgba[1] > MAX_VALUE ? MAX_VALUE : rgba[1]);
|
|
float b = rgba[2] < 0.0f ? 0.0f : (rgba[2] > MAX_VALUE ? MAX_VALUE : rgba[2]);
|
|
|
|
float mx = fmaxf(fmaxf(r, g), b);
|
|
// Smallest scale (i.e. smallest e) such that mx / scale rounds to at most 511. This puts
|
|
// the largest channel in the 9-bit range [256, 511] (lower only at e == 0,
|
|
// which also encodes zero), the maximum precision the format allows.
|
|
int e = 31;
|
|
for (int et = 0; et <= 31; ++et)
|
|
{
|
|
if (mx / std::ldexp(1.0f, et - 24) < 511.5f) { e = et; break; }
|
|
}
|
|
const float scale = std::ldexp(1.0f, e - 24);
|
|
const float inv = 1.0f / scale;
|
|
|
|
// Round to nearest, ties to even (D3D round-to-nearest-even semantics).
|
|
auto to_c = [](float v, float inv) -> uint32_t
|
|
{
|
|
double d = std::nearbyint(v * inv); // round-half-even
|
|
if (d < 0.0) d = 0.0;
|
|
if (d > 511.0) d = 511.0;
|
|
return static_cast<uint32_t>(d);
|
|
};
|
|
|
|
uint32_t packed = 0;
|
|
packed |= to_c(r, inv) << 0;
|
|
packed |= to_c(g, inv) << 9;
|
|
packed |= to_c(b, inv) << 18;
|
|
packed |= (e & 0x1Fu) << 27;
|
|
std::memcpy(destination, &packed, 4);
|
|
}
|
|
|
|
// Converts one row of RGBA16F DLSS output into the game's output format.
|
|
static void convert_row_out(DXGI_FORMAT format, const uint16_t* source, uint8_t* destination, UINT width)
|
|
{
|
|
for (UINT x = 0; x < width; ++x)
|
|
{
|
|
const uint16_t* texel = source + x * 4;
|
|
switch (format)
|
|
{
|
|
case DXGI_FORMAT_R16G16B16A16_FLOAT:
|
|
case DXGI_FORMAT_R16G16B16A16_TYPELESS:
|
|
std::memcpy(destination + x * 8, texel, 8);
|
|
break;
|
|
case DXGI_FORMAT_R32G32B32A32_FLOAT:
|
|
for (int c = 0; c < 4; ++c)
|
|
{
|
|
const float value = half_to_float(texel[c]);
|
|
std::memcpy(destination + x * 16 + c * 4, &value, 4);
|
|
}
|
|
break;
|
|
case DXGI_FORMAT_R11G11B10_FLOAT:
|
|
{
|
|
const uint32_t packed = float_to_small_float(half_to_float(texel[0]), 6) |
|
|
(float_to_small_float(half_to_float(texel[1]), 6) << 11) |
|
|
(float_to_small_float(half_to_float(texel[2]), 5) << 22);
|
|
std::memcpy(destination + x * 4, &packed, 4);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_R10G10B10A2_UNORM:
|
|
case DXGI_FORMAT_R10G10B10A2_TYPELESS:
|
|
{
|
|
const uint32_t packed = to_unorm(half_to_float(texel[0]), 10) |
|
|
(to_unorm(half_to_float(texel[1]), 10) << 10) |
|
|
(to_unorm(half_to_float(texel[2]), 10) << 20) |
|
|
(to_unorm(half_to_float(texel[3]), 2) << 30);
|
|
std::memcpy(destination + x * 4, &packed, 4);
|
|
break;
|
|
}
|
|
case DXGI_FORMAT_B8G8R8A8_UNORM:
|
|
case DXGI_FORMAT_B8G8R8A8_TYPELESS:
|
|
destination[x * 4 + 0] = static_cast<uint8_t>(to_unorm(half_to_float(texel[2]), 8));
|
|
destination[x * 4 + 1] = static_cast<uint8_t>(to_unorm(half_to_float(texel[1]), 8));
|
|
destination[x * 4 + 2] = static_cast<uint8_t>(to_unorm(half_to_float(texel[0]), 8));
|
|
destination[x * 4 + 3] = static_cast<uint8_t>(to_unorm(half_to_float(texel[3]), 8));
|
|
break;
|
|
case DXGI_FORMAT_R9G9B9E5_SHAREDEXP:
|
|
{
|
|
float rgba[4] = {half_to_float(texel[0]), half_to_float(texel[1]), half_to_float(texel[2]), 1.0f};
|
|
rgbe_pack(rgba, destination + x * 4);
|
|
break;
|
|
}
|
|
default: // RGBA8
|
|
for (int c = 0; c < 4; ++c)
|
|
destination[x * 4 + c] = static_cast<uint8_t>(to_unorm(half_to_float(texel[c]), 8));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- global state ----------------------------------------------------------------
|
|
|
|
struct Global
|
|
{
|
|
std::mutex mutex;
|
|
bool started = false;
|
|
Worker worker; // every NGX-core and CUDA call
|
|
std::thread::id workerThread;
|
|
Worker prep, finish; // CPU-only pipeline stages around it
|
|
Worker cleanup; // deferred GPU resource destruction, never under allocator locks
|
|
std::atomic<unsigned int> resourceOwners{0};
|
|
bool shutdownPending = false; // under mutex
|
|
HMODULE core = nullptr;
|
|
CoreApi ngx;
|
|
HMODULE cuda = nullptr;
|
|
CudaApi cu;
|
|
bool ngxInitialized = false;
|
|
ID3D12Device* device = nullptr;
|
|
std::vector<std::wstring> paths;
|
|
std::vector<const wchar_t*> pathPointers;
|
|
NgxFeatureCommonInfo featureInfo = {};
|
|
std::wstring dataPath;
|
|
std::atomic<unsigned int> nextHandleId{0x7200};
|
|
};
|
|
|
|
static Global g;
|
|
static std::mutex g_featureApiMutex;
|
|
|
|
template <typename Function> static bool load_export(HMODULE module, const char* name, Function& function)
|
|
{
|
|
function = reinterpret_cast<Function>(reinterpret_cast<void*>(GetProcAddress(module, name)));
|
|
if (function == nullptr)
|
|
logf("missing export %s", name);
|
|
return function != nullptr;
|
|
}
|
|
|
|
// OptiScaler's DLSS-input hook answers kernel32 LoadLibrary* calls for any
|
|
// *nvngx*.dll outside the game directory with OptiScaler itself, which would
|
|
// hide the official core's CUDA exports. ntdll's LdrLoadDll sits below those
|
|
// hooks (OptiScaler's own proxies load libraries the same way).
|
|
static HMODULE load_library_below_hooks(const wchar_t* path)
|
|
{
|
|
struct UnicodeString
|
|
{
|
|
USHORT Length;
|
|
USHORT MaximumLength;
|
|
PWSTR Buffer;
|
|
};
|
|
using LdrLoadDllFn = LONG(NTAPI*)(PWSTR, PULONG, UnicodeString*, PHANDLE);
|
|
auto ldrLoadDll = reinterpret_cast<LdrLoadDllFn>(
|
|
reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "LdrLoadDll")));
|
|
if (ldrLoadDll == nullptr)
|
|
return LoadLibraryW(path);
|
|
std::wstring copy(path);
|
|
UnicodeString name;
|
|
name.Length = static_cast<USHORT>(copy.size() * sizeof(wchar_t));
|
|
name.MaximumLength = static_cast<USHORT>(name.Length + sizeof(wchar_t));
|
|
name.Buffer = copy.data();
|
|
HANDLE module = nullptr;
|
|
const LONG status = ldrLoadDll(nullptr, nullptr, &name, &module);
|
|
if (status < 0)
|
|
{
|
|
logf("LdrLoadDll(%ls) failed: NTSTATUS 0x%08lx", path, static_cast<unsigned long>(status));
|
|
return nullptr;
|
|
}
|
|
return static_cast<HMODULE>(module);
|
|
}
|
|
|
|
// With DLSS inputs enabled, OptiScaler also hooks ntdll-level loads and
|
|
// answers every *nvngx*.dll request with itself unless the requested path
|
|
// contains the game EXE's directory in lowercase (its test for "the game's own
|
|
// NGX core"). The official core must keep its _nvngx.dll name, so request it
|
|
// as <lowercase exe dir>\..\..\<core path>: the hook sees the game
|
|
// directory, and Wine collapses the ".." segments to the staged core without
|
|
// anything being written into the game folder.
|
|
static std::wstring core_path_inside_exe_dir(const wchar_t* corePath)
|
|
{
|
|
std::wstring core(corePath);
|
|
wchar_t exe[MAX_PATH] = {};
|
|
const DWORD length = GetModuleFileNameW(nullptr, exe, MAX_PATH);
|
|
if (length == 0 || length >= MAX_PATH || core.size() < 3 || core[1] != L':' || exe[1] != L':' ||
|
|
towlower(core[0]) != towlower(exe[0]))
|
|
return core;
|
|
std::wstring directory(exe);
|
|
directory.resize(directory.find_last_of(L"\\/"));
|
|
for (wchar_t& c : directory)
|
|
c = static_cast<wchar_t>(towlower(c));
|
|
std::wstring path = directory;
|
|
size_t depth = 0;
|
|
for (size_t index = 3; index <= directory.size(); ++index)
|
|
if (index == directory.size() || directory[index] == L'\\' || directory[index] == L'/')
|
|
++depth;
|
|
for (size_t level = 0; level < depth; ++level)
|
|
path += L"\\..";
|
|
path += L"\\";
|
|
path += core.substr(3);
|
|
logf("requesting NGX core as %ls", path.c_str());
|
|
return path;
|
|
}
|
|
|
|
static bool load_libraries()
|
|
{
|
|
DWORD length = 0;
|
|
// Under Proton, nvcuda.dll resolves to Proton's own builtin, which skips the
|
|
// bridge's fixups. Loading the bridge by path first makes every later
|
|
// LoadLibrary("nvcuda.dll"), including the NGX core's, reuse it.
|
|
wchar_t bridgePath[MAX_PATH] = {};
|
|
length = GetEnvironmentVariableW(L"D4R_NVCUDA_BRIDGE", bridgePath, MAX_PATH);
|
|
if (length > 0 && length < MAX_PATH)
|
|
{
|
|
HMODULE bridge = LoadLibraryW(bridgePath);
|
|
logf("preloaded nvcuda bridge %ls: %p (error %lu)", bridgePath, static_cast<void*>(bridge),
|
|
bridge != nullptr ? 0ul : GetLastError());
|
|
// d4r.ini's Linux-side settings, before the bridge's first CUDA call loads ZLUDA
|
|
if (bridge != nullptr && g_portable.active)
|
|
{
|
|
using SetEnvFn = int(WINAPI*)(const char*, const char*, int);
|
|
const auto setEnv = reinterpret_cast<SetEnvFn>(reinterpret_cast<void*>(GetProcAddress(bridge, "d4rSetEnv")));
|
|
if (setEnv == nullptr)
|
|
logf("the nvcuda bridge has no d4rSetEnv; d4r.ini's ZLUDA settings are not applied");
|
|
for (const auto& [name, value] : g_portable.unixEnv)
|
|
if (setEnv != nullptr && setEnv(name.c_str(), value.c_str(), 0))
|
|
logf(" %s=%s%s", name.c_str(), value.c_str(),
|
|
getenv(name.c_str()) != nullptr && value != getenv(name.c_str()) ? " (set by the environment)" : "");
|
|
}
|
|
}
|
|
wchar_t corePath[MAX_PATH] = {};
|
|
length = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
|
|
if (length == 0 || length >= MAX_PATH)
|
|
{
|
|
logf("D4R_NGX_CORE is not set; cannot locate the official NGX core");
|
|
return false;
|
|
}
|
|
g.core = load_library_below_hooks(core_path_inside_exe_dir(corePath).c_str());
|
|
if (g.core == nullptr)
|
|
{
|
|
logf("loading D4R_NGX_CORE=%ls failed: %lu", corePath, GetLastError());
|
|
return false;
|
|
}
|
|
bool ok = true;
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_Init", g.ngx.init);
|
|
if (!load_export(g.core, "NVSDK_NGX_CUDA_Init_ProjectID", g.ngx.initProjectId))
|
|
g.ngx.initProjectId = nullptr;
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_Shutdown", g.ngx.shutdown);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetParameters", g.ngx.getParameters);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_AllocateParameters", g.ngx.allocateParameters);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetCapabilityParameters", g.ngx.getCapabilityParameters);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_DestroyParameters", g.ngx.destroyParameters);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_CreateFeature", g.ngx.createFeature);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_ReleaseFeature", g.ngx.releaseFeature);
|
|
ok &= load_export(g.core, "NVSDK_NGX_CUDA_EvaluateFeature", g.ngx.evaluateFeature);
|
|
|
|
g.cuda = LoadLibraryA("nvcuda.dll");
|
|
if (g.cuda == nullptr)
|
|
{
|
|
logf("LoadLibraryA(nvcuda.dll) failed: %lu", GetLastError());
|
|
return false;
|
|
}
|
|
ok &= load_export(g.cuda, "cuMemAlloc", g.cu.memAlloc);
|
|
ok &= load_export(g.cuda, "cuMemFree", g.cu.memFree);
|
|
ok &= load_export(g.cuda, "cuArrayCreate", g.cu.arrayCreate);
|
|
ok &= load_export(g.cuda, "cuArray3DCreate", g.cu.array3DCreate);
|
|
ok &= load_export(g.cuda, "cuArrayDestroy", g.cu.arrayDestroy);
|
|
ok &= load_export(g.cuda, "cuMemcpy2D", g.cu.memcpy2D);
|
|
ok &= load_export(g.cuda, "cuTexObjectCreate", g.cu.texObjectCreate);
|
|
ok &= load_export(g.cuda, "cuTexObjectDestroy", g.cu.texObjectDestroy);
|
|
ok &= load_export(g.cuda, "cuSurfObjectCreate", g.cu.surfObjectCreate);
|
|
ok &= load_export(g.cuda, "cuSurfObjectDestroy", g.cu.surfObjectDestroy);
|
|
ok &= load_export(g.cuda, "cuCtxSynchronize", g.cu.ctxSynchronize);
|
|
// The d4r bridge's own export keeps waiting when the application's syncs are elided
|
|
// (D4R_ELIDE_NGX_SYNC); cuCtxSynchronize above stays the fallback.
|
|
{
|
|
decltype(g.cu.ctxSynchronize) ownSync = nullptr;
|
|
if (load_export(g.cuda, "d4rCtxSynchronize", ownSync))
|
|
g.cu.ctxSynchronize = ownSync;
|
|
}
|
|
if (!load_export(g.cuda, "cuMemAllocHost", g.cu.memAllocHost) ||
|
|
!load_export(g.cuda, "cuMemFreeHost", g.cu.memFreeHost) ||
|
|
!load_export(g.cuda, "cuMemcpyHtoDAsync", g.cu.memcpyHtoDAsync) ||
|
|
!load_export(g.cuda, "cuMemcpyDtoHAsync", g.cu.memcpyDtoHAsync) ||
|
|
!load_export(g.cuda, "cuStreamCreate", g.cu.streamCreate) ||
|
|
!load_export(g.cuda, "cuStreamSynchronize", g.cu.streamSynchronize))
|
|
{
|
|
g.cu.memAllocHost = nullptr;
|
|
logf("overlapped staging unavailable; using synchronous copies");
|
|
}
|
|
if (!load_export(g.cuda, "d4rMemcpy2DAsync", g.cu.memcpy2DAsync))
|
|
g.cu.memcpy2DAsync = nullptr;
|
|
if (!load_export(g.cuda, "d4rStreamWaitValue32", g.cu.streamWaitValue32) ||
|
|
!load_export(g.cuda, "d4rWriteValue32", g.cu.writeValue32))
|
|
g.cu.streamWaitValue32 = nullptr, g.cu.writeValue32 = nullptr;
|
|
if (!load_export(g.cuda, "d4rSetArrayRedirect", g.cu.setArrayRedirect))
|
|
g.cu.setArrayRedirect = nullptr;
|
|
if (!load_export(g.cuda, "d4rOutputKernelNative", g.cu.outputKernelNative))
|
|
g.cu.outputKernelNative = nullptr;
|
|
if (!load_export(g.cuda, "d4rRegisterLinearTexture", g.cu.registerLinearTexture))
|
|
g.cu.registerLinearTexture = nullptr;
|
|
g.cu.dilateMotion = reinterpret_cast<decltype(g.cu.dilateMotion)>(
|
|
reinterpret_cast<void*>(GetProcAddress(g.cuda, "d4rDilateMotionVectors")));
|
|
if (profile_enabled() || env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0)
|
|
{
|
|
const bool create = load_export(g.cuda, "cuEventCreate", g.cu.eventCreate);
|
|
const bool record = load_export(g.cuda, "cuEventRecord", g.cu.eventRecord);
|
|
const bool wait = env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0
|
|
? load_export(g.cuda, "d4rEventSynchronize", g.cu.eventSynchronize) : true;
|
|
const bool query = load_export(g.cuda, "cuEventQuery", g.cu.eventQuery);
|
|
const bool destroy = load_export(g.cuda, "cuEventDestroy", g.cu.eventDestroy);
|
|
const bool elapsed = profile_enabled() ? load_export(g.cuda, "cuEventElapsedTime", g.cu.eventElapsedTime) : true;
|
|
if (profile_enabled() && !(create && record && destroy && elapsed))
|
|
logf("D4R_PROFILE: CUDA event timing unavailable; CPU stage timings remain enabled");
|
|
if (env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0 && !(create && record && wait && query && destroy))
|
|
logf("blocking output wait unavailable; using context sync");
|
|
}
|
|
logf("loaded NGX core %ls and nvcuda.dll: %s", corePath, ok ? "all exports present" : "exports missing");
|
|
return ok;
|
|
}
|
|
|
|
// A game that initialises NGX with its project ID (engine integrations such as Unreal's) is identified
|
|
// to the NGX core the same way.
|
|
struct ProjectIdentity
|
|
{
|
|
std::string id;
|
|
int engineType = 0;
|
|
std::string engineVersion;
|
|
};
|
|
|
|
static NgxResult initialize(unsigned long long applicationId, const wchar_t* dataPath, ID3D12Device* device,
|
|
const NgxFeatureCommonInfo* featureInfo, unsigned int sdkVersion,
|
|
const ProjectIdentity* project = nullptr)
|
|
{
|
|
std::lock_guard<std::mutex> lock(g.mutex);
|
|
g.shutdownPending = false;
|
|
if (g.ngxInitialized)
|
|
return NGX_SUCCESS;
|
|
if (!g.started)
|
|
{
|
|
// Allocators can retain COM callbacks and detached workers past NGX
|
|
// shutdown. Keep their code loaded until process exit.
|
|
HMODULE pinned = nullptr;
|
|
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_PIN,
|
|
reinterpret_cast<LPCWSTR>(g_selfModule), &pinned))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
g.worker.start();
|
|
g.workerThread = g.worker.call([] { return std::this_thread::get_id(); });
|
|
g.prep.start();
|
|
g.finish.start();
|
|
g.cleanup.start();
|
|
g.started = true;
|
|
}
|
|
ensure_portable_config();
|
|
if (g_portable.active && !g.core)
|
|
{
|
|
logf("portable install in %ls (settings from d4r.ini, Linux path %s)", g_portable.dir.c_str(),
|
|
g_portable.unixDir.c_str());
|
|
for (const std::string& note : g_portable.notes)
|
|
logf("d4r: %s", note.c_str());
|
|
}
|
|
if (g.core == nullptr && !load_libraries())
|
|
{
|
|
g.core = nullptr;
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
if (device != nullptr)
|
|
{
|
|
device->AddRef();
|
|
g.device = device;
|
|
// NGX only accepts a CUDA device whose LUID matches the D3D12 adapter;
|
|
// the nvcuda bridge reports this LUID from cuDeviceGetLuid.
|
|
LUID luid;
|
|
device->GetAdapterLuid(&luid);
|
|
char low[24], high[24];
|
|
std::snprintf(low, sizeof(low), "0x%08lx", static_cast<unsigned long>(luid.LowPart));
|
|
std::snprintf(high, sizeof(high), "0x%08lx", static_cast<unsigned long>(luid.HighPart));
|
|
SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", low);
|
|
SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", high);
|
|
SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", "1");
|
|
logf("CUDA device LUID set to the D3D12 adapter LUID %s:%s", high, low);
|
|
}
|
|
|
|
g.paths.clear();
|
|
wchar_t featureDirectory[MAX_PATH] = {};
|
|
DWORD length = GetEnvironmentVariableW(L"D4R_NGX_FEATURE_DIR", featureDirectory, MAX_PATH);
|
|
if (length > 0 && length < MAX_PATH)
|
|
g.paths.emplace_back(featureDirectory);
|
|
if (featureInfo != nullptr && featureInfo->PathListInfo.Path != nullptr)
|
|
for (unsigned int index = 0; index < featureInfo->PathListInfo.Length; ++index)
|
|
if (featureInfo->PathListInfo.Path[index] != nullptr)
|
|
g.paths.emplace_back(featureInfo->PathListInfo.Path[index]);
|
|
g.pathPointers.clear();
|
|
for (const std::wstring& path : g.paths)
|
|
{
|
|
g.pathPointers.push_back(path.c_str());
|
|
logf("feature search path: %ls", path.c_str());
|
|
}
|
|
g.featureInfo = {};
|
|
g.featureInfo.PathListInfo.Path = g.pathPointers.data();
|
|
g.featureInfo.PathListInfo.Length = static_cast<unsigned int>(g.pathPointers.size());
|
|
g.featureInfo.LoggingInfo.LoggingCallback = reinterpret_cast<void*>(&ngx_log_callback);
|
|
g.featureInfo.LoggingInfo.MinimumLoggingLevel = static_cast<int>(env_uint("D4R_NGX_LOG_LEVEL", 1)); // 1=on, 2=verbose
|
|
g.featureInfo.LoggingInfo.DisableOtherLoggingSinks = false;
|
|
g.dataPath = dataPath != nullptr ? dataPath : L".";
|
|
|
|
// The CUDA path takes the same application identity as the game's D3D12 init. (The SDK sample's id
|
|
// is only a last resort: NGX stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output for it.)
|
|
const unsigned long long id = applicationId != 0 ? applicationId : 241534723ULL;
|
|
const unsigned int version = sdkVersion != 0 ? sdkVersion : 0x15;
|
|
// Logging info is only read for API versions 0x14 and later.
|
|
const unsigned int apiVersion = version < 0x14 ? 0x14 : version;
|
|
NgxResult result;
|
|
if (project != nullptr && !project->id.empty() && g.ngx.initProjectId != nullptr)
|
|
{
|
|
result = g.worker.call([&] {
|
|
return g.ngx.initProjectId(project->id.c_str(), project->engineType, project->engineVersion.c_str(),
|
|
g.dataPath.c_str(), apiVersion, &g.featureInfo);
|
|
});
|
|
logf("NVSDK_NGX_CUDA_Init_ProjectID(project=%s, engine=%d %s, sdk=0x%x) -> 0x%08x", project->id.c_str(),
|
|
project->engineType, project->engineVersion.c_str(), version, result);
|
|
}
|
|
else
|
|
{
|
|
result = g.worker.call([&] { return g.ngx.init(id, g.dataPath.c_str(), &g.featureInfo, apiVersion); });
|
|
logf("NVSDK_NGX_CUDA_Init(app=%llu, sdk=0x%x) -> 0x%08x", id, version, result);
|
|
}
|
|
using LoadErrorFn = const char*(WINAPI*)();
|
|
const auto loadError = reinterpret_cast<LoadErrorFn>(reinterpret_cast<void*>(GetProcAddress(g.cuda, "d4rLoadError")));
|
|
if (loadError != nullptr && loadError()[0] != '\0')
|
|
logf("nvcuda bridge: %s", loadError());
|
|
g.ngxInitialized = result == NGX_SUCCESS;
|
|
return result;
|
|
}
|
|
|
|
// --- per-feature state ---------------------------------------------------------
|
|
|
|
constexpr int kSlots = 3;
|
|
// One more result slot than input slots: a slot is reused only once the GPU
|
|
// is past every command list that copies from it (see claim_output_slot).
|
|
constexpr int kOutputSlots = kSlots + 1;
|
|
|
|
// A VkBuffer in device-local memory on vkd3d-proton's VkDevice, exported to
|
|
// the CUDA side (see "VRAM interop").
|
|
struct VramBuffer
|
|
{
|
|
VkBuffer buffer = VK_NULL_HANDLE;
|
|
VkDeviceMemory memory = VK_NULL_HANDLE;
|
|
CudaDevicePtr device = 0;
|
|
void* external = nullptr;
|
|
size_t bytes = 0;
|
|
};
|
|
|
|
// A temporary image for colour or exposure format conversion.
|
|
struct VramImage
|
|
{
|
|
VkImage image = VK_NULL_HANDLE;
|
|
VkDeviceMemory memory = VK_NULL_HANDLE;
|
|
UINT width = 0, height = 0;
|
|
VkFormat format = VK_FORMAT_UNDEFINED;
|
|
};
|
|
|
|
struct Staging
|
|
{
|
|
ID3D12Resource* buffer = nullptr;
|
|
uint8_t* mapped = nullptr;
|
|
D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout = {};
|
|
UINT rows = 0;
|
|
UINT64 total = 0;
|
|
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
|
|
UINT width = 0;
|
|
UINT height = 0;
|
|
|
|
std::vector<ID3D12Resource*> retired;
|
|
|
|
void retire()
|
|
{
|
|
if (buffer != nullptr)
|
|
retired.push_back(buffer);
|
|
buffer = nullptr;
|
|
mapped = nullptr;
|
|
width = height = 0;
|
|
format = DXGI_FORMAT_UNKNOWN;
|
|
}
|
|
|
|
void release()
|
|
{
|
|
if (buffer != nullptr)
|
|
buffer->Release();
|
|
for (ID3D12Resource* old : retired)
|
|
old->Release();
|
|
*this = Staging{};
|
|
}
|
|
};
|
|
|
|
struct CudaImage
|
|
{
|
|
CudaArray array = nullptr;
|
|
CudaObject object = 0; // texture or surface object
|
|
UINT width = 0;
|
|
UINT height = 0;
|
|
};
|
|
|
|
struct FrameParams
|
|
{
|
|
float jitterX = 0, jitterY = 0, mvScaleX = 1, mvScaleY = 1, sharpness = 0;
|
|
float preExposure = 1, exposureScale = 1, frameTime = 16.6f;
|
|
int reset = 0, invertX = 0, invertY = 0;
|
|
unsigned int renderWidth = 0, renderHeight = 0;
|
|
unsigned int colorBaseX = 0, colorBaseY = 0, depthBaseX = 0, depthBaseY = 0;
|
|
unsigned int mvBaseX = 0, mvBaseY = 0, outputBaseX = 0, outputBaseY = 0;
|
|
bool hasExposure = false;
|
|
bool vram = false; // inputs and output stay in VRAM (VRAM interop)
|
|
bool split = false; // this frame presents its own result (split frames)
|
|
unsigned motionDilation = 0; // actually applied, worker only
|
|
};
|
|
|
|
struct FrameTiming
|
|
{
|
|
bool enabled = false;
|
|
ProfileClock::time_point gameStart, queuedAt;
|
|
double slotWait = 0, inputRecord = 0, outputRecord = 0, gameCall = 0, gameInterval = 0;
|
|
double convert[4] = {}, upload[4] = {};
|
|
double download = 0, outputConvert = 0;
|
|
double prep = 0, worker = 0, workerWait = 0, finishWait = 0, markerWait = 0;
|
|
double ngxHost = 0, ctxSync = 0, gpuEval = -1;
|
|
double motionDilationWall = 0;
|
|
bool gpuEventsRecorded = false;
|
|
double outputSyncWall = 0, outputSyncCpu = -1;
|
|
bool outputSyncBlocking = false;
|
|
double h2dTotal = 0, d2hIssue = 0, d2hWait = 0;
|
|
ProfileClock::time_point prepStart, workerQueued, finishQueued;
|
|
size_t queueDepth = 0;
|
|
uint32_t presentedFrame = 0;
|
|
};
|
|
|
|
// A plane in the canonical CUDA layout, tightly packed, in ordinary host
|
|
// memory: filled from the D3D12 staging by one pipeline stage and consumed by
|
|
// the next (see prepare_inputs / run_evaluation / publish_output).
|
|
struct HostPlane
|
|
{
|
|
uint8_t* bytes = nullptr; // page-locked when overlapped staging is on
|
|
size_t capacity = 0;
|
|
bool pinned = false;
|
|
std::vector<uint8_t> fallback;
|
|
UINT width = 0, height = 0;
|
|
size_t rowBytes = 0;
|
|
size_t size() const { return rowBytes * height; }
|
|
};
|
|
|
|
struct InputSlot
|
|
{
|
|
Staging color, depth, motion, exposure;
|
|
VramBuffer vram[4]; // VRAM interop: the planes in canonical layout
|
|
// D4R_SHIM_LINEAR_INPUTS: pitch-linear texture objects on vram[] handed to NGX (worker only)
|
|
CudaObject linearTexture[4] = {};
|
|
CudaDevicePtr linearPointer[4] = {};
|
|
UINT linearWidth[4] = {}, linearHeight[4] = {};
|
|
size_t linearPitch[4] = {};
|
|
CudaDevicePtr dilatedMotion = 0;
|
|
size_t dilatedBytes = 0;
|
|
CudaObject dilatedTexture = 0;
|
|
UINT dilatedWidth = 0, dilatedHeight = 0;
|
|
size_t dilatedPitch = 0;
|
|
std::atomic<bool> busy{false}; // readback staging owned by the pipeline
|
|
HostPlane host[4];
|
|
std::atomic<bool> hostBusy{false}; // host planes not yet uploaded by the worker
|
|
};
|
|
|
|
struct OutputSlot
|
|
{
|
|
Staging staging;
|
|
VramBuffer vram; // VRAM interop: the result in RGBA16F
|
|
std::atomic<uint32_t> lastReadFrame{0}; // last frame whose command list copies from it
|
|
std::atomic<uint32_t> producedFrame{0};
|
|
};
|
|
|
|
struct Feature
|
|
{
|
|
NgxHandle handle = {}; // returned to the caller
|
|
std::atomic<bool> retiring{false};
|
|
IUnknown* resources = nullptr; // main reference plus command-allocator references
|
|
NgxHandle* cudaHandle = nullptr;
|
|
void* cudaParams = nullptr;
|
|
CudaDevicePtr scratch = 0;
|
|
unsigned int width = 0, height = 0, outWidth = 0, outHeight = 0;
|
|
unsigned int cudaWidth = 0, cudaHeight = 0; // render size the NGX feature was created with (worker only)
|
|
unsigned int refusedWidth = 0, refusedHeight = 0; // render size NGX would not create a feature for (worker only)
|
|
int quality = 0, flags = 0;
|
|
unsigned int preset = 0;
|
|
CudaEvent profileStart = nullptr, profileEnd = nullptr;
|
|
CudaEvent outputReadyEvent = nullptr;
|
|
bool profileEventsReady = false;
|
|
|
|
CudaImage color, depth, motion, exposure, output;
|
|
CudaObject colorHandle = 0, depthHandle = 0, motionHandle = 0, exposureHandle = 0, outputHandle = 0;
|
|
|
|
InputSlot inputs[kSlots];
|
|
OutputSlot outputs[kOutputSlots];
|
|
HostPlane outputHost[2]; // downloaded results waiting for publish_output
|
|
std::atomic<bool> outputHostBusy[2] = {false, false};
|
|
unsigned int nextOutputHost = 0; // worker only
|
|
// Overlapped staging (worker only): linear device buffers the async copies
|
|
// target, moved to/from the CUDA arrays with device-to-device copies.
|
|
CudaDevicePtr planeLinear[4] = {}, outputLinear = 0;
|
|
size_t planeLinearBytes[4] = {}, outputLinearBytes = 0;
|
|
// Frames queued but not yet published (or dropped). evaluate() blocks the
|
|
// game at D4R_SHIM_MAX_IN_FLIGHT (default 3) so a backlog cannot build up
|
|
// between pipeline stages and age the presented result.
|
|
std::atomic<int> inFlight{0};
|
|
DXGI_FORMAT outputFormat = DXGI_FORMAT_UNKNOWN;
|
|
UINT outputWidth = 0, outputHeight = 0;
|
|
// VRAM interop: decided on the first evaluation, dropped for good if the
|
|
// formats later stop qualifying.
|
|
bool vram = false, vramDecided = false;
|
|
VramImage colorConversion;
|
|
VramImage depthConversion;
|
|
VramImage motionConversion;
|
|
VramImage exposureConversion;
|
|
VramImage outputConversion;
|
|
// Split frames: the rest of frame N's command list is submitted only once
|
|
// splitSemaphore reaches N (all admitted frames through N have retired).
|
|
std::atomic<bool> split{false};
|
|
VkSemaphore splitSemaphore = VK_NULL_HANDLE;
|
|
std::mutex splitMutex;
|
|
uint64_t splitSignalled = 0; // under splitMutex
|
|
D4rFrameCompletion splitCompletion; // under splitMutex; cancellation can finish out of order
|
|
// Inputs become ready on the CPU before CUDA work is queued. GPU-side waits
|
|
// for recorded-but-discarded command lists can otherwise wedge the device.
|
|
// D4R_SHIM_OUTPUT_DIRECT: this frame's result is stored by the native output kernel straight into its
|
|
// destination buffer (worker only), so the array -> buffer copy is skipped.
|
|
bool outputRedirected = false;
|
|
bool outputDirectAllowed = false; // the forced preset's output kernel honours the redirect
|
|
// D4R_SHIM_LINEAR_INPUTS: NGX samples the interop buffers directly through pitch-linear
|
|
// texture objects (rows padded to 256 bytes), so the buffer -> array copies disappear.
|
|
bool linearInputs = false;
|
|
bool dilationReported = false;
|
|
std::vector<VramBuffer> retiredBuffers; // may still be read by queued command lists
|
|
std::vector<VramImage> retiredImages;
|
|
ID3D12Resource* marker = nullptr;
|
|
volatile uint32_t* markerValue = nullptr; // [0] latest input; [1 + slot] exact input frame
|
|
uint32_t frame = 0;
|
|
uint32_t rejectedFormatCount = 0; // game thread; rate-limits format diagnostics
|
|
std::atomic<int> latestOutput{-1};
|
|
std::atomic<uint32_t> completedFrames{0};
|
|
uint32_t evaluatedFrames = 0; // worker only
|
|
// Serializes picking the slot to present (game thread) against picking
|
|
// the slot to overwrite (worker).
|
|
std::mutex outputMutex;
|
|
};
|
|
|
|
static std::mutex g_featuresMutex;
|
|
static std::vector<Feature*> g_features;
|
|
|
|
static bool create_buffer(D3D12_HEAP_TYPE heapType, UINT64 size, ID3D12Resource** buffer, uint8_t** mapped)
|
|
{
|
|
D3D12_HEAP_PROPERTIES heap = {};
|
|
heap.Type = heapType;
|
|
D3D12_RESOURCE_DESC desc = {};
|
|
desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
|
|
desc.Width = size;
|
|
desc.Height = 1;
|
|
desc.DepthOrArraySize = 1;
|
|
desc.MipLevels = 1;
|
|
desc.SampleDesc.Count = 1;
|
|
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
|
|
const D3D12_RESOURCE_STATES state =
|
|
heapType == D3D12_HEAP_TYPE_READBACK ? D3D12_RESOURCE_STATE_COPY_DEST : D3D12_RESOURCE_STATE_GENERIC_READ;
|
|
HRESULT hr = g.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, state, nullptr,
|
|
__uuidof(ID3D12Resource), reinterpret_cast<void**>(buffer));
|
|
if (FAILED(hr))
|
|
{
|
|
logf("CreateCommittedResource(heap=%d, %llu bytes) failed: 0x%08lx", heapType,
|
|
static_cast<unsigned long long>(size), hr);
|
|
return false;
|
|
}
|
|
D3D12_RANGE readRange = {0, heapType == D3D12_HEAP_TYPE_READBACK ? static_cast<SIZE_T>(size) : 0};
|
|
hr = (*buffer)->Map(0, &readRange, reinterpret_cast<void**>(mapped));
|
|
if (FAILED(hr))
|
|
{
|
|
logf("Map failed: 0x%08lx", hr);
|
|
(*buffer)->Release();
|
|
*buffer = nullptr;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Ensures staging matches the resource's first subresource; heap selects
|
|
// readback (inputs) or upload (output).
|
|
static bool ensure_staging(Staging& staging, ID3D12Resource* resource, D3D12_HEAP_TYPE heapType)
|
|
{
|
|
D3D12_RESOURCE_DESC desc;
|
|
resource->GetDesc(&desc);
|
|
if (staging.buffer != nullptr && staging.format == desc.Format && staging.width == desc.Width &&
|
|
staging.height == desc.Height)
|
|
return true;
|
|
staging.retire();
|
|
UINT64 rowSize = 0;
|
|
g.device->GetCopyableFootprints(&desc, 0, 1, 0, &staging.layout, &staging.rows, &rowSize, &staging.total);
|
|
if (!create_buffer(heapType, staging.total, &staging.buffer, &staging.mapped))
|
|
return false;
|
|
staging.format = desc.Format;
|
|
staging.width = static_cast<UINT>(desc.Width);
|
|
staging.height = desc.Height;
|
|
logf("staging %s %ux%u format=%d rowPitch=%u total=%llu", heapType == D3D12_HEAP_TYPE_READBACK ? "readback" : "upload",
|
|
staging.width, staging.height, staging.format, staging.layout.Footprint.RowPitch,
|
|
static_cast<unsigned long long>(staging.total));
|
|
return true;
|
|
}
|
|
|
|
static void transition(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES before,
|
|
D3D12_RESOURCE_STATES after)
|
|
{
|
|
if (before == after)
|
|
return;
|
|
D3D12_RESOURCE_BARRIER barrier = {};
|
|
barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
|
|
barrier.Transition.pResource = resource;
|
|
barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
|
|
barrier.Transition.StateBefore = before;
|
|
barrier.Transition.StateAfter = after;
|
|
list->ResourceBarrier(1, &barrier);
|
|
}
|
|
|
|
static void copy_to_staging(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, Staging& staging,
|
|
D3D12_RESOURCE_STATES state)
|
|
{
|
|
transition(list, resource, state, D3D12_RESOURCE_STATE_COPY_SOURCE);
|
|
D3D12_TEXTURE_COPY_LOCATION destination = {};
|
|
destination.pResource = staging.buffer;
|
|
destination.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
|
|
destination.PlacedFootprint = staging.layout;
|
|
D3D12_TEXTURE_COPY_LOCATION source = {};
|
|
source.pResource = resource;
|
|
source.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
|
|
source.SubresourceIndex = 0;
|
|
list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
|
|
transition(list, resource, D3D12_RESOURCE_STATE_COPY_SOURCE, state);
|
|
}
|
|
|
|
// --- CUDA-side resources (worker thread) -----------------------------------------
|
|
|
|
// Sampler of an input texture: clamp addressing, normalized coordinates (as NGX's own textures); bilinear
|
|
// for the colour input (D4R_SHIM_COLOR_LINEAR), optionally for motion / depth (experiments).
|
|
static CudaTextureDesc input_sampler(UINT width, uint32_t channels, bool linear)
|
|
{
|
|
CudaTextureDesc sampler = {};
|
|
sampler.addressMode[0] = 1;
|
|
sampler.addressMode[1] = 1;
|
|
static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
|
|
sampler.filterMode = linear && colorLinear ? 1 : 0;
|
|
static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
|
|
static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
|
|
if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
|
|
sampler.filterMode = 1;
|
|
sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
|
|
return sampler;
|
|
}
|
|
|
|
static bool ensure_cuda_image(CudaImage& image, UINT width, UINT height, uint32_t format, uint32_t channels,
|
|
bool surface, bool linear)
|
|
{
|
|
if (image.array != nullptr && image.width == width && image.height == height)
|
|
return true;
|
|
if (image.object != 0)
|
|
surface ? g.cu.surfObjectDestroy(image.object) : g.cu.texObjectDestroy(image.object);
|
|
if (image.array != nullptr)
|
|
g.cu.arrayDestroy(image.array);
|
|
image = CudaImage{};
|
|
int result;
|
|
if (surface)
|
|
{
|
|
const CudaArray3DDescriptor descriptor{width, height, 0, format, channels, 2u /* SURFACE_LDST */};
|
|
result = g.cu.array3DCreate(&image.array, &descriptor);
|
|
}
|
|
else
|
|
{
|
|
const CudaArrayDescriptor descriptor{width, height, format, channels};
|
|
result = g.cu.arrayCreate(&image.array, &descriptor);
|
|
}
|
|
if (result != 0)
|
|
{
|
|
logf("CUDA array create %ux%u fmt=0x%x ch=%u failed: %d", width, height, format, channels, result);
|
|
return false;
|
|
}
|
|
CudaResourceDesc resource = {};
|
|
resource.resType = 0; // CU_RESOURCE_TYPE_ARRAY
|
|
resource.res.array.hArray = image.array;
|
|
if (surface)
|
|
result = g.cu.surfObjectCreate(&image.object, &resource);
|
|
else
|
|
{
|
|
// Matches the texture objects NGX itself creates: clamp addressing,
|
|
// normalized coordinates.
|
|
CudaTextureDesc sampler = {};
|
|
sampler.addressMode[0] = 1;
|
|
sampler.addressMode[1] = 1;
|
|
// D4R_SHIM_COLOR_LINEAR=0 samples the colour input with point filtering (experiment).
|
|
static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
|
|
sampler.filterMode = linear && colorLinear ? 1 : 0;
|
|
// D4R_SHIM_MOTION_LINEAR / D4R_SHIM_DEPTH_LINEAR=1: bilinear filtering for
|
|
// those planes (identified by format; experiment).
|
|
static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
|
|
static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
|
|
if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
|
|
sampler.filterMode = 1;
|
|
sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
|
|
result = g.cu.texObjectCreate(&image.object, &resource, &sampler, nullptr);
|
|
}
|
|
if (result != 0)
|
|
{
|
|
logf("CUDA %s object create failed: %d", surface ? "surface" : "texture", result);
|
|
return false;
|
|
}
|
|
image.width = width;
|
|
image.height = height;
|
|
return true;
|
|
}
|
|
|
|
// Device <-> array copy. Queued on the null stream, where NGX evaluates, when the bridge provides
|
|
// it (D4R_SHIM_ASYNC_2D=0: ZLUDA's cuMemcpy2D); callers synchronize before the data is used.
|
|
static int copy_2d(const CudaMemcpy2D& copy)
|
|
{
|
|
static const bool async = env_uint("D4R_SHIM_ASYNC_2D", 1) != 0;
|
|
if (async && g.cu.memcpy2DAsync != nullptr && g.cu.memcpy2DAsync(©, nullptr) == 0)
|
|
return 0;
|
|
return g.cu.memcpy2D(©);
|
|
}
|
|
|
|
// Splits row-wise staging work (tens of MB per frame at 1440p, which one core
|
|
// moves at ~11 GB/s) across helper threads plus the calling worker thread.
|
|
// D4R_SHIM_COPY_THREADS sets the total thread count (default 4, 1 = serial).
|
|
class RowPool
|
|
{
|
|
public:
|
|
void run(UINT rows, size_t bytes, const std::function<void(UINT, UINT)>& task)
|
|
{
|
|
const unsigned int parts = threads();
|
|
if (parts <= 1 || bytes < (2u << 20) || rows < parts * 8)
|
|
{
|
|
task(0, rows);
|
|
return;
|
|
}
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
task_ = &task;
|
|
rows_ = rows;
|
|
parts_ = parts;
|
|
remaining_ = parts - 1;
|
|
++generation_;
|
|
}
|
|
start_.notify_all();
|
|
task(0, rows / parts);
|
|
std::unique_lock<std::mutex> lock(mutex_);
|
|
done_.wait(lock, [this] { return remaining_ == 0; });
|
|
task_ = nullptr;
|
|
}
|
|
|
|
private:
|
|
unsigned int threads()
|
|
{
|
|
std::call_once(once_, [this] {
|
|
count_ = env_uint("D4R_SHIM_COPY_THREADS", 4);
|
|
count_ = count_ < 1 ? 1 : (count_ > 16 ? 16 : count_);
|
|
for (unsigned int index = 1; index < count_; ++index)
|
|
std::thread([this, index] { helper(index); }).detach();
|
|
});
|
|
return count_;
|
|
}
|
|
|
|
void helper(unsigned int index)
|
|
{
|
|
uint64_t seen = 0;
|
|
for (;;)
|
|
{
|
|
const std::function<void(UINT, UINT)>* task;
|
|
UINT begin, end;
|
|
{
|
|
std::unique_lock<std::mutex> lock(mutex_);
|
|
start_.wait(lock, [&] { return generation_ != seen; });
|
|
seen = generation_;
|
|
task = task_;
|
|
begin = static_cast<UINT>(static_cast<uint64_t>(rows_) * index / parts_);
|
|
end = static_cast<UINT>(static_cast<uint64_t>(rows_) * (index + 1) / parts_);
|
|
}
|
|
(*task)(begin, end);
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
--remaining_;
|
|
}
|
|
done_.notify_one();
|
|
}
|
|
}
|
|
|
|
std::once_flag once_;
|
|
unsigned int count_ = 1;
|
|
std::mutex mutex_;
|
|
std::condition_variable start_, done_;
|
|
const std::function<void(UINT, UINT)>* task_ = nullptr;
|
|
UINT rows_ = 0;
|
|
unsigned int parts_ = 1, remaining_ = 0;
|
|
uint64_t generation_ = 0;
|
|
};
|
|
|
|
// One pool per CPU pipeline stage; a pool serves one caller at a time.
|
|
static RowPool g_inputRows, g_outputRows;
|
|
|
|
// Copies rows between pitched buffers.
|
|
static void copy_rows(RowPool& pool, uint8_t* destination, size_t destinationPitch, const uint8_t* source,
|
|
size_t sourcePitch, size_t rowBytes, UINT rows)
|
|
{
|
|
pool.run(rows, rowBytes * rows, [&](UINT begin, UINT end) {
|
|
if (destinationPitch == rowBytes && sourcePitch == rowBytes)
|
|
std::memcpy(destination + begin * rowBytes, source + begin * rowBytes, rowBytes * (end - begin));
|
|
else
|
|
for (UINT y = begin; y < end; ++y)
|
|
std::memcpy(destination + y * destinationPitch, source + y * sourcePitch, rowBytes);
|
|
});
|
|
}
|
|
|
|
// Overlapped staging: frame N's uploads run while frame N-1's result downloads
|
|
// (PCIe is full duplex), as async copies between page-locked host memory and
|
|
// linear device buffers on two non-blocking streams. D4R_SHIM_ASYNC_COPIES=0
|
|
// keeps the synchronous host<->array copies.
|
|
static bool async_copies()
|
|
{
|
|
static const bool enabled = env_uint("D4R_SHIM_ASYNC_COPIES", 1) != 0;
|
|
return enabled && g.cu.memAllocHost != nullptr;
|
|
}
|
|
|
|
struct CopyStreams
|
|
{
|
|
void* up = nullptr;
|
|
void* down = nullptr;
|
|
};
|
|
|
|
// Worker thread only.
|
|
static CopyStreams* copy_streams()
|
|
{
|
|
static CopyStreams streams;
|
|
static bool attempted = false;
|
|
if (!attempted)
|
|
{
|
|
attempted = true;
|
|
constexpr unsigned int kNonBlocking = 1; // CU_STREAM_NON_BLOCKING
|
|
if (g.cu.streamCreate(&streams.up, kNonBlocking) != 0 || g.cu.streamCreate(&streams.down, kNonBlocking) != 0)
|
|
{
|
|
logf("cuStreamCreate failed; using synchronous copies");
|
|
streams = {};
|
|
}
|
|
}
|
|
return streams.up != nullptr ? &streams : nullptr;
|
|
}
|
|
|
|
// Grows a host plane's buffer. Page-locked memory is allocated on the CUDA
|
|
// worker thread (the only one with a current context).
|
|
static void reserve_host(HostPlane& host, size_t bytes)
|
|
{
|
|
if (bytes <= host.capacity)
|
|
return;
|
|
auto allocate = [&host, bytes] {
|
|
if (host.pinned)
|
|
g.cu.memFreeHost(host.bytes);
|
|
host = HostPlane{};
|
|
void* pinned = nullptr;
|
|
if (async_copies() && g.cu.memAllocHost(&pinned, bytes) == 0)
|
|
{
|
|
host.bytes = static_cast<uint8_t*>(pinned);
|
|
host.pinned = true;
|
|
}
|
|
else
|
|
{
|
|
host.fallback.resize(bytes);
|
|
host.bytes = host.fallback.data();
|
|
}
|
|
host.capacity = bytes;
|
|
return 0;
|
|
};
|
|
if (async_copies() && std::this_thread::get_id() != g.workerThread)
|
|
g.worker.call(allocate);
|
|
else
|
|
allocate();
|
|
}
|
|
|
|
static void release_host(HostPlane& host)
|
|
{
|
|
if (host.pinned)
|
|
g.cu.memFreeHost(host.bytes);
|
|
host = HostPlane{};
|
|
}
|
|
|
|
// Worker thread only.
|
|
static bool ensure_linear(CudaDevicePtr& buffer, size_t& capacity, size_t bytes)
|
|
{
|
|
if (bytes <= capacity)
|
|
return true;
|
|
if (buffer != 0)
|
|
g.cu.memFree(buffer);
|
|
buffer = 0;
|
|
capacity = 0;
|
|
if (g.cu.memAlloc(&buffer, bytes) != 0)
|
|
{
|
|
buffer = 0;
|
|
logf("cuMemAlloc(%zu) for staging failed", bytes);
|
|
return false;
|
|
}
|
|
capacity = bytes;
|
|
return true;
|
|
}
|
|
|
|
static uint32_t plane_format(Plane plane)
|
|
{
|
|
return (plane == Plane::Color || plane == Plane::Motion) ? CUDA_FORMAT_HALF : CUDA_FORMAT_FLOAT;
|
|
}
|
|
|
|
static uint32_t plane_channels(Plane plane)
|
|
{
|
|
return plane == Plane::Color ? 4 : (plane == Plane::Motion ? 2 : 1);
|
|
}
|
|
|
|
// --- VRAM interop ------------------------------------------------------------------
|
|
//
|
|
// With D4R_SHIM_VRAM_INTEROP=1 the inputs and the result never leave VRAM.
|
|
// vkd3d-proton's interop interface hands out the game's VkDevice, the VkImage
|
|
// behind each D3D12 texture, and the raw VkCommandBuffer behind the game's
|
|
// D3D12 command list. evaluate() records Vulkan copies of the inputs into
|
|
// exportable device-local buffers (colour formats other than RGBA16F are
|
|
// converted by a blit first), and of the latest result out of one; the Wine
|
|
// nvcuda bridge (d4rImportVulkanMemory) maps those buffers into CUDA. The
|
|
// worker then only moves data between them and the CUDA arrays on the GPU.
|
|
// Frames whose formats do not qualify use the readback/upload path.
|
|
|
|
struct VulkanInterop
|
|
{
|
|
std::once_flag once;
|
|
bool ready = false;
|
|
ID3D12DXVKInteropDevice1* interop = nullptr;
|
|
ID3D12DXVKInteropDeviceD4R2* lifetime = nullptr;
|
|
ID3D12DXVKInteropDeviceD4R* split = nullptr; // only with the d4r vkd3d-proton patch
|
|
VkDevice device = VK_NULL_HANDLE;
|
|
VkPhysicalDeviceMemoryProperties memory = {};
|
|
VkPhysicalDevice physical = VK_NULL_HANDLE;
|
|
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
|
|
PFN_vkCreateBuffer createBuffer = nullptr;
|
|
PFN_vkDestroyBuffer destroyBuffer = nullptr;
|
|
PFN_vkGetBufferMemoryRequirements bufferRequirements = nullptr;
|
|
PFN_vkBindBufferMemory bindBuffer = nullptr;
|
|
PFN_vkCreateImage createImage = nullptr;
|
|
PFN_vkDestroyImage destroyImage = nullptr;
|
|
PFN_vkGetImageMemoryRequirements imageRequirements = nullptr;
|
|
PFN_vkBindImageMemory bindImage = nullptr;
|
|
PFN_vkAllocateMemory allocate = nullptr;
|
|
PFN_vkFreeMemory free = nullptr;
|
|
PFN_vkCmdPipelineBarrier barrier = nullptr;
|
|
PFN_vkCmdCopyImageToBuffer copyImageToBuffer = nullptr;
|
|
PFN_vkCmdCopyBufferToImage copyBufferToImage = nullptr;
|
|
PFN_vkCmdBlitImage blit = nullptr;
|
|
PFN_vkCmdFillBuffer fill = nullptr;
|
|
PFN_vkCreateSemaphore createSemaphore = nullptr;
|
|
PFN_vkDestroySemaphore destroySemaphore = nullptr;
|
|
PFN_vkSignalSemaphore signalSemaphore = nullptr;
|
|
int(WINAPI* import)(VkDevice, uint64_t, uint64_t, CudaDevicePtr*, void**) = nullptr;
|
|
int(WINAPI* release)(void*) = nullptr;
|
|
};
|
|
static VulkanInterop g_vk;
|
|
|
|
static bool vram_interop_requested()
|
|
{
|
|
static const bool requested = env_uint("D4R_SHIM_VRAM_INTEROP", 0) != 0;
|
|
return requested;
|
|
}
|
|
|
|
// Game thread, once.
|
|
static void init_vram_interop()
|
|
{
|
|
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDevice1), reinterpret_cast<void**>(&g_vk.interop))))
|
|
{
|
|
logf("VRAM interop: ID3D12DXVKInteropDevice1 unavailable (not vkd3d-proton?)");
|
|
return;
|
|
}
|
|
VkInstance instance = VK_NULL_HANDLE;
|
|
VkPhysicalDevice physical = VK_NULL_HANDLE;
|
|
if (FAILED(g_vk.interop->GetVulkanHandles(&instance, &physical, &g_vk.device)))
|
|
{
|
|
logf("VRAM interop: GetVulkanHandles failed");
|
|
return;
|
|
}
|
|
UINT count = 0;
|
|
g_vk.interop->GetDeviceExtensions(&count, nullptr);
|
|
std::vector<const char*> extensions(count);
|
|
g_vk.interop->GetDeviceExtensions(&count, extensions.data());
|
|
bool exportable = false;
|
|
for (const char* name : extensions)
|
|
exportable |= std::strcmp(name, "VK_KHR_external_memory_win32") == 0;
|
|
HMODULE vulkan = GetModuleHandleA("vulkan-1.dll");
|
|
if (vulkan == nullptr)
|
|
vulkan = LoadLibraryA("vulkan-1.dll");
|
|
PFN_vkGetInstanceProcAddr instanceProc = nullptr;
|
|
if (vulkan != nullptr)
|
|
load_export(vulkan, "vkGetInstanceProcAddr", instanceProc);
|
|
load_export(g.cuda, "d4rImportVulkanMemory", g_vk.import);
|
|
load_export(g.cuda, "d4rReleaseVulkanMemory", g_vk.release);
|
|
if (!exportable || instanceProc == nullptr || g_vk.import == nullptr || g_vk.release == nullptr)
|
|
{
|
|
logf("VRAM interop: unavailable (external_memory_win32 %d, vulkan-1 %p, bridge exports %d)", exportable,
|
|
reinterpret_cast<void*>(instanceProc), g_vk.import != nullptr && g_vk.release != nullptr);
|
|
return;
|
|
}
|
|
auto deviceProc = reinterpret_cast<PFN_vkGetDeviceProcAddr>(instanceProc(instance, "vkGetDeviceProcAddr"));
|
|
auto memoryProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(
|
|
instanceProc(instance, "vkGetPhysicalDeviceMemoryProperties"));
|
|
if (deviceProc == nullptr || memoryProperties == nullptr)
|
|
return;
|
|
memoryProperties(physical, &g_vk.memory);
|
|
g_vk.physical = physical;
|
|
g_vk.formatProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceFormatProperties>(
|
|
instanceProc(instance, "vkGetPhysicalDeviceFormatProperties"));
|
|
bool ok = g_vk.formatProperties != nullptr;
|
|
auto load = [&](auto& function, const char* name) {
|
|
function = reinterpret_cast<std::remove_reference_t<decltype(function)>>(deviceProc(g_vk.device, name));
|
|
ok &= function != nullptr;
|
|
};
|
|
load(g_vk.createBuffer, "vkCreateBuffer");
|
|
load(g_vk.destroyBuffer, "vkDestroyBuffer");
|
|
load(g_vk.bufferRequirements, "vkGetBufferMemoryRequirements");
|
|
load(g_vk.bindBuffer, "vkBindBufferMemory");
|
|
load(g_vk.createImage, "vkCreateImage");
|
|
load(g_vk.destroyImage, "vkDestroyImage");
|
|
load(g_vk.imageRequirements, "vkGetImageMemoryRequirements");
|
|
load(g_vk.bindImage, "vkBindImageMemory");
|
|
load(g_vk.allocate, "vkAllocateMemory");
|
|
load(g_vk.free, "vkFreeMemory");
|
|
load(g_vk.barrier, "vkCmdPipelineBarrier");
|
|
load(g_vk.copyImageToBuffer, "vkCmdCopyImageToBuffer");
|
|
load(g_vk.copyBufferToImage, "vkCmdCopyBufferToImage");
|
|
load(g_vk.blit, "vkCmdBlitImage");
|
|
g_vk.fill = reinterpret_cast<PFN_vkCmdFillBuffer>(deviceProc(g_vk.device, "vkCmdFillBuffer"));
|
|
load(g_vk.createSemaphore, "vkCreateSemaphore");
|
|
load(g_vk.destroySemaphore, "vkDestroySemaphore");
|
|
load(g_vk.signalSemaphore, "vkSignalSemaphore");
|
|
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R), reinterpret_cast<void**>(&g_vk.split))))
|
|
g_vk.split = nullptr;
|
|
if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R2), reinterpret_cast<void**>(&g_vk.lifetime))))
|
|
g_vk.lifetime = nullptr;
|
|
g_vk.ready = ok;
|
|
logf("VRAM interop: %s (VkDevice %p), split frames %s", ok ? "ready" : "missing Vulkan entry points",
|
|
static_cast<void*>(g_vk.device), g_vk.split != nullptr ? "available" : "unavailable (stock vkd3d-proton)");
|
|
}
|
|
|
|
static bool vram_interop_available()
|
|
{
|
|
if (!vram_interop_requested())
|
|
return false;
|
|
std::call_once(g_vk.once, init_vram_interop);
|
|
return g_vk.ready;
|
|
}
|
|
|
|
// Device-local memory; prefers memory the CPU cannot map (no BAR window).
|
|
static int device_local_type(uint32_t typeBits)
|
|
{
|
|
for (int pass = 0; pass < 2; ++pass)
|
|
for (uint32_t index = 0; index < g_vk.memory.memoryTypeCount; ++index)
|
|
{
|
|
const VkMemoryPropertyFlags flags = g_vk.memory.memoryTypes[index].propertyFlags;
|
|
if ((typeBits & (1u << index)) && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
|
|
(pass == 1 || !(flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)))
|
|
return static_cast<int>(index);
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
// Buffers of released features, reused by later ones. ROCm up to at least 7.2.4 never frees memory mapped with
|
|
// hipExternalMemoryGetMappedBuffer (the mapping's buffer view is retained twice but released once by hipFree), so
|
|
// destroying a buffer would leak its memory: each DLSS quality change (a new feature) would add ~100 MB of VRAM.
|
|
static std::mutex g_vramPoolMutex;
|
|
static std::vector<VramBuffer> g_vramPool;
|
|
constexpr size_t kVramPoolLimit = 64;
|
|
|
|
// The smallest pooled buffer that holds `bytes`.
|
|
static bool take_pooled_vram_buffer(VramBuffer& target, size_t bytes)
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
|
|
auto best = g_vramPool.end();
|
|
for (auto it = g_vramPool.begin(); it != g_vramPool.end(); ++it)
|
|
if (it->bytes >= bytes && (best == g_vramPool.end() || it->bytes < best->bytes))
|
|
best = it;
|
|
if (best == g_vramPool.end())
|
|
return false;
|
|
target = *best;
|
|
g_vramPool.erase(best);
|
|
return true;
|
|
}
|
|
|
|
// Game thread. The CUDA import runs on the worker, which owns the context.
|
|
static bool create_vram_buffer(VramBuffer& target, size_t bytes)
|
|
{
|
|
if (take_pooled_vram_buffer(target, bytes))
|
|
return true;
|
|
VramBuffer buffer;
|
|
VkExternalMemoryBufferCreateInfo external = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
|
|
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
|
|
VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
|
|
info.size = bytes;
|
|
info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
|
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
|
|
VkResult result = g_vk.createBuffer(g_vk.device, &info, nullptr, &buffer.buffer);
|
|
VkMemoryRequirements requirements = {};
|
|
if (result == VK_SUCCESS)
|
|
g_vk.bufferRequirements(g_vk.device, buffer.buffer, &requirements);
|
|
const int type = device_local_type(requirements.memoryTypeBits);
|
|
VkMemoryDedicatedAllocateInfo dedicated = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
|
|
dedicated.buffer = buffer.buffer;
|
|
VkExportMemoryAllocateInfo exportInfo = {VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, &dedicated};
|
|
exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
|
|
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo};
|
|
allocation.allocationSize = requirements.size;
|
|
allocation.memoryTypeIndex = static_cast<uint32_t>(type);
|
|
if (result == VK_SUCCESS && type < 0)
|
|
result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
|
|
if (result == VK_SUCCESS)
|
|
result = g_vk.allocate(g_vk.device, &allocation, nullptr, &buffer.memory);
|
|
if (result == VK_SUCCESS)
|
|
result = g_vk.bindBuffer(g_vk.device, buffer.buffer, buffer.memory, 0);
|
|
int imported = -1;
|
|
if (result == VK_SUCCESS)
|
|
imported = g.worker.call([&] {
|
|
return g_vk.import(g_vk.device, reinterpret_cast<uint64_t>(buffer.memory), requirements.size,
|
|
&buffer.device, &buffer.external);
|
|
});
|
|
if (result != VK_SUCCESS || imported != 0)
|
|
{
|
|
logf("VRAM interop: exportable buffer of %zu bytes failed (VkResult %d, import %d)", bytes, result, imported);
|
|
if (buffer.memory != VK_NULL_HANDLE)
|
|
g_vk.free(g_vk.device, buffer.memory, nullptr);
|
|
if (buffer.buffer != VK_NULL_HANDLE)
|
|
g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
|
|
return false;
|
|
}
|
|
buffer.bytes = bytes;
|
|
target = buffer;
|
|
return true;
|
|
}
|
|
|
|
// Only once no command list or CUDA work uses the buffer.
|
|
static void destroy_vram_buffer(VramBuffer& buffer)
|
|
{
|
|
if (buffer.external != nullptr)
|
|
g.worker.call([&buffer] {
|
|
const int freed = g.cu.memFree(buffer.device);
|
|
const int released = g_vk.release(buffer.external);
|
|
if (freed != 0 || released != 0)
|
|
logf("VRAM interop: freeing %zu-byte buffer: cuMemFree %d, release %d", buffer.bytes, freed, released);
|
|
return released;
|
|
});
|
|
if (buffer.memory != VK_NULL_HANDLE)
|
|
g_vk.free(g_vk.device, buffer.memory, nullptr);
|
|
if (buffer.buffer != VK_NULL_HANDLE)
|
|
g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
|
|
buffer = VramBuffer{};
|
|
}
|
|
|
|
// Only once no command list or CUDA work uses the buffer: keeps it for a later feature (see g_vramPool).
|
|
static void recycle_vram_buffer(VramBuffer& buffer)
|
|
{
|
|
if (buffer.buffer == VK_NULL_HANDLE)
|
|
return;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
|
|
if (g_vramPool.size() < kVramPoolLimit)
|
|
{
|
|
g_vramPool.push_back(buffer);
|
|
buffer = VramBuffer{};
|
|
return;
|
|
}
|
|
}
|
|
destroy_vram_buffer(buffer);
|
|
}
|
|
|
|
static void free_vram_pool()
|
|
{
|
|
std::vector<VramBuffer> pool;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_vramPoolMutex);
|
|
pool.swap(g_vramPool);
|
|
}
|
|
for (VramBuffer& buffer : pool)
|
|
destroy_vram_buffer(buffer);
|
|
}
|
|
|
|
static void destroy_vram_image(VramImage& image)
|
|
{
|
|
if (image.memory != VK_NULL_HANDLE)
|
|
g_vk.free(g_vk.device, image.memory, nullptr);
|
|
if (image.image != VK_NULL_HANDLE)
|
|
g_vk.destroyImage(g_vk.device, image.image, nullptr);
|
|
image = VramImage{};
|
|
}
|
|
|
|
static bool create_vram_image(VramImage& image, UINT width, UINT height, VkFormat format)
|
|
{
|
|
VkImageCreateInfo info = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
|
|
info.imageType = VK_IMAGE_TYPE_2D;
|
|
info.format = format;
|
|
info.extent = {width, height, 1};
|
|
info.mipLevels = 1;
|
|
info.arrayLayers = 1;
|
|
info.samples = VK_SAMPLE_COUNT_1_BIT;
|
|
info.tiling = VK_IMAGE_TILING_OPTIMAL;
|
|
info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
|
|
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
|
|
info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
VkResult result = g_vk.createImage(g_vk.device, &info, nullptr, &image.image);
|
|
VkMemoryRequirements requirements = {};
|
|
if (result == VK_SUCCESS)
|
|
g_vk.imageRequirements(g_vk.device, image.image, &requirements);
|
|
const int type = device_local_type(requirements.memoryTypeBits);
|
|
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
|
allocation.allocationSize = requirements.size;
|
|
allocation.memoryTypeIndex = static_cast<uint32_t>(type);
|
|
if (result == VK_SUCCESS && type < 0)
|
|
result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
|
|
if (result == VK_SUCCESS)
|
|
result = g_vk.allocate(g_vk.device, &allocation, nullptr, &image.memory);
|
|
if (result == VK_SUCCESS)
|
|
result = g_vk.bindImage(g_vk.device, image.image, image.memory, 0);
|
|
if (result != VK_SUCCESS)
|
|
{
|
|
logf("VRAM interop: %ux%u conversion image failed: %d", width, height, result);
|
|
destroy_vram_image(image);
|
|
return false;
|
|
}
|
|
image.width = width;
|
|
image.height = height;
|
|
image.format = format;
|
|
return true;
|
|
}
|
|
|
|
static size_t canonical_texel_bytes(Plane plane)
|
|
{
|
|
return (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
|
|
}
|
|
|
|
static size_t vk_texel_bytes(VkFormat format)
|
|
{
|
|
switch (format)
|
|
{
|
|
case VK_FORMAT_R16G16B16A16_SFLOAT:
|
|
case VK_FORMAT_R16G16B16A16_UINT:
|
|
case VK_FORMAT_R16G16B16A16_SINT:
|
|
case VK_FORMAT_R16G16B16A16_UNORM:
|
|
return 8;
|
|
case VK_FORMAT_R32_SFLOAT:
|
|
case VK_FORMAT_R32_UINT:
|
|
case VK_FORMAT_R32_SINT:
|
|
case VK_FORMAT_R16G16_SFLOAT:
|
|
case VK_FORMAT_R16G16_UINT:
|
|
case VK_FORMAT_R16G16_SINT:
|
|
case VK_FORMAT_D32_SFLOAT:
|
|
case VK_FORMAT_D32_SFLOAT_S8_UINT: // the depth aspect alone
|
|
return 4;
|
|
case VK_FORMAT_R16_SFLOAT:
|
|
case VK_FORMAT_R16_UNORM:
|
|
case VK_FORMAT_R16_UINT:
|
|
return 2;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
// How one input or the output moves between its D3D12 texture and a buffer.
|
|
struct VramCopy
|
|
{
|
|
ID3D12Resource* resource = nullptr;
|
|
VkImage image = VK_NULL_HANDLE;
|
|
VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT;
|
|
bool convert = false; // blit to/from the plane's canonical format
|
|
UINT width = 0, height = 0;
|
|
};
|
|
|
|
static bool vram_blit_supported(VkFormat source, VkFormat destination)
|
|
{
|
|
VkFormatProperties sourceProperties = {}, destinationProperties = {};
|
|
g_vk.formatProperties(g_vk.physical, source, &sourceProperties);
|
|
g_vk.formatProperties(g_vk.physical, destination, &destinationProperties);
|
|
return (sourceProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) != 0 &&
|
|
(destinationProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_DST_BIT) != 0;
|
|
}
|
|
|
|
// sRGB is deliberately excluded: the host staging path treats its bytes as
|
|
// UNORM and a Vulkan blit would apply an sRGB transfer function instead.
|
|
static bool vram_color_blit_supported(VkFormat format, bool output)
|
|
{
|
|
switch (format)
|
|
{
|
|
case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
|
|
case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
|
|
case VK_FORMAT_R8G8B8A8_UNORM:
|
|
case VK_FORMAT_B8G8R8A8_UNORM:
|
|
case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32:
|
|
break;
|
|
default:
|
|
return false;
|
|
}
|
|
return output ? vram_blit_supported(VK_FORMAT_R16G16B16A16_SFLOAT, format)
|
|
: vram_blit_supported(format, VK_FORMAT_R16G16B16A16_SFLOAT);
|
|
}
|
|
|
|
// The host path takes only the first (R) component of an exposure texel.
|
|
static bool vram_exposure_blit_supported(VkFormat format)
|
|
{
|
|
switch (format)
|
|
{
|
|
case VK_FORMAT_R16_SFLOAT:
|
|
case VK_FORMAT_R16G16B16A16_SFLOAT:
|
|
case VK_FORMAT_R32G32B32A32_SFLOAT:
|
|
return vram_blit_supported(format, VK_FORMAT_R32_SFLOAT);
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Depth in a 16-bit format is widened to R32F by a blit (UNORM normalizes, SFLOAT converts).
|
|
static bool vram_depth_blit_supported(VkFormat format)
|
|
{
|
|
switch (format)
|
|
{
|
|
case VK_FORMAT_R16_SFLOAT:
|
|
case VK_FORMAT_R16_UNORM:
|
|
return vram_blit_supported(format, VK_FORMAT_R32_SFLOAT);
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Motion vectors in a wider float format keep their first two components (as the host path does).
|
|
static bool vram_motion_blit_supported(VkFormat format)
|
|
{
|
|
switch (format)
|
|
{
|
|
case VK_FORMAT_R16G16B16A16_SFLOAT:
|
|
case VK_FORMAT_R32G32_SFLOAT:
|
|
case VK_FORMAT_R32G32B32A32_SFLOAT:
|
|
case VK_FORMAT_R8G8B8A8_UNORM:
|
|
return vram_blit_supported(format, VK_FORMAT_R16G16_SFLOAT);
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Whether `resource` can be copied raw (or, for colour/motion/exposure, blitted)
|
|
// to/from the canonical layout of `plane`.
|
|
static bool describe_vram_copy(ID3D12Resource* resource, Plane plane, VramCopy& copy, bool output = false)
|
|
{
|
|
UINT64 handle = 0, offset = 0;
|
|
VkFormat format = VK_FORMAT_UNDEFINED;
|
|
if (FAILED(g_vk.interop->GetVulkanResourceInfo1(resource, &handle, &offset, &format)) || handle == 0)
|
|
return false;
|
|
D3D12_RESOURCE_DESC desc;
|
|
resource->GetDesc(&desc);
|
|
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1)
|
|
return false;
|
|
copy.resource = resource;
|
|
copy.image = reinterpret_cast<VkImage>(handle);
|
|
copy.width = static_cast<UINT>(desc.Width);
|
|
copy.height = desc.Height;
|
|
copy.aspect = (format == VK_FORMAT_D32_SFLOAT || format == VK_FORMAT_D32_SFLOAT_S8_UINT)
|
|
? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
|
|
if (plane == Plane::Color)
|
|
{
|
|
copy.convert = vram_color_blit_supported(format, output);
|
|
return copy.convert || format == VK_FORMAT_R16G16B16A16_SFLOAT;
|
|
}
|
|
if (plane == Plane::Exposure)
|
|
copy.convert = vram_exposure_blit_supported(format);
|
|
else if (plane == Plane::Motion)
|
|
copy.convert = vram_motion_blit_supported(format);
|
|
else if (plane == Plane::Depth)
|
|
copy.convert = vram_depth_blit_supported(format);
|
|
return copy.convert || vk_texel_bytes(format) == canonical_texel_bytes(plane);
|
|
}
|
|
|
|
static const VkMemoryBarrier kBeforeTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_MEMORY_WRITE_BIT,
|
|
VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT};
|
|
static const VkMemoryBarrier kAfterTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT,
|
|
VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT};
|
|
|
|
static VkBufferImageCopy full_region(const VramCopy& copy)
|
|
{
|
|
VkBufferImageCopy region = {};
|
|
region.imageSubresource.aspectMask = copy.aspect;
|
|
region.imageSubresource.layerCount = 1;
|
|
region.imageExtent = {copy.width, copy.height, 1};
|
|
return region;
|
|
}
|
|
|
|
// Records the input copies into the slot's buffers. The resources are in
|
|
// COPY_SOURCE state already.
|
|
static bool record_vram_inputs(Feature& feature, ID3D12GraphicsCommandList* list, InputSlot& slot,
|
|
const VramCopy* copies, int count)
|
|
{
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
|
|
return false;
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
|
|
nullptr, 0, nullptr);
|
|
for (int index = 0; index < count; ++index)
|
|
{
|
|
const VramCopy& copy = copies[index];
|
|
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
|
|
g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, &layout);
|
|
VkBufferImageCopy region = full_region(copy);
|
|
static const Plane kPlanes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
|
|
if (feature.linearInputs)
|
|
region.bufferRowLength = static_cast<uint32_t>(slot.host[index].rowBytes / canonical_texel_bytes(kPlanes[index]));
|
|
if (!copy.convert)
|
|
{
|
|
g_vk.copyImageToBuffer(cmd, copy.image, layout, slot.vram[index].buffer, 1, ®ion);
|
|
continue;
|
|
}
|
|
// Convert colour to RGBA16F, motion to RG16F or exposure's first component
|
|
// to R32F, then copy from the canonical image into the shared buffer.
|
|
VkImage conversion = kPlanes[index] == Plane::Exposure ? feature.exposureConversion.image
|
|
: kPlanes[index] == Plane::Motion ? feature.motionConversion.image
|
|
: kPlanes[index] == Plane::Depth ? feature.depthConversion.image
|
|
: feature.colorConversion.image;
|
|
VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
|
|
toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
|
|
toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
|
|
toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
toDestination.image = conversion;
|
|
toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
|
|
&toDestination);
|
|
VkImageBlit blit = {};
|
|
blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
|
|
blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
|
|
blit.dstSubresource = blit.srcSubresource;
|
|
blit.dstOffsets[1] = blit.srcOffsets[1];
|
|
g_vk.blit(cmd, copy.image, layout, conversion, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit,
|
|
VK_FILTER_NEAREST);
|
|
VkImageMemoryBarrier toSource = toDestination;
|
|
toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
|
|
toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
|
|
toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
|
|
&toSource);
|
|
g_vk.copyImageToBuffer(cmd, conversion, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, slot.vram[index].buffer, 1,
|
|
®ion);
|
|
}
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
|
|
nullptr, 0, nullptr);
|
|
return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
|
|
}
|
|
|
|
// Records the copy of a finished result into the output texture, which is in
|
|
// COPY_DEST state already.
|
|
static bool record_vram_output(ID3D12GraphicsCommandList* list, const VramBuffer& buffer, const VramCopy& copy,
|
|
const VramImage& conversion)
|
|
{
|
|
VkCommandBuffer cmd = VK_NULL_HANDLE;
|
|
if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
|
|
return false;
|
|
VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
|
|
g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_DEST, &layout);
|
|
const VkBufferImageCopy region = full_region(copy);
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
|
|
nullptr, 0, nullptr);
|
|
if (!copy.convert)
|
|
g_vk.copyBufferToImage(cmd, buffer.buffer, copy.image, layout, 1, ®ion);
|
|
else
|
|
{
|
|
VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
|
|
toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
|
|
toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
|
|
toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
|
|
toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
|
|
toDestination.image = conversion.image;
|
|
toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
|
|
&toDestination);
|
|
g_vk.copyBufferToImage(cmd, buffer.buffer, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion);
|
|
VkImageMemoryBarrier toSource = toDestination;
|
|
toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
|
|
toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
|
|
toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
|
|
toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
|
|
&toSource);
|
|
VkImageBlit blit = {};
|
|
blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
|
|
blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
|
|
blit.dstSubresource = blit.srcSubresource;
|
|
blit.dstOffsets[1] = blit.srcOffsets[1];
|
|
g_vk.blit(cmd, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, copy.image, layout, 1, &blit,
|
|
VK_FILTER_NEAREST);
|
|
}
|
|
g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
|
|
nullptr, 0, nullptr);
|
|
return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
|
|
}
|
|
|
|
// --- Split frames -------------------------------------------------------------------
|
|
//
|
|
// To present frame N's own result, everything the game records after its DLSS
|
|
// call must wait for DLSS. With the d4r vkd3d-proton patch, evaluate() splits
|
|
// the game's command list right after the input copies and the frame marker;
|
|
// vkd3d-proton submits the second half separately, waiting (at queue submission
|
|
// level, so no GPU ring is ever blocked) for splitSemaphore to reach N. The
|
|
// worker writes N's result into output slot N % kOutputSlots. Completed and
|
|
// dropped frames advance the signal only past earlier retired frames. The
|
|
// watchdog reports delays without releasing an unfinished producer.
|
|
|
|
static void release_split_frame(Feature* feature, uint32_t frame)
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->splitMutex);
|
|
const uint64_t value = feature->splitCompletion.retire(frame);
|
|
if (value <= feature->splitSignalled)
|
|
return;
|
|
VkSemaphoreSignalInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO};
|
|
info.semaphore = feature->splitSemaphore;
|
|
info.value = value;
|
|
const VkResult result = g_vk.signalSemaphore(g_vk.device, &info);
|
|
if (result == VK_SUCCESS)
|
|
feature->splitSignalled = value;
|
|
if (result != VK_SUCCESS)
|
|
logf("split frame %llu signal failed (VkResult %d)", static_cast<unsigned long long>(value), result);
|
|
}
|
|
|
|
static void split_watchdog()
|
|
{
|
|
const auto timeout = std::chrono::milliseconds(env_uint("D4R_SHIM_SPLIT_TIMEOUT_MS", 200));
|
|
struct Seen
|
|
{
|
|
uint32_t marker = 0;
|
|
std::chrono::steady_clock::time_point at;
|
|
};
|
|
std::vector<std::pair<Feature*, Seen>> seen;
|
|
for (;;)
|
|
{
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(5));
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
const auto now = std::chrono::steady_clock::now();
|
|
for (Feature* feature : g_features)
|
|
{
|
|
if (!feature->split || feature->markerValue == nullptr)
|
|
continue;
|
|
auto it = std::find_if(seen.begin(), seen.end(), [feature](const auto& entry) { return entry.first == feature; });
|
|
if (it == seen.end())
|
|
it = seen.insert(seen.end(), {feature, Seen{}});
|
|
const uint32_t marker = *feature->markerValue;
|
|
uint64_t signalled;
|
|
{
|
|
std::lock_guard<std::mutex> splitLock(feature->splitMutex);
|
|
signalled = feature->splitSignalled;
|
|
}
|
|
if (marker <= signalled)
|
|
continue; // nothing on the GPU is waiting on us
|
|
if (it->second.marker != marker)
|
|
it->second = Seen{marker, now};
|
|
else if (now - it->second.at > timeout)
|
|
{
|
|
// A slow producer is not a completed producer. Releasing this
|
|
// wait would race the game's output copy with a late CUDA write.
|
|
logf("split frame %u still waiting for DLSS; preserving output synchronization", marker);
|
|
it->second.at = now;
|
|
}
|
|
}
|
|
// Forget released features.
|
|
seen.erase(std::remove_if(seen.begin(), seen.end(),
|
|
[](const auto& entry) {
|
|
return std::find(g_features.begin(), g_features.end(), entry.first) ==
|
|
g_features.end();
|
|
}),
|
|
seen.end());
|
|
}
|
|
}
|
|
|
|
static bool create_split_semaphore(Feature& feature)
|
|
{
|
|
VkSemaphoreTypeCreateInfo type = {VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
|
|
type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
|
|
type.initialValue = 0;
|
|
const VkSemaphoreCreateInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type};
|
|
if (g_vk.createSemaphore(g_vk.device, &info, nullptr, &feature.splitSemaphore) != VK_SUCCESS)
|
|
{
|
|
logf("split frames: vkCreateSemaphore failed");
|
|
return false;
|
|
}
|
|
static std::once_flag watchdog;
|
|
std::call_once(watchdog, [] { std::thread(split_watchdog).detach(); });
|
|
return true;
|
|
}
|
|
|
|
// Game thread: grows a buffer; an outgrown one is kept until the feature is
|
|
// released since queued command lists may still reference it.
|
|
static bool ensure_vram_buffer(Feature& feature, VramBuffer& buffer, size_t bytes)
|
|
{
|
|
if (buffer.buffer != VK_NULL_HANDLE && buffer.bytes >= bytes)
|
|
return true;
|
|
if (buffer.buffer != VK_NULL_HANDLE)
|
|
feature.retiredBuffers.push_back(buffer);
|
|
buffer = VramBuffer{};
|
|
return create_vram_buffer(buffer, bytes);
|
|
}
|
|
|
|
static bool ensure_conversion_image(Feature& feature, VramImage& image, UINT width, UINT height, VkFormat format)
|
|
{
|
|
if (image.image != VK_NULL_HANDLE && image.width == width && image.height == height && image.format == format)
|
|
return true;
|
|
if (image.image != VK_NULL_HANDLE)
|
|
feature.retiredImages.push_back(image);
|
|
image = VramImage{};
|
|
return create_vram_image(image, width, height, format);
|
|
}
|
|
|
|
static void release_vram(Feature& feature)
|
|
{
|
|
if (feature.splitSemaphore != VK_NULL_HANDLE)
|
|
g_vk.destroySemaphore(g_vk.device, feature.splitSemaphore, nullptr);
|
|
for (InputSlot& slot : feature.inputs)
|
|
for (VramBuffer& buffer : slot.vram)
|
|
recycle_vram_buffer(buffer);
|
|
for (OutputSlot& slot : feature.outputs)
|
|
recycle_vram_buffer(slot.vram);
|
|
for (VramBuffer& buffer : feature.retiredBuffers)
|
|
recycle_vram_buffer(buffer);
|
|
feature.retiredBuffers.clear();
|
|
destroy_vram_image(feature.colorConversion);
|
|
destroy_vram_image(feature.depthConversion);
|
|
destroy_vram_image(feature.motionConversion);
|
|
destroy_vram_image(feature.exposureConversion);
|
|
destroy_vram_image(feature.outputConversion);
|
|
for (VramImage& image : feature.retiredImages)
|
|
destroy_vram_image(image);
|
|
feature.retiredImages.clear();
|
|
}
|
|
|
|
// Prep stage: converts one readback staging buffer into the canonical layout.
|
|
static void stage_plane(Plane plane, const Staging& staging, HostPlane& host, FrameTiming* timing, int timingIndex)
|
|
{
|
|
const size_t texelBytes = (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
|
|
reserve_host(host, texelBytes * staging.width * staging.height);
|
|
host.width = staging.width;
|
|
host.height = staging.height;
|
|
host.rowBytes = texelBytes * staging.width;
|
|
uint8_t* rows = host.bytes;
|
|
const size_t rowBytes = host.rowBytes;
|
|
const uint8_t* source = staging.mapped + staging.layout.Offset;
|
|
const size_t sourcePitch = staging.layout.Footprint.RowPitch;
|
|
const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
|
|
if (canonical_input(plane, staging.format))
|
|
copy_rows(g_inputRows, rows, rowBytes, source, sourcePitch, rowBytes, staging.height);
|
|
else
|
|
g_inputRows.run(staging.height, rowBytes * staging.height, [&](UINT begin, UINT end) {
|
|
for (UINT y = begin; y < end; ++y)
|
|
if (plane == Plane::Color && staging.format == DXGI_FORMAT_R11G11B10_FLOAT)
|
|
convert_r11g11b10_row(source + y * sourcePitch,
|
|
reinterpret_cast<uint16_t*>(rows + rowBytes * y), staging.width);
|
|
else
|
|
convert_row_in(plane, staging.format, source + y * sourcePitch, rows + rowBytes * y,
|
|
staging.width);
|
|
});
|
|
if (timing != nullptr)
|
|
timing->convert[timingIndex] = profile_ms(conversionStart, ProfileClock::now());
|
|
}
|
|
|
|
// Worker stage: uploads one staged plane into its CUDA array.
|
|
// D4R_SHIM_MV_STATS=N logs, every N frames, what the game's motion vectors
|
|
// look like (staged RG16F plane): how much of the texture is populated, the
|
|
// populated bounding box (a render-size sub-rectangle means render-resolution
|
|
// vectors in a display-size texture), magnitudes in texture units, and the
|
|
// global image shift between the previous and current colour frames measured
|
|
// in render pixels, so the vectors' unit can be read off their ratio.
|
|
static std::vector<float> downsampled_luma(const HostPlane& color, UINT& width, UINT& height)
|
|
{
|
|
// RGBA16F colour, 2x2 box-downsampled luma.
|
|
width = color.width / 2, height = color.height / 2;
|
|
std::vector<float> luma(static_cast<size_t>(width) * height);
|
|
const uint8_t* bytes = color.bytes != nullptr ? color.bytes : color.fallback.data();
|
|
for (UINT y = 0; y < height; ++y)
|
|
for (UINT x = 0; x < width; ++x)
|
|
{
|
|
float sum = 0.0f;
|
|
for (UINT dy = 0; dy < 2; ++dy)
|
|
{
|
|
const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + (2 * y + dy) * color.rowBytes);
|
|
for (UINT dx = 0; dx < 2; ++dx)
|
|
for (UINT c = 0; c < 3; ++c)
|
|
sum += static_cast<float>(std::bit_cast<_Float16>(row[(2 * x + dx) * 4 + c]));
|
|
}
|
|
luma[static_cast<size_t>(y) * width + x] = sum / (sum + 12.0f); // tone-mapped (HDR input)
|
|
}
|
|
return luma;
|
|
}
|
|
|
|
static void log_motion_stats(const HostPlane& host, const HostPlane& color, uint32_t frame)
|
|
{
|
|
static const uint32_t every = env_uint("D4R_SHIM_MV_STATS", 0);
|
|
static std::vector<float> previousLuma;
|
|
static uint32_t previousFrame = 0;
|
|
if (every == 0 || host.width == 0 || color.width == 0)
|
|
return;
|
|
const bool report = frame % every == 0;
|
|
if (!report && (frame + 1) % every != 0)
|
|
return;
|
|
UINT lumaWidth = 0, lumaHeight = 0;
|
|
std::vector<float> luma = downsampled_luma(color, lumaWidth, lumaHeight);
|
|
if (!report)
|
|
{
|
|
previousLuma = std::move(luma);
|
|
previousFrame = frame;
|
|
return;
|
|
}
|
|
const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
|
|
size_t nonzero = 0;
|
|
double sumMagnitude = 0.0;
|
|
float maxMagnitude = 0.0f;
|
|
UINT minX = host.width, minY = host.height, maxX = 0, maxY = 0;
|
|
std::vector<float> xs, ys;
|
|
for (UINT y = 0; y < host.height; ++y)
|
|
{
|
|
const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + y * host.rowBytes);
|
|
for (UINT x = 0; x < host.width; ++x)
|
|
{
|
|
const float mx = static_cast<float>(std::bit_cast<_Float16>(row[x * 2]));
|
|
const float my = static_cast<float>(std::bit_cast<_Float16>(row[x * 2 + 1]));
|
|
if ((x % 8) == 0 && (y % 8) == 0)
|
|
{
|
|
xs.push_back(mx);
|
|
ys.push_back(my);
|
|
}
|
|
if (mx == 0.0f && my == 0.0f)
|
|
continue;
|
|
++nonzero;
|
|
const float magnitude = std::sqrt(mx * mx + my * my);
|
|
sumMagnitude += magnitude;
|
|
maxMagnitude = std::max(maxMagnitude, magnitude);
|
|
minX = std::min(minX, x), maxX = std::max(maxX, x);
|
|
minY = std::min(minY, y), maxY = std::max(maxY, y);
|
|
}
|
|
}
|
|
auto median = [](std::vector<float>& values) {
|
|
if (values.empty())
|
|
return 0.0f;
|
|
std::nth_element(values.begin(), values.begin() + values.size() / 2, values.end());
|
|
return values[values.size() / 2];
|
|
};
|
|
const float medianX = median(xs), medianY = median(ys);
|
|
// Global shift d (render pixels, current -> previous, the vectors'
|
|
// convention) minimising |cur(p) - prev(p + d)| over the central region.
|
|
int bestX = 0, bestY = 0;
|
|
double bestCost = -1.0, zeroCost = 0.0;
|
|
if (previousFrame + 1 == frame && previousLuma.size() == luma.size())
|
|
{
|
|
const int range = 80, rangeY = 40; // in downsampled pixels
|
|
const int x0 = range, x1 = static_cast<int>(lumaWidth) - range;
|
|
const int y0 = rangeY, y1 = static_cast<int>(lumaHeight) - rangeY;
|
|
for (int dy = -rangeY; dy <= rangeY; ++dy)
|
|
for (int dx = -range; dx <= range; ++dx)
|
|
{
|
|
double cost = 0.0;
|
|
for (int y = y0; y < y1; y += 2)
|
|
for (int x = x0; x < x1; x += 2)
|
|
cost += std::fabs(luma[static_cast<size_t>(y) * lumaWidth + x] -
|
|
previousLuma[static_cast<size_t>(y + dy) * lumaWidth + (x + dx)]);
|
|
if (dx == 0 && dy == 0)
|
|
zeroCost = cost;
|
|
if (bestCost < 0.0 || cost < bestCost)
|
|
bestCost = cost, bestX = dx, bestY = dy;
|
|
}
|
|
}
|
|
const double pixels = static_cast<double>(host.width) * host.height;
|
|
logf("MV_STATS frame=%u size=%ux%u nonzero=%.4f bbox=%u,%u-%u,%u median=(%.3f,%.3f) mean_abs=%.3f max=%.3f "
|
|
"image_shift_render_px=(%d,%d) match=%.3f",
|
|
frame, host.width, host.height, nonzero / pixels, nonzero ? minX : 0, nonzero ? minY : 0, maxX, maxY,
|
|
medianX, medianY, nonzero ? sumMagnitude / nonzero : 0.0, maxMagnitude, 2 * bestX, 2 * bestY,
|
|
zeroCost > 0.0 ? bestCost / zeroCost : 1.0);
|
|
}
|
|
|
|
static bool upload_plane(Plane plane, const HostPlane& host, CudaImage& image, uint32_t frame, FrameTiming* timing,
|
|
int timingIndex)
|
|
{
|
|
if (!ensure_cuda_image(image, host.width, host.height, plane_format(plane), plane_channels(plane), false,
|
|
plane == Plane::Color))
|
|
return false;
|
|
if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
|
|
{
|
|
uint64_t hash = 1469598103934665603ull;
|
|
for (size_t index = 0; index < host.size(); ++index)
|
|
hash = (hash ^ host.bytes[index]) * 1099511628211ull;
|
|
logf("frame %u upload plane=%u array=%p object=0x%llx bytes=%zu hash=%016llx", frame,
|
|
static_cast<unsigned int>(plane), image.array,
|
|
static_cast<unsigned long long>(image.object), host.size(),
|
|
static_cast<unsigned long long>(hash));
|
|
}
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_HOST;
|
|
copy.srcHost = host.bytes;
|
|
copy.srcPitch = host.rowBytes;
|
|
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.dstArray = image.array;
|
|
copy.WidthInBytes = host.rowBytes;
|
|
copy.Height = host.height;
|
|
const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
|
|
const int result = g.cu.memcpy2D(©);
|
|
if (timing != nullptr)
|
|
timing->upload[timingIndex] = profile_ms(copyStart, ProfileClock::now());
|
|
if (result != 0)
|
|
logf("cuMemcpy2D upload failed: %d", result);
|
|
if (result != 0)
|
|
return false;
|
|
if (GetEnvironmentVariableA("D4R_SHIM_SYNC_UPLOAD", nullptr, 0) != 0)
|
|
{
|
|
const int syncResult = g.cu.ctxSynchronize();
|
|
if (syncResult != 0)
|
|
logf("cuCtxSynchronize after upload failed: %d", syncResult);
|
|
return syncResult == 0;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static void log_output_hash(const HostPlane& host, uint32_t frame)
|
|
{
|
|
if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) == 0)
|
|
return;
|
|
uint64_t hash = 1469598103934665603ull;
|
|
for (size_t index = 0; index < host.size(); ++index)
|
|
hash = (hash ^ host.bytes[index]) * 1099511628211ull;
|
|
logf("frame %u output hash=%016llx", frame, static_cast<unsigned long long>(hash));
|
|
}
|
|
|
|
static std::atomic<uint32_t> g_captureFirst{0}; // first frame of a triggered capture
|
|
|
|
static bool dump_frame_selected(uint32_t frame, const char* startName, const char* countName, const char* everyName)
|
|
{
|
|
char trigger[MAX_PATH];
|
|
const DWORD triggerLength = GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH);
|
|
if (triggerLength > 0 && triggerLength < MAX_PATH)
|
|
{
|
|
uint32_t captured = g_captureFirst.load();
|
|
if (captured == 0 && GetFileAttributesA(trigger) != INVALID_FILE_ATTRIBUTES)
|
|
{
|
|
uint32_t expected = 0;
|
|
if (g_captureFirst.compare_exchange_strong(expected, frame))
|
|
logf("raw input/output capture triggered on frame %u", frame);
|
|
captured = g_captureFirst.load();
|
|
}
|
|
// D4R_SHIM_CAPTURE_COUNT=N keeps capturing N consecutive frames (replay sequences).
|
|
static const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
|
|
return captured != 0 && frame >= captured && frame - captured < count;
|
|
}
|
|
const uint32_t start = env_uint(startName, 1);
|
|
const uint32_t count = env_uint(countName, 1);
|
|
const uint32_t every = std::max(1u, env_uint(everyName, 1));
|
|
return frame >= start && count != 0 && (frame - start) / every < count && (frame - start) % every == 0;
|
|
}
|
|
|
|
static bool output_dump_path(uint32_t frame, char (&path)[MAX_PATH])
|
|
{
|
|
char directory[MAX_PATH];
|
|
const DWORD length = GetEnvironmentVariableA("D4R_SHIM_OUTPUT_DUMP_DIR", directory, MAX_PATH);
|
|
if (length == 0 || length >= MAX_PATH)
|
|
return false;
|
|
if (!dump_frame_selected(frame, "D4R_SHIM_OUTPUT_DUMP_START", "D4R_SHIM_OUTPUT_DUMP_COUNT",
|
|
"D4R_SHIM_OUTPUT_DUMP_EVERY"))
|
|
return false;
|
|
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u.rgba16f", directory, frame);
|
|
return written > 0 && written < MAX_PATH;
|
|
}
|
|
|
|
// Capture files. With D4R_SHIM_CAPTURE_MEMORY=1 they are kept in memory and
|
|
// written by a background thread once the triggered capture is complete, so a
|
|
// sequence can be recorded at the game's normal frame rate.
|
|
static std::mutex g_captureMutex;
|
|
static std::vector<std::pair<std::string, std::vector<uint8_t>>> g_captureFiles;
|
|
|
|
// Pinned staging for memory captures (D4R_SHIM_CAPTURE_POOL_MB, default 3072):
|
|
// device readbacks land in it directly, so a capture costs a few ms per frame.
|
|
struct CapturePool
|
|
{
|
|
uint8_t* base = nullptr;
|
|
size_t size = 0, used = 0;
|
|
bool tried = false;
|
|
};
|
|
static CapturePool g_capturePool;
|
|
static std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> g_capturePinned;
|
|
|
|
// Worker thread (a CUDA context is current). Returns nullptr when memory
|
|
// captures are off or the pool is exhausted; the caller then uses ordinary memory.
|
|
static uint8_t* capture_pinned(const char* path, size_t bytes)
|
|
{
|
|
if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 || g.cu.memAllocHost == nullptr)
|
|
return nullptr;
|
|
std::lock_guard<std::mutex> lock(g_captureMutex);
|
|
CapturePool& pool = g_capturePool;
|
|
if (!pool.tried)
|
|
{
|
|
pool.tried = true;
|
|
for (size_t mb = env_uint("D4R_SHIM_CAPTURE_POOL_MB", 3072); mb >= 256 && pool.base == nullptr; mb /= 2)
|
|
{
|
|
void* memory = nullptr;
|
|
if (g.cu.memAllocHost(&memory, mb << 20) == 0)
|
|
pool.base = static_cast<uint8_t*>(memory), pool.size = mb << 20;
|
|
}
|
|
logf("capture pool: %zu MB pinned", pool.size >> 20);
|
|
}
|
|
const size_t aligned = (bytes + 4095) & ~size_t(4095);
|
|
if (pool.base == nullptr || pool.used + aligned > pool.size)
|
|
return nullptr;
|
|
uint8_t* block = pool.base + pool.used;
|
|
pool.used += aligned;
|
|
g_capturePinned.emplace_back(path, std::make_pair(block, bytes));
|
|
return block;
|
|
}
|
|
|
|
static void write_capture_file(uint32_t frame, const char* what, const char* path, const void* data, size_t bytes)
|
|
{
|
|
static const bool memory = env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) != 0;
|
|
if (memory)
|
|
{
|
|
if (uint8_t* pinned = capture_pinned(path, bytes))
|
|
{
|
|
std::memcpy(pinned, data, bytes);
|
|
return;
|
|
}
|
|
const auto* first = static_cast<const uint8_t*>(data);
|
|
std::lock_guard<std::mutex> lock(g_captureMutex);
|
|
g_captureFiles.emplace_back(path, std::vector<uint8_t>(first, first + bytes));
|
|
return;
|
|
}
|
|
FILE* file = std::fopen(path, "wb");
|
|
if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
|
|
logf("frame %u %s dump failed: %s", frame, what, path);
|
|
else
|
|
logf("frame %u %s saved: %s", frame, what, path);
|
|
if (file != nullptr)
|
|
std::fclose(file);
|
|
}
|
|
|
|
// Called after each frame's output dump: once the last triggered frame is in,
|
|
// the buffered files go to disk.
|
|
static void flush_capture_if_complete(uint32_t frame)
|
|
{
|
|
char trigger[MAX_PATH];
|
|
if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 ||
|
|
GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH) == 0)
|
|
return;
|
|
const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
|
|
const uint32_t first = g_captureFirst.load();
|
|
if (first == 0 || frame != first + count - 1)
|
|
return;
|
|
std::vector<std::pair<std::string, std::vector<uint8_t>>> files;
|
|
std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> pinned;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_captureMutex);
|
|
files.swap(g_captureFiles);
|
|
pinned.swap(g_capturePinned);
|
|
}
|
|
logf("capture of frames %u-%u complete in memory; writing %zu files", first, frame, files.size() + pinned.size());
|
|
// The pinned pool is not reused, so its blocks stay valid for the writer.
|
|
std::thread([files = std::move(files), pinned = std::move(pinned), first, frame] {
|
|
size_t failed = 0;
|
|
auto write = [&failed](const std::string& path, const uint8_t* data, size_t bytes) {
|
|
FILE* file = std::fopen(path.c_str(), "wb");
|
|
if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
|
|
++failed;
|
|
if (file != nullptr)
|
|
std::fclose(file);
|
|
};
|
|
for (const auto& [path, block] : pinned)
|
|
write(path, block.first, block.second);
|
|
for (const auto& [path, bytes] : files)
|
|
write(path, bytes.data(), bytes.size());
|
|
logf("capture of frames %u-%u written (%zu files, %zu failed)", first, frame, files.size() + pinned.size(),
|
|
failed);
|
|
}).detach();
|
|
}
|
|
|
|
static void dump_output(const HostPlane& host, uint32_t frame, const char* path)
|
|
{
|
|
write_capture_file(frame, "raw DLSS output", path, host.bytes, host.size());
|
|
flush_capture_if_complete(frame);
|
|
}
|
|
|
|
static bool input_dump_directory(uint32_t frame, char (&directory)[MAX_PATH])
|
|
{
|
|
const DWORD length = GetEnvironmentVariableA("D4R_SHIM_INPUT_DUMP_DIR", directory, MAX_PATH);
|
|
return length != 0 && length < MAX_PATH &&
|
|
dump_frame_selected(frame, "D4R_SHIM_INPUT_DUMP_START", "D4R_SHIM_INPUT_DUMP_COUNT",
|
|
"D4R_SHIM_INPUT_DUMP_EVERY");
|
|
}
|
|
|
|
static void dump_input_planes(const HostPlane (&planes)[4], uint32_t frame, const FrameParams& p,
|
|
const char* directory)
|
|
{
|
|
constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
|
|
for (int index = 0; index < (planes[3].width != 0 ? 4 : 3); ++index)
|
|
{
|
|
const HostPlane& host = planes[index];
|
|
const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
|
|
if (bytes == nullptr || host.width == 0 || host.height == 0)
|
|
continue;
|
|
char path[MAX_PATH];
|
|
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
|
|
if (written > 0 && written < MAX_PATH)
|
|
write_capture_file(frame, names[index], path, bytes, host.size());
|
|
}
|
|
// The evaluation parameters, so the harness can replay the sequence.
|
|
char path[MAX_PATH];
|
|
const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-params.txt", directory, frame);
|
|
if (written <= 0 || written >= MAX_PATH)
|
|
return;
|
|
char text[1024];
|
|
const int length = std::snprintf(
|
|
text, sizeof(text),
|
|
"jitter %.9g %.9g\nmv_scale %.9g %.9g\nsharpness %.9g\npre_exposure %.9g\n"
|
|
"exposure_scale %.9g\nframe_time %.9g\nreset %d\ninvert %d %d\nrender %u %u\n"
|
|
"color_base %u %u\ndepth_base %u %u\nmv_base %u %u\noutput_base %u %u\nhas_exposure %d\n",
|
|
p.jitterX, p.jitterY, p.mvScaleX, p.mvScaleY, p.sharpness, p.preExposure, p.exposureScale, p.frameTime,
|
|
p.reset, p.invertX, p.invertY, p.renderWidth, p.renderHeight, p.colorBaseX, p.colorBaseY, p.depthBaseX,
|
|
p.depthBaseY, p.mvBaseX, p.mvBaseY, p.outputBaseX, p.outputBaseY, p.hasExposure ? 1 : 0);
|
|
if (length > 0)
|
|
write_capture_file(frame, "params", path, text, static_cast<size_t>(length));
|
|
}
|
|
|
|
// VRAM interop without D4R_SHIM_VRAM_VERIFY: reads the selected frame's input
|
|
// planes straight from the imported buffers DLSS is about to consume.
|
|
static void dump_vram_inputs(const InputSlot& slot, uint32_t frame, const FrameParams& p, int count)
|
|
{
|
|
char directory[MAX_PATH];
|
|
if (!input_dump_directory(frame, directory))
|
|
return;
|
|
HostPlane planes[4];
|
|
constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
|
|
bool allPinned = true;
|
|
for (int index = 0; index < count; ++index)
|
|
{
|
|
const HostPlane& geometry = slot.host[index];
|
|
HostPlane& plane = planes[index];
|
|
plane.width = geometry.width, plane.height = geometry.height, plane.rowBytes = geometry.rowBytes;
|
|
char path[MAX_PATH];
|
|
std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
|
|
plane.bytes = capture_pinned(path, geometry.size());
|
|
if (plane.bytes == nullptr)
|
|
{
|
|
allPinned = false;
|
|
plane.fallback.resize(geometry.size());
|
|
}
|
|
CudaMemcpy2D readback = {};
|
|
readback.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
readback.srcDevice = slot.vram[index].device;
|
|
readback.srcPitch = geometry.rowBytes;
|
|
readback.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readback.dstHost = plane.bytes != nullptr ? plane.bytes : plane.fallback.data();
|
|
readback.dstPitch = geometry.rowBytes;
|
|
readback.WidthInBytes = geometry.rowBytes;
|
|
readback.Height = geometry.height;
|
|
if (g.cu.memcpy2D(&readback) != 0)
|
|
{
|
|
logf("frame %u: VRAM input readback for the dump failed", frame);
|
|
return;
|
|
}
|
|
}
|
|
// Pinned planes are already registered for writing; only the rest (and the
|
|
// parameters) go through dump_input_planes.
|
|
for (HostPlane& plane : planes)
|
|
if (plane.bytes != nullptr)
|
|
plane.width = 0;
|
|
(void)allPinned;
|
|
dump_input_planes(planes, frame, p, directory);
|
|
}
|
|
|
|
// Worker stage: downloads the DLSS output into host memory.
|
|
static bool download_output(Feature& feature, HostPlane& host, uint32_t frame, FrameTiming* timing)
|
|
{
|
|
reserve_host(host, static_cast<size_t>(feature.output.width) * 8 * feature.output.height);
|
|
host.width = feature.output.width;
|
|
host.height = feature.output.height;
|
|
host.rowBytes = static_cast<size_t>(feature.output.width) * 8;
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.srcArray = feature.output.array;
|
|
copy.dstMemoryType = CUDA_MEMORY_HOST;
|
|
copy.dstHost = host.bytes;
|
|
copy.dstPitch = host.rowBytes;
|
|
copy.WidthInBytes = host.rowBytes;
|
|
copy.Height = host.height;
|
|
const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
|
|
const int result = g.cu.memcpy2D(©);
|
|
if (timing != nullptr)
|
|
timing->download = profile_ms(copyStart, ProfileClock::now());
|
|
if (result != 0)
|
|
{
|
|
logf("cuMemcpy2D download failed: %d", result);
|
|
return false;
|
|
}
|
|
log_output_hash(host, frame);
|
|
char dumpPath[MAX_PATH];
|
|
if (output_dump_path(frame, dumpPath))
|
|
dump_output(host, frame, dumpPath);
|
|
return true;
|
|
}
|
|
|
|
// Finish stage: converts a downloaded result into an upload staging buffer.
|
|
static void write_output(Staging& staging, const HostPlane& host, FrameTiming* timing)
|
|
{
|
|
const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
|
|
uint8_t* destination = staging.mapped + staging.layout.Offset;
|
|
const size_t destinationPitch = staging.layout.Footprint.RowPitch;
|
|
const UINT width = staging.width < host.width ? staging.width : host.width;
|
|
const UINT height = staging.height < host.height ? staging.height : host.height;
|
|
const uint8_t* rows = host.bytes;
|
|
if (staging.format == DXGI_FORMAT_R16G16B16A16_FLOAT || staging.format == DXGI_FORMAT_R16G16B16A16_TYPELESS)
|
|
copy_rows(g_outputRows, destination, destinationPitch, rows, host.rowBytes, static_cast<size_t>(width) * 8,
|
|
height);
|
|
else
|
|
g_outputRows.run(height, host.rowBytes * height, [&](UINT begin, UINT end) {
|
|
for (UINT y = begin; y < end; ++y)
|
|
convert_row_out(staging.format, reinterpret_cast<const uint16_t*>(rows + host.rowBytes * y),
|
|
destination + y * destinationPitch, width);
|
|
});
|
|
if (timing != nullptr)
|
|
timing->outputConvert = profile_ms(conversionStart, ProfileClock::now());
|
|
}
|
|
|
|
static void set_evaluation_parameters(Feature& feature, const FrameParams& p)
|
|
{
|
|
void* params = feature.cudaParams;
|
|
feature.colorHandle = feature.color.object;
|
|
feature.depthHandle = feature.depth.object;
|
|
feature.motionHandle = feature.motion.object;
|
|
feature.exposureHandle = feature.exposure.object;
|
|
feature.outputHandle = feature.output.object;
|
|
// CUDA resource parameters point to variables holding the object handles.
|
|
d4r_ngx_set_void(params, "Color", &feature.colorHandle);
|
|
d4r_ngx_set_void(params, "Depth", &feature.depthHandle);
|
|
d4r_ngx_set_void(params, "MotionVectors", &feature.motionHandle);
|
|
d4r_ngx_set_void(params, "Output", &feature.outputHandle);
|
|
d4r_ngx_set_void(params, "ExposureTexture", p.hasExposure ? &feature.exposureHandle : nullptr);
|
|
d4r_ngx_set_void(params, "TransparencyMask", nullptr);
|
|
d4r_ngx_set_void(params, "DLSS.Input.Bias.Current.Color.Mask", nullptr);
|
|
d4r_ngx_set_float(params, "Jitter.Offset.X", p.jitterX);
|
|
d4r_ngx_set_float(params, "Jitter.Offset.Y", p.jitterY);
|
|
d4r_ngx_set_float(params, "MV.Scale.X", p.mvScaleX);
|
|
d4r_ngx_set_float(params, "MV.Scale.Y", p.mvScaleY);
|
|
d4r_ngx_set_float(params, "Sharpness", p.sharpness);
|
|
// The DLSS library stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output in this setup; the SDK's
|
|
// Disable.Watermark parameter turns that off. D4R_SHIM_WATERMARK=1 keeps it.
|
|
static const bool watermark = env_uint("D4R_SHIM_WATERMARK", 0) != 0;
|
|
d4r_ngx_set_int(params, "Disable.Watermark", watermark ? 0 : 1);
|
|
d4r_ngx_set_int(params, "Reset", p.reset);
|
|
d4r_ngx_set_float(params, "DLSS.Pre.Exposure", p.preExposure);
|
|
d4r_ngx_set_float(params, "DLSS.Exposure.Scale", p.exposureScale);
|
|
d4r_ngx_set_float(params, "FrameTimeDeltaInMsec", p.frameTime);
|
|
d4r_ngx_set_int(params, "DLSS.Indicator.Invert.X.Axis", p.invertX);
|
|
d4r_ngx_set_int(params, "DLSS.Indicator.Invert.Y.Axis", p.invertY);
|
|
d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Width", p.renderWidth);
|
|
d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Height", p.renderHeight);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.X", p.colorBaseX);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.Y", p.colorBaseY);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.X", p.depthBaseX);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.Y", p.depthBaseY);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.X", p.mvBaseX);
|
|
d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.Y", p.mvBaseY);
|
|
d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.X", p.outputBaseX);
|
|
d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.Y", p.outputBaseY);
|
|
if (feature.scratch != 0)
|
|
{
|
|
d4r_ngx_set_void(params, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature.scratch)));
|
|
d4r_ngx_set_ull(params, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
|
|
}
|
|
}
|
|
|
|
// DLSS evaluations run as a three-stage pipeline so the CPU copies of one
|
|
// frame overlap the GPU work of another:
|
|
// prepare_inputs (prep thread) waits for the game's input copies and
|
|
// converts the readback staging into host planes;
|
|
// run_evaluation (CUDA worker) uploads them, evaluates, downloads the output;
|
|
// publish_output (finish thread) writes it into an upload staging buffer and
|
|
// makes it the result evaluate() presents.
|
|
// Every NGX-core and CUDA call stays on the worker. A stage only reuses a
|
|
// buffer once the next stage has released it (hostBusy / outputHostBusy).
|
|
static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing);
|
|
static int claim_output_slot(Feature* feature);
|
|
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
|
|
const FrameTiming& timing, ProfileClock::time_point start);
|
|
|
|
// Spins until `flag` clears; false after two seconds.
|
|
static bool wait_for_release(const std::atomic<bool>& flag)
|
|
{
|
|
const auto start = std::chrono::steady_clock::now();
|
|
while (flag.load())
|
|
{
|
|
if (std::chrono::steady_clock::now() - start > std::chrono::seconds(2))
|
|
return false;
|
|
d4r_sleep_us(100);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static void release_split_frame(Feature* feature, uint32_t frame);
|
|
|
|
// Every queued frame ends in exactly one call of this (published or dropped).
|
|
static void frame_retired(Feature* feature, uint32_t frame)
|
|
{
|
|
if (feature->split)
|
|
release_split_frame(feature, frame);
|
|
feature->inFlight.fetch_sub(1);
|
|
}
|
|
|
|
// Overlapped staging, worker thread only. Uploads the staged planes with async
|
|
// copies into linear buffers on the "up" stream, then moves them into the CUDA
|
|
// arrays device-to-device (on the null stream, ahead of NGX's kernels).
|
|
static bool upload_inputs_async(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
|
|
FrameTiming* timing)
|
|
{
|
|
CopyStreams* streams = copy_streams();
|
|
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
|
|
CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
|
|
const int count = hasExposure ? 4 : 3;
|
|
const auto start = ProfileClock::now();
|
|
for (int index = 0; index < count; ++index)
|
|
{
|
|
const HostPlane& host = slot.host[index];
|
|
if (!ensure_cuda_image(*images[index], host.width, host.height, plane_format(planes[index]),
|
|
plane_channels(planes[index]), false, planes[index] == Plane::Color) ||
|
|
!ensure_linear(feature.planeLinear[index], feature.planeLinearBytes[index], host.size()))
|
|
return false;
|
|
if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
|
|
{
|
|
uint64_t hash = 1469598103934665603ull;
|
|
for (size_t byte = 0; byte < host.size(); ++byte)
|
|
hash = (hash ^ host.bytes[byte]) * 1099511628211ull;
|
|
logf("frame %u upload plane=%d bytes=%zu hash=%016llx (async)", frame, index, host.size(),
|
|
static_cast<unsigned long long>(hash));
|
|
}
|
|
const auto issue = ProfileClock::now();
|
|
const int result = g.cu.memcpyHtoDAsync(feature.planeLinear[index], host.bytes, host.size(), streams->up);
|
|
if (timing != nullptr)
|
|
timing->upload[index] = profile_ms(issue, ProfileClock::now());
|
|
if (result != 0)
|
|
{
|
|
logf("cuMemcpyHtoDAsync failed: %d", result);
|
|
return false;
|
|
}
|
|
}
|
|
int result = g.cu.streamSynchronize(streams->up);
|
|
for (int index = 0; index < count && result == 0; ++index)
|
|
{
|
|
const HostPlane& host = slot.host[index];
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
copy.srcDevice = feature.planeLinear[index];
|
|
copy.srcPitch = host.rowBytes;
|
|
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.dstArray = images[index]->array;
|
|
copy.WidthInBytes = host.rowBytes;
|
|
copy.Height = host.height;
|
|
result = copy_2d(copy);
|
|
}
|
|
if (timing != nullptr)
|
|
timing->h2dTotal = profile_ms(start, ProfileClock::now());
|
|
if (result != 0)
|
|
logf("staged upload failed: %d", result);
|
|
return result == 0;
|
|
}
|
|
|
|
// Starts the output download: array -> linear buffer, then an async copy on
|
|
// the "down" stream that the next frame's uploads overlap.
|
|
static bool start_download(Feature& feature, HostPlane& host, FrameTiming* timing)
|
|
{
|
|
const size_t rowBytes = static_cast<size_t>(feature.output.width) * 8;
|
|
const size_t bytes = rowBytes * feature.output.height;
|
|
reserve_host(host, bytes);
|
|
host.width = feature.output.width;
|
|
host.height = feature.output.height;
|
|
host.rowBytes = rowBytes;
|
|
if (!ensure_linear(feature.outputLinear, feature.outputLinearBytes, bytes))
|
|
return false;
|
|
const auto start = ProfileClock::now();
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.srcArray = feature.output.array;
|
|
copy.dstMemoryType = CUDA_MEMORY_DEVICE;
|
|
copy.dstDevice = feature.outputLinear;
|
|
copy.dstPitch = rowBytes;
|
|
copy.WidthInBytes = rowBytes;
|
|
copy.Height = host.height;
|
|
int result = feature.outputRedirected ? 0 : copy_2d(copy);
|
|
// The null-stream copy must land before the non-blocking stream reads it.
|
|
if (result == 0)
|
|
result = g.cu.ctxSynchronize();
|
|
if (result == 0)
|
|
result = g.cu.memcpyDtoHAsync(host.bytes, feature.outputLinear, bytes, copy_streams()->down);
|
|
if (timing != nullptr)
|
|
timing->d2hIssue = profile_ms(start, ProfileClock::now());
|
|
if (result != 0)
|
|
logf("staged download failed: %d", result);
|
|
return result == 0;
|
|
}
|
|
|
|
// A result whose download is still in flight so the next frame's uploads can
|
|
// overlap it. Worker thread only.
|
|
struct PendingDownload
|
|
{
|
|
Feature* feature = nullptr;
|
|
int hostIndex = 0;
|
|
uint32_t frame = 0;
|
|
FrameParams params;
|
|
FrameTiming timing;
|
|
};
|
|
static PendingDownload g_pendingDownload;
|
|
|
|
static void post_publish(Feature* feature, int hostIndex, uint32_t frame, const FrameParams& params,
|
|
FrameTiming timing)
|
|
{
|
|
if (timing.enabled)
|
|
timing.finishQueued = ProfileClock::now();
|
|
g.finish.post([feature, hostIndex, frame, params, timing] {
|
|
publish_output(feature, hostIndex, frame, params, timing);
|
|
});
|
|
}
|
|
|
|
// Waits for the in-flight download, if any, and hands it to publish_output.
|
|
// `inline_completion` marks the case without a next frame to overlap: the wait
|
|
// then belongs to this frame's worker time.
|
|
static void complete_pending_download(bool inline_completion = false)
|
|
{
|
|
PendingDownload pending = g_pendingDownload;
|
|
if (pending.feature == nullptr)
|
|
return;
|
|
g_pendingDownload = {};
|
|
const auto waitStart = ProfileClock::now();
|
|
const int result = g.cu.streamSynchronize(copy_streams()->down);
|
|
if (pending.timing.enabled)
|
|
{
|
|
pending.timing.d2hWait = profile_ms(waitStart, ProfileClock::now());
|
|
pending.timing.download = pending.timing.d2hIssue + pending.timing.d2hWait;
|
|
if (inline_completion)
|
|
pending.timing.worker += pending.timing.d2hWait;
|
|
}
|
|
if (result != 0)
|
|
{
|
|
logf("frame %u: output download failed: %d", pending.frame, result);
|
|
pending.feature->outputHostBusy[pending.hostIndex] = false;
|
|
frame_retired(pending.feature, pending.frame);
|
|
return;
|
|
}
|
|
log_output_hash(pending.feature->outputHost[pending.hostIndex], pending.frame);
|
|
char dumpPath[MAX_PATH];
|
|
if (output_dump_path(pending.frame, dumpPath))
|
|
dump_output(pending.feature->outputHost[pending.hostIndex], pending.frame, dumpPath);
|
|
post_publish(pending.feature, pending.hostIndex, pending.frame, pending.params, pending.timing);
|
|
}
|
|
|
|
// Linear inputs, worker thread: pitch-linear texture objects on the slot's interop buffers (recreated
|
|
// when a buffer or its geometry changes), reported to NGX as arrays by the bridge.
|
|
static bool ensure_linear_inputs(InputSlot& slot, bool hasExposure)
|
|
{
|
|
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
|
|
const int count = hasExposure ? 4 : 3;
|
|
for (int index = 0; index < count; ++index)
|
|
{
|
|
const HostPlane& geometry = slot.host[index];
|
|
const VramBuffer& buffer = slot.vram[index];
|
|
if (slot.linearTexture[index] != 0 && slot.linearPointer[index] == buffer.device &&
|
|
slot.linearWidth[index] == geometry.width && slot.linearHeight[index] == geometry.height &&
|
|
slot.linearPitch[index] == geometry.rowBytes)
|
|
continue;
|
|
if (slot.linearTexture[index] != 0)
|
|
g.cu.texObjectDestroy(slot.linearTexture[index]);
|
|
slot.linearTexture[index] = 0;
|
|
struct Pitch2D
|
|
{
|
|
CudaDevicePtr pointer;
|
|
uint32_t format, channels;
|
|
size_t width, height, pitch;
|
|
} pitch2D = {buffer.device, plane_format(planes[index]), plane_channels(planes[index]), geometry.width,
|
|
geometry.height, geometry.rowBytes};
|
|
static_assert(sizeof(Pitch2D) == 40, "CUDA_RESOURCE_DESC pitch2D");
|
|
CudaResourceDesc resource = {};
|
|
resource.resType = 3; // CU_RESOURCE_TYPE_PITCH2D
|
|
memcpy(&resource.res, &pitch2D, sizeof(pitch2D));
|
|
const CudaTextureDesc sampler = input_sampler(geometry.width, pitch2D.channels, planes[index] == Plane::Color);
|
|
CudaObject object = 0;
|
|
int result = g.cu.texObjectCreate(&object, &resource, &sampler, nullptr);
|
|
if (result == 0)
|
|
result = g.cu.registerLinearTexture(object, geometry.width, geometry.height, pitch2D.format, pitch2D.channels);
|
|
if (result != 0)
|
|
{
|
|
logf("linear input texture %d (%ux%u, pitch %zu) failed: %d", index, geometry.width, geometry.height,
|
|
geometry.rowBytes, result);
|
|
if (object != 0)
|
|
g.cu.texObjectDestroy(object);
|
|
return false;
|
|
}
|
|
slot.linearTexture[index] = object;
|
|
slot.linearPointer[index] = buffer.device;
|
|
slot.linearWidth[index] = geometry.width;
|
|
slot.linearHeight[index] = geometry.height;
|
|
slot.linearPitch[index] = geometry.rowBytes;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// VRAM interop, worker thread: moves the planes from the imported buffers
|
|
// Opt-in thin-feature motion coverage. Keep the original imported velocity buffer
|
|
// immutable; the new texture is owned by this input slot until its GPU work retires.
|
|
static CudaObject prepare_dilated_motion(Feature& feature, InputSlot& slot, const FrameParams& p)
|
|
{
|
|
const unsigned radius = env_uint("D4R_MOTION_DILATION", 0);
|
|
if (radius == 0)
|
|
return 0;
|
|
const auto& depth = slot.host[1];
|
|
const auto& motion = slot.host[2];
|
|
D4rMotionDilationParams args = {slot.vram[1].device, slot.vram[2].device, slot.dilatedMotion,
|
|
depth.rowBytes, motion.rowBytes, p.renderWidth, p.renderHeight, motion.width, motion.height,
|
|
p.jitterX, p.jitterY, radius, (feature.flags & 8u) != 0};
|
|
const bool motionGeometry = d4r_motion_dilation_geometry(&args, feature.flags,
|
|
feature.outWidth, feature.outHeight);
|
|
const bool supported = radius <= 2 && p.vram && feature.linearInputs && g.cu.dilateMotion != nullptr &&
|
|
motionGeometry && p.colorBaseX == 0 && p.colorBaseY == 0 && p.depthBaseX == 0 &&
|
|
p.depthBaseY == 0 && p.mvBaseX == 0 && p.mvBaseY == 0 && p.outputBaseX == 0 && p.outputBaseY == 0 &&
|
|
p.renderWidth > 0 && p.renderHeight > 0 && std::isfinite(p.jitterX) && std::isfinite(p.jitterY) &&
|
|
p.renderWidth <= depth.width && p.renderHeight <= depth.height;
|
|
if (!supported)
|
|
{
|
|
if (!feature.dilationReported)
|
|
logf("motion-vector dilation unavailable for this feature; original motion vectors retained");
|
|
feature.dilationReported = true;
|
|
return 0;
|
|
}
|
|
const size_t bytes = motion.size();
|
|
const bool recreate = slot.dilatedBytes < bytes || slot.dilatedWidth != motion.width ||
|
|
slot.dilatedHeight != motion.height || slot.dilatedPitch != motion.rowBytes;
|
|
if (recreate)
|
|
{
|
|
if (slot.dilatedTexture != 0)
|
|
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
|
|
if (slot.dilatedMotion != 0)
|
|
g.cu.memFree(slot.dilatedMotion), slot.dilatedMotion = 0;
|
|
slot.dilatedBytes = 0;
|
|
if (g.cu.memAlloc(&slot.dilatedMotion, bytes) != 0)
|
|
return 0;
|
|
slot.dilatedBytes = bytes;
|
|
struct Pitch2D { CudaDevicePtr pointer; uint32_t format, channels; size_t width, height, pitch; };
|
|
const Pitch2D pitch = {slot.dilatedMotion, plane_format(Plane::Motion), 2, motion.width, motion.height, motion.rowBytes};
|
|
CudaResourceDesc resource = {};
|
|
resource.resType = 3;
|
|
static_assert(sizeof(pitch) == 40);
|
|
std::memcpy(&resource.res, &pitch, sizeof(pitch));
|
|
const CudaTextureDesc sampler = input_sampler(motion.width, 2, false);
|
|
int result = g.cu.texObjectCreate(&slot.dilatedTexture, &resource, &sampler, nullptr);
|
|
if (result == 0)
|
|
result = g.cu.registerLinearTexture(slot.dilatedTexture, motion.width, motion.height, pitch.format, 2);
|
|
if (result != 0)
|
|
{
|
|
if (slot.dilatedTexture != 0)
|
|
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
|
|
return 0;
|
|
}
|
|
slot.dilatedWidth = motion.width, slot.dilatedHeight = motion.height, slot.dilatedPitch = motion.rowBytes;
|
|
}
|
|
if (slot.dilatedTexture == 0)
|
|
return 0;
|
|
args.output = slot.dilatedMotion;
|
|
// Preserve allocation padding, which DLSS may sample at the active edge.
|
|
// The dilation kernel only writes the active render grid for low-res MVs.
|
|
if (args.motion_width != motion.width || args.motion_height != motion.height)
|
|
{
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = copy.dstMemoryType = CUDA_MEMORY_DEVICE;
|
|
copy.srcDevice = args.motion;
|
|
copy.dstDevice = args.output;
|
|
copy.srcPitch = copy.dstPitch = motion.rowBytes;
|
|
copy.WidthInBytes = motion.rowBytes;
|
|
copy.Height = motion.height;
|
|
if (copy_2d(copy) != 0)
|
|
return 0;
|
|
}
|
|
const int result = g.cu.dilateMotion(&args);
|
|
if (!feature.dilationReported)
|
|
logf("motion-vector dilation %s: radius=%u render pixels, GPU status=%d, %s active=%ux%u allocation=%ux%u",
|
|
result == 0 ? "enabled" : "unavailable", radius, result,
|
|
(feature.flags & 2u) != 0 ? "low-resolution" : "display-resolution",
|
|
args.motion_width, args.motion_height, motion.width, motion.height);
|
|
feature.dilationReported = true;
|
|
return result == 0 ? slot.dilatedTexture : 0;
|
|
}
|
|
|
|
// VRAM interop, worker thread: moves the planes from the imported buffers
|
|
// into the CUDA arrays on the GPU. With D4R_SHIM_VRAM_VERIFY also compares
|
|
// them with the host-staged planes of the same frame.
|
|
static bool upload_inputs_vram(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
|
|
FrameTiming* timing)
|
|
{
|
|
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
|
|
CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
|
|
const int count = hasExposure ? 4 : 3;
|
|
const bool verify = env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
|
|
const auto start = ProfileClock::now();
|
|
int result = 0;
|
|
for (int index = 0; index < count && result == 0; ++index)
|
|
{
|
|
const HostPlane& geometry = slot.host[index];
|
|
if (!ensure_cuda_image(*images[index], geometry.width, geometry.height, plane_format(planes[index]),
|
|
plane_channels(planes[index]), false, planes[index] == Plane::Color))
|
|
return false;
|
|
if (verify)
|
|
{
|
|
std::vector<uint8_t> vram(geometry.size());
|
|
CudaMemcpy2D readback = {};
|
|
readback.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
readback.srcDevice = slot.vram[index].device;
|
|
readback.srcPitch = geometry.rowBytes;
|
|
readback.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readback.dstHost = vram.data();
|
|
readback.dstPitch = geometry.rowBytes;
|
|
readback.WidthInBytes = geometry.rowBytes;
|
|
readback.Height = geometry.height;
|
|
size_t differing = 0;
|
|
if (g.cu.memcpy2D(&readback) == 0)
|
|
for (size_t byte = 0; byte < vram.size(); ++byte)
|
|
differing += vram[byte] != geometry.bytes[byte];
|
|
if (differing != 0 || frame <= 3 || frame % 120 == 0)
|
|
logf("frame %u VRAM verify plane=%d: %zu of %zu bytes differ from host staging", frame, index,
|
|
differing, vram.size());
|
|
}
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
copy.srcDevice = slot.vram[index].device;
|
|
copy.srcPitch = geometry.rowBytes;
|
|
copy.dstMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.dstArray = images[index]->array;
|
|
copy.WidthInBytes = geometry.rowBytes;
|
|
copy.Height = geometry.height;
|
|
result = copy_2d(copy);
|
|
// D4R_SHIM_VRAM_ARRAY_VERIFY=1: read the CUDA array DLSS samples back
|
|
// and compare it with the imported buffer it was filled from.
|
|
static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
|
|
if (arrayVerify && result == 0 && g.cu.ctxSynchronize() == 0)
|
|
{
|
|
std::vector<uint8_t> fromBuffer(geometry.size()), fromArray(geometry.size());
|
|
CudaMemcpy2D readBuffer = {};
|
|
readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
readBuffer.srcDevice = slot.vram[index].device;
|
|
readBuffer.srcPitch = geometry.rowBytes;
|
|
readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readBuffer.dstHost = fromBuffer.data();
|
|
readBuffer.dstPitch = geometry.rowBytes;
|
|
readBuffer.WidthInBytes = geometry.rowBytes;
|
|
readBuffer.Height = geometry.height;
|
|
CudaMemcpy2D readArray = {};
|
|
readArray.srcMemoryType = CUDA_MEMORY_ARRAY;
|
|
readArray.srcArray = images[index]->array;
|
|
readArray.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readArray.dstHost = fromArray.data();
|
|
readArray.dstPitch = geometry.rowBytes;
|
|
readArray.WidthInBytes = geometry.rowBytes;
|
|
readArray.Height = geometry.height;
|
|
if (g.cu.memcpy2D(&readBuffer) == 0 && g.cu.memcpy2D(&readArray) == 0)
|
|
{
|
|
size_t differing = 0, firstRow = geometry.height;
|
|
for (size_t byte = 0; byte < fromArray.size(); ++byte)
|
|
if (fromArray[byte] != fromBuffer[byte])
|
|
{
|
|
if (differing++ == 0)
|
|
firstRow = byte / geometry.rowBytes;
|
|
}
|
|
if (differing != 0 || frame <= 3 || frame % 120 == 0)
|
|
logf("frame %u VRAM array verify plane=%d: %zu of %zu bytes differ (first row %zu)", frame, index,
|
|
differing, fromArray.size(), firstRow);
|
|
}
|
|
}
|
|
}
|
|
if (timing != nullptr)
|
|
timing->h2dTotal = profile_ms(start, ProfileClock::now());
|
|
if (result != 0)
|
|
logf("VRAM input copy failed: %d", result);
|
|
return result == 0;
|
|
}
|
|
|
|
// Only called on the CUDA worker. A completion event covers queued work on
|
|
// the default stream; do not rely on HIP's blocking flag to release the CPU.
|
|
static int synchronize_default_stream(Feature& feature, bool* querySleep = nullptr)
|
|
{
|
|
if (feature.outputReadyEvent != nullptr)
|
|
{
|
|
const int recorded = g.cu.eventRecord(feature.outputReadyEvent, nullptr);
|
|
if (recorded == 0)
|
|
{
|
|
const int result = d4r_wait_event(
|
|
[&] { return g.cu.eventQuery(feature.outputReadyEvent); },
|
|
[] { d4r_sleep_us(200); });
|
|
if (result == 0)
|
|
{
|
|
if (querySleep != nullptr) *querySleep = true;
|
|
// The event is already complete; this also flushes bridge kernel profiles.
|
|
return g.cu.eventSynchronize(feature.outputReadyEvent);
|
|
}
|
|
logf("output event query failed (%d); using context synchronization", result);
|
|
}
|
|
else
|
|
logf("output event record failed (%d); using context synchronization", recorded);
|
|
}
|
|
return g.cu.ctxSynchronize();
|
|
}
|
|
|
|
// VRAM interop, worker thread: copies the finished result into a free output
|
|
// slot's imported buffer and publishes it; evaluate() copies it on to the game.
|
|
static void publish_vram(Feature* feature, uint32_t frame, const FrameParams& params, FrameTiming& timing,
|
|
ProfileClock::time_point workerStart)
|
|
{
|
|
const auto start = ProfileClock::now();
|
|
// A split frame's second half reads slot frame % kOutputSlots; its previous
|
|
// reader, frame - kOutputSlots, ran before this frame's marker.
|
|
const int target = params.split ? static_cast<int>(frame % kOutputSlots) : claim_output_slot(feature);
|
|
if (target < 0)
|
|
{
|
|
logf("frame %u: no free output slot; dropping result", frame);
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
const VramBuffer& buffer = feature->outputs[target].vram;
|
|
const size_t rowBytes = static_cast<size_t>(feature->output.width) * 8;
|
|
CudaMemcpy2D copy = {};
|
|
copy.srcMemoryType = CUDA_MEMORY_ARRAY;
|
|
copy.srcArray = feature->output.array;
|
|
copy.dstMemoryType = CUDA_MEMORY_DEVICE;
|
|
copy.dstDevice = buffer.device;
|
|
copy.dstPitch = rowBytes;
|
|
copy.WidthInBytes = rowBytes;
|
|
copy.Height = feature->output.height;
|
|
int result = rowBytes * feature->output.height > buffer.bytes ? -1 : feature->outputRedirected ? 0 : copy_2d(copy);
|
|
if (result == 0)
|
|
{
|
|
const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
const double cpuStart = timing.enabled ? profile_thread_cpu_ms() : -1.0;
|
|
result = synchronize_default_stream(*feature, &timing.outputSyncBlocking);
|
|
if (timing.enabled)
|
|
{
|
|
timing.outputSyncWall = profile_ms(syncStart, ProfileClock::now());
|
|
const double cpuEnd = profile_thread_cpu_ms();
|
|
if (cpuStart >= 0.0 && cpuEnd >= 0.0)
|
|
timing.outputSyncCpu = std::max(0.0, cpuEnd - cpuStart);
|
|
}
|
|
}
|
|
if (result == 0 && timing.gpuEventsRecorded)
|
|
{
|
|
float gpuEvalMs = -1.0f;
|
|
if (g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) == 0)
|
|
timing.gpuEval = gpuEvalMs;
|
|
}
|
|
if (result != 0)
|
|
{
|
|
logf("frame %u: VRAM output copy failed: %d", frame, result);
|
|
std::lock_guard<std::mutex> lock(feature->outputMutex);
|
|
feature->outputs[target].lastReadFrame = 0;
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
|
|
if (arrayVerify)
|
|
{
|
|
// The result as DLSS left it in its array vs the buffer the game copies from.
|
|
std::vector<uint8_t> fromArray(rowBytes * feature->output.height), fromBuffer(fromArray.size());
|
|
CudaMemcpy2D readArray = copy;
|
|
readArray.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readArray.dstHost = fromArray.data();
|
|
CudaMemcpy2D readBuffer = {};
|
|
readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
|
|
readBuffer.srcDevice = buffer.device;
|
|
readBuffer.srcPitch = rowBytes;
|
|
readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
|
|
readBuffer.dstHost = fromBuffer.data();
|
|
readBuffer.dstPitch = rowBytes;
|
|
readBuffer.WidthInBytes = rowBytes;
|
|
readBuffer.Height = feature->output.height;
|
|
if (g.cu.memcpy2D(&readArray) == 0 && g.cu.memcpy2D(&readBuffer) == 0)
|
|
{
|
|
size_t differing = 0;
|
|
for (size_t byte = 0; byte < fromArray.size(); ++byte)
|
|
differing += fromArray[byte] != fromBuffer[byte];
|
|
if (differing != 0 || frame <= 3 || frame % 120 == 0)
|
|
logf("frame %u VRAM output verify: %zu of %zu bytes differ between result array and buffer", frame,
|
|
differing, fromArray.size());
|
|
}
|
|
}
|
|
char dumpPath[MAX_PATH];
|
|
const bool saveRaw = output_dump_path(frame, dumpPath);
|
|
if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) != 0 || saveRaw)
|
|
{
|
|
HostPlane host;
|
|
uint8_t* pinned = saveRaw ? capture_pinned(dumpPath, rowBytes * feature->output.height) : nullptr;
|
|
if (pinned == nullptr)
|
|
host.fallback.resize(rowBytes * feature->output.height);
|
|
host.bytes = pinned != nullptr ? pinned : host.fallback.data();
|
|
host.width = feature->output.width;
|
|
host.height = feature->output.height;
|
|
host.rowBytes = rowBytes;
|
|
copy.dstMemoryType = CUDA_MEMORY_HOST;
|
|
copy.dstHost = host.bytes;
|
|
if (g.cu.memcpy2D(©) == 0)
|
|
{
|
|
log_output_hash(host, frame);
|
|
if (saveRaw && pinned != nullptr)
|
|
flush_capture_if_complete(frame); // already registered for writing
|
|
else if (saveRaw)
|
|
dump_output(host, frame, dumpPath);
|
|
}
|
|
else if (saveRaw)
|
|
logf("frame %u raw DLSS output readback failed", frame);
|
|
}
|
|
if (timing.enabled)
|
|
{
|
|
timing.d2hIssue = profile_ms(start, ProfileClock::now());
|
|
timing.download = timing.d2hIssue;
|
|
timing.worker = profile_ms(workerStart, ProfileClock::now());
|
|
}
|
|
finish_publish(feature, target, frame, params, timing, start);
|
|
}
|
|
|
|
// Worker. Streamline can create the feature at one quality's render size while the game still renders at
|
|
// another's (PRAGMATA starts with an Ultra Performance 854x480 frame on a Performance 1280x720 feature), and NGX
|
|
// rejects a subrect outside the feature's dynamic range on every frame until the game recreates it: a black
|
|
// screen. Recreate the NGX feature at the size actually rendered; the game's handle and output stay the same.
|
|
static bool recreate_for_render_size(Feature& feature, FrameParams& params, uint32_t frame)
|
|
{
|
|
if ((params.renderWidth == feature.cudaWidth && params.renderHeight == feature.cudaHeight) ||
|
|
(params.renderWidth == feature.refusedWidth && params.renderHeight == feature.refusedHeight))
|
|
return false;
|
|
logf("frame %u: NGX rejected render %ux%u on a %ux%u feature; recreating it at %ux%u", frame, params.renderWidth,
|
|
params.renderHeight, feature.cudaWidth, feature.cudaHeight, params.renderWidth, params.renderHeight);
|
|
// earlier frames' kernels may still be running on the old feature
|
|
synchronize_default_stream(feature);
|
|
if (feature.cudaHandle != nullptr)
|
|
g.ngx.releaseFeature(feature.cudaHandle);
|
|
feature.cudaHandle = nullptr;
|
|
void* p = feature.cudaParams;
|
|
d4r_ngx_set_uint(p, "Width", params.renderWidth);
|
|
d4r_ngx_set_uint(p, "Height", params.renderHeight);
|
|
NgxResult created = g.ngx.createFeature(NGX_FEATURE_SUPER_SAMPLING, p, &feature.cudaHandle);
|
|
unsigned int width = params.renderWidth, height = params.renderHeight;
|
|
if (created != NGX_SUCCESS)
|
|
{
|
|
logf("frame %u: recreating the NGX feature at %ux%u failed (0x%08x); restoring %ux%u", frame, width, height,
|
|
created, feature.cudaWidth, feature.cudaHeight);
|
|
width = feature.cudaWidth;
|
|
height = feature.cudaHeight;
|
|
d4r_ngx_set_uint(p, "Width", width);
|
|
d4r_ngx_set_uint(p, "Height", height);
|
|
if (width != 0 && height != 0)
|
|
created = g.ngx.createFeature(NGX_FEATURE_SUPER_SAMPLING, p, &feature.cudaHandle);
|
|
feature.refusedWidth = created == NGX_SUCCESS ? params.renderWidth : 0;
|
|
feature.refusedHeight = created == NGX_SUCCESS ? params.renderHeight : 0;
|
|
if (created != NGX_SUCCESS)
|
|
{
|
|
logf("frame %u: restoring the NGX feature failed (0x%08x)", frame, created);
|
|
feature.cudaHandle = nullptr;
|
|
width = height = 0;
|
|
}
|
|
}
|
|
// a new feature has no history
|
|
feature.cudaWidth = width;
|
|
feature.cudaHeight = height;
|
|
params.reset = 1;
|
|
d4r_ngx_set_int(p, "Reset", 1);
|
|
return feature.cudaHandle != nullptr && width == params.renderWidth && height == params.renderHeight;
|
|
}
|
|
|
|
static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
|
|
{
|
|
InputSlot& slot = feature->inputs[slotIndex];
|
|
const auto start = ProfileClock::now();
|
|
if (timing.enabled)
|
|
timing.workerWait = profile_ms(timing.workerQueued, start);
|
|
FrameTiming* stages = timing.enabled ? &timing : nullptr;
|
|
const bool overlapped = async_copies() && copy_streams() != nullptr;
|
|
if (params.vram && env_uint("D4R_SHIM_VRAM_VERIFY", 0) == 0)
|
|
dump_vram_inputs(slot, frame, params, params.hasExposure ? 4 : 3);
|
|
const bool linearInputs = params.vram && feature->linearInputs;
|
|
bool ok = linearInputs ? ensure_linear_inputs(slot, params.hasExposure)
|
|
: params.vram ? upload_inputs_vram(*feature, slot, params.hasExposure, frame, stages)
|
|
: overlapped
|
|
? upload_inputs_async(*feature, slot, params.hasExposure, frame, stages)
|
|
: upload_plane(Plane::Color, slot.host[0], feature->color, frame, stages, 0) &&
|
|
upload_plane(Plane::Depth, slot.host[1], feature->depth, frame, stages, 1) &&
|
|
upload_plane(Plane::Motion, slot.host[2], feature->motion, frame, stages, 2) &&
|
|
(!params.hasExposure ||
|
|
upload_plane(Plane::Exposure, slot.host[3], feature->exposure, frame, stages, 3));
|
|
ok = ok && ensure_cuda_image(feature->output, feature->outputWidth, feature->outputHeight, CUDA_FORMAT_HALF, 4,
|
|
true, false);
|
|
const auto dilationStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
const CudaObject dilatedMotion = ok ? prepare_dilated_motion(*feature, slot, params) : 0;
|
|
// Device-to-array copies (VRAM interop, staged uploads) return before they
|
|
// finish, and NGX does not evaluate on the stream they were issued on, so
|
|
// without this DLSS can sample a mix of this frame's and the previous
|
|
// frame's colour/depth/motion: invisible when the camera is still, beaded
|
|
// thin geometry in motion. D4R_SHIM_INPUT_SYNC=0 restores the old behaviour.
|
|
static const bool inputSync = env_uint("D4R_SHIM_INPUT_SYNC", 1) != 0;
|
|
// Linear inputs are already ready; other copies need completion before NGX samples them.
|
|
if (ok && (dilatedMotion != 0 || (inputSync && !linearInputs)))
|
|
{
|
|
const int syncResult = synchronize_default_stream(*feature);
|
|
if (syncResult != 0)
|
|
{
|
|
logf("frame %u: input copy synchronize failed: %d", frame, syncResult);
|
|
ok = false;
|
|
}
|
|
}
|
|
if (dilatedMotion != 0 && ok)
|
|
{
|
|
params.motionDilation = env_uint("D4R_MOTION_DILATION", 0);
|
|
if (timing.enabled)
|
|
timing.motionDilationWall = profile_ms(dilationStart, ProfileClock::now());
|
|
}
|
|
slot.hostBusy = false; // the host planes are no longer needed
|
|
// The previous frame's download ran alongside these uploads.
|
|
complete_pending_download();
|
|
if (!ok)
|
|
{
|
|
slot.busy = false;
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
|
|
// The first frame NGX evaluates for a feature starts its history. Decided
|
|
// here, in evaluation order, so it does not depend on thread timing.
|
|
if (feature->evaluatedFrames++ == 0)
|
|
params.reset = 1;
|
|
set_evaluation_parameters(*feature, params);
|
|
if (linearInputs)
|
|
{
|
|
// the parameters point at these variables
|
|
feature->colorHandle = slot.linearTexture[0];
|
|
feature->depthHandle = slot.linearTexture[1];
|
|
feature->motionHandle = dilatedMotion != 0 ? dilatedMotion : slot.linearTexture[2];
|
|
feature->exposureHandle = params.hasExposure ? slot.linearTexture[3] : feature->exposureHandle;
|
|
}
|
|
static const bool outputDirect = env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0;
|
|
// The redirect is carried out by the native output kernel's stores (ZLUDA's compile of NVIDIA's kernel
|
|
// writes the array), so it is set while the previous frame's output kernel was the native one and
|
|
// undone below when this frame's was not.
|
|
const bool nativeOutput = g.cu.outputKernelNative == nullptr || g.cu.outputKernelNative() == 1;
|
|
if (outputDirect && feature->outputDirectAllowed && g.cu.setArrayRedirect != nullptr && feature->output.array != nullptr &&
|
|
(nativeOutput || feature->outputRedirected))
|
|
{
|
|
// split frames write their own slot; the staged path downloads from outputLinear
|
|
const size_t rowBytes = static_cast<size_t>(feature->output.width) * 8, bytes = rowBytes * feature->output.height;
|
|
CudaDevicePtr target = 0;
|
|
if (params.vram && params.split && feature->outputs[frame % kOutputSlots].vram.bytes >= bytes)
|
|
target = feature->outputs[frame % kOutputSlots].vram.device;
|
|
else if (!params.vram && overlapped && ensure_linear(feature->outputLinear, feature->outputLinearBytes, bytes))
|
|
target = feature->outputLinear;
|
|
const bool redirect = nativeOutput && target != 0 &&
|
|
g.cu.setArrayRedirect(feature->output.array, target, static_cast<uint32_t>(rowBytes)) == 0;
|
|
if (!redirect && feature->outputRedirected)
|
|
g.cu.setArrayRedirect(feature->output.array, 0, 0);
|
|
static bool logged = false;
|
|
if (redirect && !logged)
|
|
{
|
|
logf("frame %u: direct output (the native output kernel writes the shared buffer; no array copy)", frame);
|
|
logged = true;
|
|
}
|
|
feature->outputRedirected = redirect;
|
|
}
|
|
const int eventStartResult = timing.enabled && feature->profileEventsReady
|
|
? g.cu.eventRecord(feature->profileStart, nullptr) : -1;
|
|
const auto evalStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
NgxResult result = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
|
|
if (result == NGX_FAIL_INVALID_PARAMETER && recreate_for_render_size(*feature, params, frame))
|
|
result = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
|
|
if (feature->outputRedirected && g.cu.outputKernelNative != nullptr && g.cu.outputKernelNative() != 1)
|
|
{
|
|
// this frame's output kernel ignored the redirect: publish from the array as usual
|
|
g.cu.setArrayRedirect(feature->output.array, 0, 0);
|
|
feature->outputRedirected = false;
|
|
}
|
|
const auto evalReturned = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
const int eventEndResult = eventStartResult == 0 ? g.cu.eventRecord(feature->profileEnd, nullptr) : -1;
|
|
const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
// D4R_SHIM_EVAL_SYNC=0 (VRAM interop): no wait between NGX's kernels and the output copy queued
|
|
// behind them on the same stream; publish_vram synchronises before the result is released.
|
|
static const bool evalSync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0;
|
|
const int syncResult = params.vram && !evalSync ? 0 : synchronize_default_stream(*feature);
|
|
const auto evaluated = ProfileClock::now();
|
|
const bool gpuEventsRecorded = eventStartResult == 0 && eventEndResult == 0;
|
|
float gpuEvalMs = -1.0f;
|
|
if (gpuEventsRecorded && syncResult == 0 && (!params.vram || evalSync) &&
|
|
g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) != 0)
|
|
gpuEvalMs = -1.0f;
|
|
// Keep the input slot owned until output completion, including linear reads.
|
|
if (result != NGX_SUCCESS || syncResult != 0)
|
|
{
|
|
logf("frame %u: CUDA_EvaluateFeature -> 0x%08x, sync %d", frame, result, syncResult);
|
|
slot.busy = false;
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
if (timing.enabled)
|
|
{
|
|
timing.ngxHost = profile_ms(evalStart, evalReturned);
|
|
timing.ctxSync = profile_ms(syncStart, evaluated);
|
|
timing.gpuEval = gpuEvalMs;
|
|
timing.gpuEventsRecorded = gpuEventsRecorded && params.vram && !evalSync;
|
|
}
|
|
if (params.vram)
|
|
{
|
|
publish_vram(feature, frame, params, timing, start);
|
|
slot.busy = false; // output synchronization completed all reads of this input slot
|
|
return;
|
|
}
|
|
|
|
const int hostIndex = static_cast<int>(feature->nextOutputHost++ % 2);
|
|
if (!wait_for_release(feature->outputHostBusy[hostIndex]))
|
|
{
|
|
logf("frame %u: output host buffer still busy; dropping result", frame);
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
if (overlapped)
|
|
{
|
|
if (!start_download(*feature, feature->outputHost[hostIndex], stages))
|
|
{
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
feature->outputHostBusy[hostIndex] = true;
|
|
if (timing.enabled)
|
|
timing.worker = profile_ms(start, ProfileClock::now());
|
|
g_pendingDownload = {feature, hostIndex, frame, params, timing};
|
|
// Without a queued frame to overlap, finish it now rather than add latency.
|
|
if (g.worker.pending() == 0)
|
|
complete_pending_download(true);
|
|
return;
|
|
}
|
|
if (!download_output(*feature, feature->outputHost[hostIndex], frame, stages))
|
|
{
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
feature->outputHostBusy[hostIndex] = true;
|
|
if (timing.enabled)
|
|
timing.worker = profile_ms(start, ProfileClock::now());
|
|
post_publish(feature, hostIndex, frame, params, timing);
|
|
}
|
|
|
|
static void prepare_inputs(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
|
|
{
|
|
InputSlot& slot = feature->inputs[slotIndex];
|
|
const auto start = ProfileClock::now();
|
|
// A later input marker does not prove this slot's copies were submitted:
|
|
// the game may have discarded this frame's command list. Require its own
|
|
// tag, otherwise stale input data can enter DLSS's temporal history.
|
|
const volatile uint32_t* inputMarker = feature->markerValue + 1 + slotIndex;
|
|
while (*inputMarker != frame)
|
|
{
|
|
std::atomic_thread_fence(std::memory_order_acquire);
|
|
const bool skipped = static_cast<int32_t>(*feature->markerValue - frame) > 0;
|
|
// The GPU may have filled our slot between the loop condition and
|
|
// the aggregate-marker read. Recheck before deciding it was skipped.
|
|
if (*inputMarker == frame)
|
|
break;
|
|
if (feature->retiring.load() || skipped || std::chrono::steady_clock::now() - start > std::chrono::seconds(5))
|
|
{
|
|
logf("frame %u: %s waiting for GPU marker (at %u); dropping", frame,
|
|
feature->retiring.load() ? "feature retired while" : skipped ? "input submission skipped while" : "timed out",
|
|
*inputMarker);
|
|
slot.busy = false;
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
// D4R_SHIM_MARKER_POLL_US: poll interval (0 = yield); detection latency is on the critical path
|
|
static const unsigned markerPollUs = env_uint("D4R_SHIM_MARKER_POLL_US", 200);
|
|
if (markerPollUs == 0)
|
|
std::this_thread::yield();
|
|
else
|
|
d4r_sleep_us(markerPollUs);
|
|
}
|
|
const auto ready = ProfileClock::now();
|
|
// With VRAM interop the inputs are already on the GPU; host staging runs
|
|
// only to cross-check them (D4R_SHIM_VRAM_VERIFY).
|
|
if (!params.vram || env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0)
|
|
{
|
|
if (!wait_for_release(slot.hostBusy))
|
|
{
|
|
logf("frame %u: host planes of slot %d still busy; dropping", frame, slotIndex);
|
|
slot.busy = false;
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
FrameTiming* stages = timing.enabled ? &timing : nullptr;
|
|
stage_plane(Plane::Color, slot.color, slot.host[0], stages, 0);
|
|
stage_plane(Plane::Depth, slot.depth, slot.host[1], stages, 1);
|
|
stage_plane(Plane::Motion, slot.motion, slot.host[2], stages, 2);
|
|
log_motion_stats(slot.host[2], slot.host[0], frame);
|
|
if (params.hasExposure)
|
|
stage_plane(Plane::Exposure, slot.exposure, slot.host[3], stages, 3);
|
|
char dumpDirectory[MAX_PATH];
|
|
if (input_dump_directory(frame, dumpDirectory))
|
|
dump_input_planes(slot.host, frame, params, dumpDirectory);
|
|
slot.hostBusy = true;
|
|
}
|
|
if (!params.vram)
|
|
slot.busy = false; // the readback buffers are no longer needed
|
|
if (timing.enabled)
|
|
{
|
|
timing.prepStart = start;
|
|
timing.markerWait = profile_ms(start, ready);
|
|
timing.prep = profile_ms(start, ProfileClock::now());
|
|
timing.workerQueued = ProfileClock::now();
|
|
}
|
|
g.worker.post([feature, slotIndex, frame, params, timing] {
|
|
run_evaluation(feature, slotIndex, frame, params, timing);
|
|
});
|
|
}
|
|
|
|
// Picks a result slot that no queued command list still copies from, and
|
|
// marks it so evaluate() does not present it while it is rewritten. A reader
|
|
// frame's output copy is recorded after its marker, so the GPU is past it
|
|
// only once a later frame's marker has landed. Waits briefly for that.
|
|
static int claim_output_slot(Feature* feature)
|
|
{
|
|
const auto start = std::chrono::steady_clock::now();
|
|
for (;;)
|
|
{
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->outputMutex);
|
|
const uint32_t gpuFrame = *feature->markerValue;
|
|
for (int index = 0; index < kOutputSlots; ++index)
|
|
{
|
|
if (index == feature->latestOutput.load())
|
|
continue;
|
|
if (static_cast<int32_t>(gpuFrame - feature->outputs[index].lastReadFrame.load()) > 0)
|
|
{
|
|
feature->outputs[index].lastReadFrame = 0x7fffffffu + gpuFrame;
|
|
return index;
|
|
}
|
|
}
|
|
}
|
|
if (std::chrono::steady_clock::now() - start > std::chrono::milliseconds(50))
|
|
return -1;
|
|
d4r_sleep_us(200);
|
|
}
|
|
}
|
|
|
|
// Makes `target` the result evaluate() presents and retires the frame.
|
|
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
|
|
const FrameTiming& timing, ProfileClock::time_point start)
|
|
{
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->outputMutex);
|
|
feature->outputs[target].lastReadFrame = 0;
|
|
feature->outputs[target].producedFrame = frame;
|
|
feature->latestOutput = target;
|
|
}
|
|
const uint32_t completed = ++feature->completedFrames;
|
|
frame_retired(feature, frame);
|
|
const auto finished = ProfileClock::now();
|
|
if (timing.enabled)
|
|
{
|
|
static thread_local ProfileClock::time_point lastFinished;
|
|
const double interval = lastFinished.time_since_epoch().count() != 0
|
|
? profile_ms(lastFinished, finished) : -1.0;
|
|
lastFinished = finished;
|
|
const InputSlot& slot = feature->inputs[frame % kSlots];
|
|
logf("D4R_PROFILE frame=%u render=%ux%u output=%ux%u color=%ux%u depth=%ux%u "
|
|
"motion=%ux%u exposure=%d quality=%d flags=0x%x preset=%u "
|
|
"jitter=%.6f,%.6f mv_scale=%.3f,%.3f reset=%d pre_exposure=%.6f exposure_scale=%.6f "
|
|
"frame_dt=%.3f game_interval=%.3f "
|
|
"queue_depth=%zu queue_wait=%.3f marker_wait=%.3f slot_wait=%.3f "
|
|
"d3d_input_record=%.3f d3d_output_record=%.3f game_call=%.3f "
|
|
"color_convert=%.3f depth_convert=%.3f motion_convert=%.3f exposure_convert=%.3f "
|
|
"color_h2d=%.3f depth_h2d=%.3f motion_h2d=%.3f exposure_h2d=%.3f "
|
|
"ngx_host=%.3f ctx_sync=%.3f gpu_eval=%.3f output_sync_wall=%.3f "
|
|
"output_sync_cpu=%.3f output_sync_blocking=%d output_sync_method=%s d2h=%.3f output_convert=%.3f "
|
|
"h2d_total=%.3f d2h_issue=%.3f d2h_wait=%.3f "
|
|
"prep=%.3f worker_wait=%.3f worker=%.3f finish_wait=%.3f finish=%.3f pipeline=%.3f "
|
|
"throughput_interval=%.3f presented=%u age=%d motion_dilation=%u motion_dilation_wall=%.3f",
|
|
frame, params.renderWidth, params.renderHeight, feature->outWidth, feature->outHeight,
|
|
slot.host[0].width, slot.host[0].height, slot.host[1].width, slot.host[1].height,
|
|
slot.host[2].width, slot.host[2].height, params.hasExposure, feature->quality, feature->flags,
|
|
feature->preset, params.jitterX, params.jitterY, params.mvScaleX, params.mvScaleY,
|
|
params.reset, params.preExposure, params.exposureScale, params.frameTime, timing.gameInterval,
|
|
timing.queueDepth, profile_ms(timing.queuedAt, timing.prepStart), timing.markerWait, timing.slotWait,
|
|
timing.inputRecord, timing.outputRecord, timing.gameCall,
|
|
timing.convert[0], timing.convert[1], timing.convert[2], timing.convert[3],
|
|
timing.upload[0], timing.upload[1], timing.upload[2], timing.upload[3],
|
|
timing.ngxHost, timing.ctxSync, timing.gpuEval, timing.outputSyncWall, timing.outputSyncCpu,
|
|
timing.outputSyncBlocking, timing.outputSyncBlocking ? "event_query_sleep" : "context",
|
|
timing.download, timing.outputConvert,
|
|
timing.h2dTotal, timing.d2hIssue, timing.d2hWait,
|
|
timing.prep, timing.workerWait, timing.worker, timing.finishWait, profile_ms(start, finished),
|
|
profile_ms(timing.prepStart, finished), interval,
|
|
timing.presentedFrame, timing.presentedFrame != 0 ? static_cast<int>(frame - timing.presentedFrame) : -1,
|
|
params.motionDilation, timing.motionDilationWall);
|
|
}
|
|
if (completed <= 5 || completed % 120 == 0)
|
|
logf("frame %u done (slot %d)", frame, target);
|
|
}
|
|
|
|
static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing)
|
|
{
|
|
const auto start = ProfileClock::now();
|
|
if (timing.enabled)
|
|
timing.finishWait = profile_ms(timing.finishQueued, start);
|
|
const int target = claim_output_slot(feature);
|
|
if (target < 0)
|
|
{
|
|
feature->outputHostBusy[hostIndex] = false;
|
|
logf("frame %u: no free output slot; dropping result", frame);
|
|
frame_retired(feature, frame);
|
|
return;
|
|
}
|
|
write_output(feature->outputs[target].staging, feature->outputHost[hostIndex], timing.enabled ? &timing : nullptr);
|
|
feature->outputHostBusy[hostIndex] = false;
|
|
finish_publish(feature, target, frame, params, timing, start);
|
|
}
|
|
|
|
// Waits until every frame queued so far has passed through all three stages.
|
|
static void drain_pipeline()
|
|
{
|
|
g.prep.call([] { return 0; });
|
|
g.worker.call([] {
|
|
complete_pending_download();
|
|
return 0;
|
|
});
|
|
g.finish.call([] { return 0; });
|
|
}
|
|
|
|
static void release_feature(Feature* feature);
|
|
static void destroy_recorded_resources(Feature* feature);
|
|
static void finish_shutdown();
|
|
|
|
class DeferredResources final : public IUnknown
|
|
{
|
|
public:
|
|
explicit DeferredResources(Feature* feature) : feature_(feature) { ++g.resourceOwners; }
|
|
HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, void** out) override
|
|
{
|
|
if (out == nullptr) return E_POINTER;
|
|
*out = nullptr;
|
|
if (iid != __uuidof(IUnknown)) return E_NOINTERFACE;
|
|
*out = static_cast<IUnknown*>(this);
|
|
AddRef();
|
|
return S_OK;
|
|
}
|
|
ULONG STDMETHODCALLTYPE AddRef() override { return ++refs_; }
|
|
ULONG STDMETHODCALLTYPE Release() override
|
|
{
|
|
const ULONG remaining = --refs_;
|
|
if (remaining == 0)
|
|
g.cleanup.post([this] {
|
|
destroy_recorded_resources(feature_);
|
|
delete this;
|
|
--g.resourceOwners;
|
|
finish_shutdown();
|
|
});
|
|
return remaining;
|
|
}
|
|
private:
|
|
std::atomic<ULONG> refs_{1};
|
|
Feature* feature_;
|
|
};
|
|
|
|
// --- exported NGX D3D12 API ------------------------------------------------------
|
|
|
|
#define D4R_EXPORT extern "C" __declspec(dllexport)
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init(unsigned long long applicationId, const wchar_t* dataPath,
|
|
ID3D12Device* device, const NgxFeatureCommonInfo* featureInfo,
|
|
unsigned int sdkVersion)
|
|
{
|
|
logf("NVSDK_NGX_D3D12_Init(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
|
|
return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_Ext(unsigned long long applicationId, const wchar_t* dataPath,
|
|
ID3D12Device* device, unsigned int sdkVersion,
|
|
const NgxFeatureCommonInfo* featureInfo)
|
|
{
|
|
logf("NVSDK_NGX_D3D12_Init_Ext(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
|
|
return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_ProjectID(const char* projectId, int engineType, const char* engineVersion,
|
|
const wchar_t* dataPath, ID3D12Device* device,
|
|
unsigned int sdkVersion, const NgxFeatureCommonInfo* featureInfo)
|
|
{
|
|
logf("NVSDK_NGX_D3D12_Init_ProjectID(project=%s, engine=%d %s, device=%p, sdk=0x%x)",
|
|
projectId != nullptr ? projectId : "", engineType, engineVersion != nullptr ? engineVersion : "", device,
|
|
sdkVersion);
|
|
ProjectIdentity project;
|
|
project.id = projectId != nullptr ? projectId : "";
|
|
project.engineType = engineType;
|
|
project.engineVersion = engineVersion != nullptr ? engineVersion : "";
|
|
return initialize(0, dataPath, device, featureInfo, sdkVersion, &project);
|
|
}
|
|
|
|
// Parameter objects handed to callers are the shim's own implementation: the
|
|
// CUDA-path objects of the official core cannot hold D3D12 resources.
|
|
static std::mutex g_parametersMutex;
|
|
static std::vector<void*> g_ownedParameters;
|
|
|
|
static void* new_parameters()
|
|
{
|
|
void* parameters = d4r_ngx_parameters_create();
|
|
std::lock_guard<std::mutex> lock(g_parametersMutex);
|
|
g_ownedParameters.push_back(parameters);
|
|
return parameters;
|
|
}
|
|
|
|
// Copies the official capability values (availability, driver requirements,
|
|
// and the optimal-settings/stats callbacks, which work on any parameter
|
|
// implementation) into one of our parameter objects.
|
|
static NgxResult fill_capabilities(void* parameters)
|
|
{
|
|
return g.worker.call([&]() -> NgxResult {
|
|
void* official = nullptr;
|
|
const NgxResult result = g.ngx.getCapabilityParameters(&official);
|
|
if (result != NGX_SUCCESS || official == nullptr)
|
|
return result;
|
|
const char* integers[] = {"SuperSampling.Available", "SuperSampling.NeedsUpdatedDriver",
|
|
"SuperSampling.FeatureInitResult"};
|
|
for (const char* name : integers)
|
|
{
|
|
int value = 0;
|
|
if (d4r_ngx_get_int(official, name, &value) == NGX_SUCCESS)
|
|
d4r_ngx_set_int(parameters, name, value);
|
|
}
|
|
const char* unsignedValues[] = {"SuperSampling.MinDriverVersionMajor", "SuperSampling.MinDriverVersionMinor"};
|
|
for (const char* name : unsignedValues)
|
|
{
|
|
unsigned int value = 0;
|
|
if (d4r_ngx_get_uint(official, name, &value) == NGX_SUCCESS)
|
|
d4r_ngx_set_uint(parameters, name, value);
|
|
}
|
|
const char* pointers[] = {"DLSSOptimalSettingsCallback", "DLSSGetStatsCallback"};
|
|
for (const char* name : pointers)
|
|
{
|
|
void* value = nullptr;
|
|
if (d4r_ngx_get_void(official, name, &value) == NGX_SUCCESS && value != nullptr)
|
|
d4r_ngx_set_void(parameters, name, value);
|
|
}
|
|
g.ngx.destroyParameters(official);
|
|
return NGX_SUCCESS;
|
|
});
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetParameters(void** parameters)
|
|
{
|
|
if (!g.ngxInitialized)
|
|
return NGX_FAIL_NOT_INITIALIZED;
|
|
*parameters = new_parameters();
|
|
return fill_capabilities(*parameters);
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_AllocateParameters(void** parameters)
|
|
{
|
|
if (!g.ngxInitialized)
|
|
return NGX_FAIL_NOT_INITIALIZED;
|
|
*parameters = new_parameters();
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetCapabilityParameters(void** parameters)
|
|
{
|
|
if (!g.ngxInitialized)
|
|
return NGX_FAIL_NOT_INITIALIZED;
|
|
*parameters = new_parameters();
|
|
const NgxResult result = fill_capabilities(*parameters);
|
|
int available = -1, initResult = -1, needsDriver = -1;
|
|
d4r_ngx_get_int(*parameters, "SuperSampling.Available", &available);
|
|
d4r_ngx_get_int(*parameters, "SuperSampling.FeatureInitResult", &initResult);
|
|
d4r_ngx_get_int(*parameters, "SuperSampling.NeedsUpdatedDriver", &needsDriver);
|
|
logf("NVSDK_NGX_D3D12_GetCapabilityParameters -> 0x%08x, SuperSampling.Available=%d FeatureInitResult=0x%08x "
|
|
"NeedsUpdatedDriver=%d",
|
|
result, available, static_cast<unsigned int>(initResult), needsDriver);
|
|
return result;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_DestroyParameters(void* parameters)
|
|
{
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_parametersMutex);
|
|
for (auto it = g_ownedParameters.begin(); it != g_ownedParameters.end(); ++it)
|
|
if (*it == parameters)
|
|
{
|
|
g_ownedParameters.erase(it);
|
|
d4r_ngx_parameters_destroy(parameters);
|
|
return NGX_SUCCESS;
|
|
}
|
|
}
|
|
return NGX_FAIL_INVALID_PARAMETER;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetScratchBufferSize(unsigned int, const void*, size_t* size)
|
|
{
|
|
// Scratch memory for the CUDA path is allocated internally.
|
|
if (size != nullptr)
|
|
*size = 0;
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetFeatureRequirements(IUnknown*, const void*, NgxFeatureRequirement* requirement)
|
|
{
|
|
if (requirement != nullptr)
|
|
{
|
|
std::memset(requirement, 0, sizeof(*requirement));
|
|
requirement->FeatureSupported = 0; // supported
|
|
requirement->MinHWArchitecture = 0x160;
|
|
}
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
static unsigned int get_uint_or(void* parameters, const char* name, unsigned int fallback)
|
|
{
|
|
unsigned int value = fallback;
|
|
if (d4r_ngx_get_uint(parameters, name, &value) != NGX_SUCCESS)
|
|
{
|
|
int signedValue = 0;
|
|
if (d4r_ngx_get_int(parameters, name, &signedValue) == NGX_SUCCESS)
|
|
return static_cast<unsigned int>(signedValue);
|
|
return fallback;
|
|
}
|
|
return value;
|
|
}
|
|
|
|
static int get_int_or(void* parameters, const char* name, int fallback)
|
|
{
|
|
int value = fallback;
|
|
if (d4r_ngx_get_int(parameters, name, &value) != NGX_SUCCESS)
|
|
{
|
|
unsigned int unsignedValue = 0;
|
|
if (d4r_ngx_get_uint(parameters, name, &unsignedValue) == NGX_SUCCESS)
|
|
return static_cast<int>(unsignedValue);
|
|
return fallback;
|
|
}
|
|
return value;
|
|
}
|
|
|
|
static float get_float_or(void* parameters, const char* name, float fallback)
|
|
{
|
|
float value = fallback;
|
|
return d4r_ngx_get_float(parameters, name, &value) == NGX_SUCCESS ? value : fallback;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned int featureId, void* parameters,
|
|
NgxHandle** handle)
|
|
{
|
|
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
|
|
std::lock_guard<std::mutex> stateLock(g.mutex);
|
|
if (!g.ngxInitialized)
|
|
return NGX_FAIL_NOT_INITIALIZED;
|
|
std::call_once(g_vk.once, init_vram_interop);
|
|
if (g_vk.lifetime == nullptr)
|
|
{
|
|
logf("DLSS requires lifetime-aware d4r d3d12.dll/d3d12core.dll; update both files with the shim");
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
if (featureId != NGX_FEATURE_SUPER_SAMPLING || parameters == nullptr || handle == nullptr)
|
|
{
|
|
logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u) unsupported", featureId);
|
|
return NGX_FAIL_FEATURE_NOT_SUPPORTED;
|
|
}
|
|
auto* feature = new Feature();
|
|
feature->width = get_uint_or(parameters, "Width", 0);
|
|
feature->height = get_uint_or(parameters, "Height", 0);
|
|
feature->outWidth = get_uint_or(parameters, "OutWidth", 0);
|
|
feature->outHeight = get_uint_or(parameters, "OutHeight", 0);
|
|
const int quality = get_int_or(parameters, "PerfQualityValue", 0);
|
|
const int flags = get_int_or(parameters, "DLSS.Feature.Create.Flags", 0);
|
|
feature->quality = quality;
|
|
feature->flags = flags;
|
|
const int subrects = get_int_or(parameters, "DLSS.Enable.Output.Subrects", 0);
|
|
const char* presetNames[] = {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality",
|
|
"DLSS.Hint.Render.Preset.Balanced", "DLSS.Hint.Render.Preset.Performance",
|
|
"DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"};
|
|
unsigned int presets[6];
|
|
for (int index = 0; index < 6; ++index)
|
|
presets[index] = get_uint_or(parameters, presetNames[index], 0);
|
|
// D4R_DLSS_PRESET forces one render preset (NVSDK_NGX_DLSS_Hint_Render_Preset
|
|
// value, e.g. 5 = E) for every quality mode.
|
|
const std::string forced = env_string("D4R_DLSS_PRESET");
|
|
if (!forced.empty())
|
|
{
|
|
const unsigned int value = static_cast<unsigned int>(strtoul(forced.c_str(), nullptr, 0));
|
|
const char* model = value == 5 ? "E" : value == 11 ? "K" : value == 12 ? "L" : value == 13 ? "M" : "custom";
|
|
logf("DLSS model override: D4R_DLSS_PRESET=%s (model %s) overrides game/OptiScaler presets "
|
|
"[DLAA=%u Quality=%u Balanced=%u Performance=%u UltraPerformance=%u UltraQuality=%u]; "
|
|
"using preset=%u for all quality modes",
|
|
forced.c_str(), model, presets[0], presets[1], presets[2], presets[3], presets[4], presets[5], value);
|
|
for (unsigned int& preset : presets)
|
|
preset = value;
|
|
}
|
|
feature->preset = presets[1];
|
|
// Direct output needs the native output kernel of the preset in use (verified: K = 11 through
|
|
// hiluma_engine_output, M = 13 through rrlite_downsample_kernel, its last kernel). Only a forced
|
|
// preset qualifies.
|
|
if (!forced.empty())
|
|
{
|
|
char allowed[128] = "11,13";
|
|
if (const std::string list = env_string("D4R_SHIM_OUTPUT_DIRECT_PRESETS"); !list.empty())
|
|
snprintf(allowed, sizeof(allowed), "%s", list.c_str());
|
|
for (char* token = strtok(allowed, ","); token != nullptr; token = strtok(nullptr, ","))
|
|
feature->outputDirectAllowed |= strtoul(token, nullptr, 0) == presets[1];
|
|
}
|
|
logf("NVSDK_NGX_D3D12_CreateFeature: %ux%u -> %ux%u quality=%d flags=0x%x subrects=%d preset=%u", feature->width,
|
|
feature->height, feature->outWidth, feature->outHeight, quality, flags, subrects, presets[1]);
|
|
|
|
const NgxResult result = g.worker.call([&]() -> NgxResult {
|
|
NgxResult allocation = g.ngx.allocateParameters(&feature->cudaParams);
|
|
if (allocation != NGX_SUCCESS)
|
|
return allocation;
|
|
void* p = feature->cudaParams;
|
|
d4r_ngx_set_uint(p, "Width", feature->width);
|
|
d4r_ngx_set_uint(p, "Height", feature->height);
|
|
d4r_ngx_set_uint(p, "OutWidth", feature->outWidth);
|
|
d4r_ngx_set_uint(p, "OutHeight", feature->outHeight);
|
|
d4r_ngx_set_int(p, "PerfQualityValue", quality);
|
|
d4r_ngx_set_int(p, "DLSS.Feature.Create.Flags", flags);
|
|
d4r_ngx_set_int(p, "DLSS.Enable.Output.Subrects", subrects);
|
|
d4r_ngx_set_uint(p, "CreationNodeMask", 1);
|
|
d4r_ngx_set_uint(p, "VisibilityNodeMask", 1);
|
|
for (int index = 0; index < 6; ++index)
|
|
d4r_ngx_set_uint(p, presetNames[index], presets[index]);
|
|
if (g.cu.memAlloc(&feature->scratch, 64ull * 1024 * 1024) == 0)
|
|
{
|
|
d4r_ngx_set_void(p, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature->scratch)));
|
|
d4r_ngx_set_ull(p, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
|
|
}
|
|
else
|
|
feature->scratch = 0;
|
|
const NgxResult created = g.ngx.createFeature(NGX_FEATURE_SUPER_SAMPLING, p, &feature->cudaHandle);
|
|
feature->cudaWidth = feature->width;
|
|
feature->cudaHeight = feature->height;
|
|
if (created == NGX_SUCCESS && env_uint("D4R_SHIM_BLOCKING_SYNC", 1) != 0 &&
|
|
g.cu.eventCreate != nullptr && g.cu.eventRecord != nullptr &&
|
|
g.cu.eventSynchronize != nullptr && g.cu.eventQuery != nullptr && g.cu.eventDestroy != nullptr)
|
|
{
|
|
// Explicit query/sleep waits avoid HIP's spinning event-sync path.
|
|
// Timing is unnecessary for this completion event.
|
|
constexpr unsigned int kBlockingSyncNoTiming = 0x1u | 0x2u;
|
|
const int eventResult = g.cu.eventCreate(&feature->outputReadyEvent, kBlockingSyncNoTiming);
|
|
if (eventResult != 0)
|
|
{
|
|
logf("blocking output wait unavailable (cuEventCreate %d); using context sync", eventResult);
|
|
feature->outputReadyEvent = nullptr;
|
|
}
|
|
else
|
|
logf("output event query/sleep wait enabled for this feature");
|
|
}
|
|
if (created == NGX_SUCCESS && profile_enabled() && g.cu.eventCreate != nullptr &&
|
|
g.cu.eventRecord != nullptr && g.cu.eventElapsedTime != nullptr && g.cu.eventDestroy != nullptr)
|
|
{
|
|
const int startResult = g.cu.eventCreate(&feature->profileStart, 0);
|
|
const int endResult = startResult == 0 ? g.cu.eventCreate(&feature->profileEnd, 0) : -1;
|
|
feature->profileEventsReady = startResult == 0 && endResult == 0;
|
|
if (!feature->profileEventsReady)
|
|
{
|
|
logf("D4R_PROFILE: CUDA event creation failed (%d, %d)", startResult, endResult);
|
|
if (feature->profileStart != nullptr)
|
|
g.cu.eventDestroy(feature->profileStart);
|
|
if (feature->profileEnd != nullptr)
|
|
g.cu.eventDestroy(feature->profileEnd);
|
|
feature->profileStart = feature->profileEnd = nullptr;
|
|
}
|
|
}
|
|
return created;
|
|
});
|
|
logf("NVSDK_NGX_CUDA_CreateFeature -> 0x%08x", result);
|
|
if (result != NGX_SUCCESS)
|
|
{
|
|
release_feature(feature);
|
|
return result;
|
|
}
|
|
if (!create_buffer(D3D12_HEAP_TYPE_READBACK, 256, &feature->marker,
|
|
reinterpret_cast<uint8_t**>(const_cast<uint32_t**>(&feature->markerValue))))
|
|
{
|
|
release_feature(feature);
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
for (int index = 0; index <= kSlots; ++index)
|
|
feature->markerValue[index] = 0;
|
|
feature->resources = new DeferredResources(feature);
|
|
feature->handle.Id = g.nextHandleId++;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
g_features.push_back(feature);
|
|
}
|
|
*handle = &feature->handle;
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
static Feature* find_feature(const NgxHandle* handle)
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
for (Feature* feature : g_features)
|
|
if (&feature->handle == handle && !feature->retiring.load())
|
|
return feature;
|
|
return nullptr;
|
|
}
|
|
|
|
static ID3D12Resource* get_resource(void* parameters, const char* name)
|
|
{
|
|
ID3D12Resource* resource = nullptr;
|
|
if (d4r_ngx_get_d3d12_resource(parameters, name, &resource) == NGX_SUCCESS && resource != nullptr)
|
|
return resource;
|
|
void* raw = nullptr;
|
|
if (d4r_ngx_get_void(parameters, name, &raw) == NGX_SUCCESS)
|
|
return static_cast<ID3D12Resource*>(raw);
|
|
return nullptr;
|
|
}
|
|
|
|
static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* handle, void* parameters)
|
|
{
|
|
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
|
|
FrameTiming timing;
|
|
timing.enabled = profile_enabled();
|
|
if (timing.enabled)
|
|
{
|
|
timing.gameStart = ProfileClock::now();
|
|
static thread_local ProfileClock::time_point lastGameStart;
|
|
if (lastGameStart.time_since_epoch().count() != 0)
|
|
timing.gameInterval = profile_ms(lastGameStart, timing.gameStart);
|
|
lastGameStart = timing.gameStart;
|
|
}
|
|
Feature* feature = find_feature(handle);
|
|
if (feature == nullptr || list == nullptr || parameters == nullptr)
|
|
return NGX_FAIL_INVALID_PARAMETER;
|
|
ID3D12Resource* color = get_resource(parameters, "Color");
|
|
ID3D12Resource* depth = get_resource(parameters, "Depth");
|
|
ID3D12Resource* motion = get_resource(parameters, "MotionVectors");
|
|
ID3D12Resource* output = get_resource(parameters, "Output");
|
|
ID3D12Resource* exposure = get_resource(parameters, "ExposureTexture");
|
|
const uint32_t frame = ++feature->frame;
|
|
if (color == nullptr || depth == nullptr || motion == nullptr || output == nullptr)
|
|
{
|
|
if (frame <= 3)
|
|
logf("evaluate frame %u: missing resource color=%p depth=%p mv=%p output=%p", frame, color, depth, motion,
|
|
output);
|
|
return NGX_FAIL_INVALID_PARAMETER;
|
|
}
|
|
|
|
D3D12_RESOURCE_DESC colorDesc, depthDesc, motionDesc, outputDesc;
|
|
color->GetDesc(&colorDesc);
|
|
depth->GetDesc(&depthDesc);
|
|
motion->GetDesc(&motionDesc);
|
|
output->GetDesc(&outputDesc);
|
|
if (frame == 1)
|
|
logf("evaluate: color %llux%u fmt=%d, depth %llux%u fmt=%d, mv %llux%u fmt=%d, output %llux%u fmt=%d, "
|
|
"exposure=%p",
|
|
colorDesc.Width, colorDesc.Height, colorDesc.Format, depthDesc.Width, depthDesc.Height, depthDesc.Format,
|
|
motionDesc.Width, motionDesc.Height, motionDesc.Format, outputDesc.Width, outputDesc.Height,
|
|
outputDesc.Format, exposure);
|
|
const bool colorSupported = supported_input(Plane::Color, colorDesc.Format);
|
|
const bool depthSupported = supported_input(Plane::Depth, depthDesc.Format);
|
|
const bool motionSupported = supported_input(Plane::Motion, motionDesc.Format);
|
|
const bool outputSupported = supported_output(outputDesc.Format);
|
|
if (!colorSupported || !depthSupported || !motionSupported || !outputSupported)
|
|
{
|
|
// Log the first failures even if they occur well after startup, then periodically.
|
|
const uint32_t rejected = ++feature->rejectedFormatCount;
|
|
if (rejected <= 3 || rejected % 120 == 0)
|
|
logf("evaluate feature %u frame %u: unsupported format (rejection %u): "
|
|
"color %llux%u fmt=%d%s, depth %llux%u fmt=%d%s, "
|
|
"mv %llux%u fmt=%d%s, output %llux%u fmt=%d%s",
|
|
feature->handle.Id, frame, rejected,
|
|
colorDesc.Width, colorDesc.Height, colorDesc.Format, colorSupported ? "" : " (unsupported)",
|
|
depthDesc.Width, depthDesc.Height, depthDesc.Format, depthSupported ? "" : " (unsupported)",
|
|
motionDesc.Width, motionDesc.Height, motionDesc.Format, motionSupported ? "" : " (unsupported)",
|
|
outputDesc.Width, outputDesc.Height, outputDesc.Format, outputSupported ? "" : " (unsupported)");
|
|
return NGX_FAIL_UNSUPPORTED_FORMAT;
|
|
}
|
|
if (exposure != nullptr)
|
|
{
|
|
D3D12_RESOURCE_DESC exposureDesc;
|
|
exposure->GetDesc(&exposureDesc);
|
|
if (!supported_input(Plane::Exposure, exposureDesc.Format))
|
|
exposure = nullptr;
|
|
}
|
|
|
|
FrameParams p;
|
|
p.jitterX = get_float_or(parameters, "Jitter.Offset.X", 0.0f);
|
|
p.jitterY = get_float_or(parameters, "Jitter.Offset.Y", 0.0f);
|
|
p.mvScaleX = get_float_or(parameters, "MV.Scale.X", 1.0f);
|
|
p.mvScaleY = get_float_or(parameters, "MV.Scale.Y", 1.0f);
|
|
p.sharpness = get_float_or(parameters, "Sharpness", 0.0f);
|
|
p.preExposure = get_float_or(parameters, "DLSS.Pre.Exposure", 1.0f);
|
|
p.exposureScale = get_float_or(parameters, "DLSS.Exposure.Scale", 1.0f);
|
|
p.frameTime = get_float_or(parameters, "FrameTimeDeltaInMsec", 16.6f);
|
|
p.reset = get_int_or(parameters, "Reset", 0);
|
|
p.invertX = get_int_or(parameters, "DLSS.Indicator.Invert.X.Axis", 0);
|
|
p.invertY = get_int_or(parameters, "DLSS.Indicator.Invert.Y.Axis", 0);
|
|
p.renderWidth = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Width", 0);
|
|
p.renderHeight = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Height", 0);
|
|
if (p.renderWidth == 0 || p.renderHeight == 0)
|
|
{
|
|
p.renderWidth = feature->width;
|
|
p.renderHeight = feature->height;
|
|
}
|
|
p.colorBaseX = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.X", 0);
|
|
p.colorBaseY = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.Y", 0);
|
|
p.depthBaseX = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.X", 0);
|
|
p.depthBaseY = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.Y", 0);
|
|
p.mvBaseX = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.X", 0);
|
|
p.mvBaseY = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.Y", 0);
|
|
p.outputBaseX = get_uint_or(parameters, "DLSS.Output.Subrect.Base.X", 0);
|
|
p.outputBaseY = get_uint_or(parameters, "DLSS.Output.Subrect.Base.Y", 0);
|
|
p.hasExposure = exposure != nullptr;
|
|
|
|
const int slotIndex = static_cast<int>(frame % kSlots);
|
|
InputSlot& slot = feature->inputs[slotIndex];
|
|
// Throttle the game to DLSS speed when the pipeline still owns this slot
|
|
// or already holds the maximum number of frames.
|
|
static const int maxInFlight = static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) > 0
|
|
? static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) : 3;
|
|
// Split frames reuse frame N's result slot for frame N + kOutputSlots.
|
|
const int cap = feature->split && maxInFlight > kOutputSlots - 1 ? kOutputSlots - 1 : maxInFlight;
|
|
const auto waitStart = std::chrono::steady_clock::now();
|
|
while (slot.busy.load() || feature->inFlight.load() >= cap)
|
|
{
|
|
if (std::chrono::steady_clock::now() - waitStart > std::chrono::seconds(2))
|
|
{
|
|
logf("frame %u: input slot %d busy or %d frames in flight; skipping evaluation", frame, slotIndex, feature->inFlight.load());
|
|
return NGX_SUCCESS;
|
|
}
|
|
d4r_sleep_us(200);
|
|
}
|
|
if (timing.enabled)
|
|
timing.slotWait = profile_ms(waitStart, ProfileClock::now());
|
|
|
|
if (g_vk.lifetime == nullptr ||
|
|
FAILED(g_vk.lifetime->RetainExternalResources(list, feature->resources)))
|
|
{
|
|
logf("frame %u: unable to retain command-list resources; evaluation not recorded", frame);
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
|
|
// VRAM interop when every resource qualifies (decided once per feature).
|
|
ID3D12Resource* const inputs[4] = {color, depth, motion, exposure};
|
|
const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
|
|
const int inputCount = exposure != nullptr ? 4 : 3;
|
|
VramCopy vramInputs[4], vramOutput;
|
|
if ((!feature->vramDecided || feature->vram) && vram_interop_available())
|
|
{
|
|
p.vram = describe_vram_copy(output, Plane::Color, vramOutput, true);
|
|
for (int index = 0; index < inputCount && p.vram; ++index)
|
|
p.vram = describe_vram_copy(inputs[index], planes[index], vramInputs[index]);
|
|
}
|
|
if (!feature->vramDecided)
|
|
{
|
|
feature->vramDecided = true;
|
|
feature->vram = p.vram;
|
|
feature->split = p.vram && env_uint("D4R_SHIM_SPLIT_FRAME", 0) != 0 && g_vk.split != nullptr &&
|
|
create_split_semaphore(*feature);
|
|
feature->linearInputs = feature->split && env_uint("D4R_SHIM_LINEAR_INPUTS", 0) != 0 &&
|
|
g.cu.registerLinearTexture != nullptr;
|
|
if (feature->linearInputs)
|
|
logf("linear inputs: NGX samples the interop buffers directly");
|
|
logf("VRAM interop %s for this feature%s%s", p.vram ? "on" : "off",
|
|
feature->split ? ", presenting each frame's own result (split frames)" : "",
|
|
p.vram ? ", waiting for input readiness with CPU sleeps" : "");
|
|
}
|
|
else if (feature->vram && !p.vram)
|
|
{
|
|
logf("frame %u: resources no longer qualify for VRAM interop; staging through host memory from now on", frame);
|
|
drain_pipeline();
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->splitMutex);
|
|
feature->split = false;
|
|
}
|
|
feature->vram = false;
|
|
feature->linearInputs = false;
|
|
feature->latestOutput = -1;
|
|
feature->outputFormat = DXGI_FORMAT_UNKNOWN; // recreates the output staging below
|
|
}
|
|
const bool verify = p.vram && env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
|
|
if (p.vram)
|
|
{
|
|
for (int index = 0; index < inputCount; ++index)
|
|
{
|
|
HostPlane& geometry = slot.host[index]; // geometry only; the bytes stay in VRAM
|
|
geometry.width = vramInputs[index].width;
|
|
geometry.height = vramInputs[index].height;
|
|
geometry.rowBytes = canonical_texel_bytes(planes[index]) * geometry.width;
|
|
if (feature->linearInputs)
|
|
geometry.rowBytes = (geometry.rowBytes + 255) & ~static_cast<size_t>(255); // texture pitch alignment
|
|
if (!ensure_vram_buffer(*feature, slot.vram[index], geometry.size()))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
if (vramInputs[0].convert && !ensure_conversion_image(*feature, feature->colorConversion,
|
|
vramInputs[0].width, vramInputs[0].height,
|
|
VK_FORMAT_R16G16B16A16_SFLOAT))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
if (vramInputs[1].convert && !ensure_conversion_image(*feature, feature->depthConversion,
|
|
vramInputs[1].width, vramInputs[1].height,
|
|
VK_FORMAT_R32_SFLOAT))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
if (vramInputs[2].convert && !ensure_conversion_image(*feature, feature->motionConversion,
|
|
vramInputs[2].width, vramInputs[2].height,
|
|
VK_FORMAT_R16G16_SFLOAT))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
if (inputCount == 4 && vramInputs[3].convert &&
|
|
!ensure_conversion_image(*feature, feature->exposureConversion,
|
|
vramInputs[3].width, vramInputs[3].height, VK_FORMAT_R32_SFLOAT))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
if (vramOutput.convert && !ensure_conversion_image(*feature, feature->outputConversion,
|
|
vramOutput.width, vramOutput.height,
|
|
VK_FORMAT_R16G16B16A16_SFLOAT))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
if ((!p.vram || verify) &&
|
|
(!ensure_staging(slot.color, color, D3D12_HEAP_TYPE_READBACK) ||
|
|
!ensure_staging(slot.depth, depth, D3D12_HEAP_TYPE_READBACK) ||
|
|
!ensure_staging(slot.motion, motion, D3D12_HEAP_TYPE_READBACK) ||
|
|
(exposure != nullptr && !ensure_staging(slot.exposure, exposure, D3D12_HEAP_TYPE_READBACK))))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
if (outputDesc.Format != feature->outputFormat || outputDesc.Width != feature->outputWidth ||
|
|
outputDesc.Height != feature->outputHeight)
|
|
{
|
|
// A new output size invalidates earlier results; let queued work finish
|
|
// before the output buffers go away.
|
|
drain_pipeline();
|
|
feature->latestOutput = -1;
|
|
for (OutputSlot& outputSlot : feature->outputs)
|
|
{
|
|
outputSlot.staging.retire();
|
|
if (p.vram ? !ensure_vram_buffer(*feature, outputSlot.vram, static_cast<size_t>(outputDesc.Width) * 8 *
|
|
outputDesc.Height)
|
|
: !ensure_staging(outputSlot.staging, output, D3D12_HEAP_TYPE_UPLOAD))
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
feature->outputFormat = outputDesc.Format;
|
|
feature->outputWidth = static_cast<UINT>(outputDesc.Width);
|
|
feature->outputHeight = outputDesc.Height;
|
|
}
|
|
|
|
const auto inputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_INPUT_STATE", 0x40));
|
|
const auto depthState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_DEPTH_STATE", 0x40));
|
|
const auto outputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_OUTPUT_STATE", 0x8));
|
|
const auto inputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
if (p.vram)
|
|
{
|
|
const D3D12_RESOURCE_STATES states[4] = {inputState, depthState, inputState, inputState};
|
|
for (int index = 0; index < inputCount; ++index)
|
|
transition(list, inputs[index], states[index], D3D12_RESOURCE_STATE_COPY_SOURCE);
|
|
const bool recorded = record_vram_inputs(*feature, list, slot, vramInputs, inputCount);
|
|
for (int index = 0; index < inputCount; ++index)
|
|
transition(list, inputs[index], D3D12_RESOURCE_STATE_COPY_SOURCE, states[index]);
|
|
if (!recorded)
|
|
{
|
|
logf("frame %u: BeginVkCommandBufferInterop failed", frame);
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
}
|
|
if (!p.vram || verify)
|
|
{
|
|
copy_to_staging(list, color, slot.color, inputState);
|
|
copy_to_staging(list, depth, slot.depth, depthState);
|
|
copy_to_staging(list, motion, slot.motion, inputState);
|
|
if (exposure != nullptr)
|
|
copy_to_staging(list, exposure, slot.exposure, inputState);
|
|
}
|
|
if (timing.enabled)
|
|
timing.inputRecord = profile_ms(inputRecordStart, ProfileClock::now());
|
|
|
|
// Frame marker, written once the copies above have completed.
|
|
ID3D12GraphicsCommandList2* list2 = nullptr;
|
|
if (FAILED(list->QueryInterface(__uuidof(ID3D12GraphicsCommandList2), reinterpret_cast<void**>(&list2))))
|
|
{
|
|
logf("ID3D12GraphicsCommandList2 unavailable; cannot place frame marker");
|
|
return NGX_FAIL_PLATFORM_ERROR;
|
|
}
|
|
const D3D12_GPU_VIRTUAL_ADDRESS markerAddress = feature->marker->GetGPUVirtualAddress();
|
|
D3D12_WRITEBUFFERIMMEDIATE_PARAMETER markers[] = {
|
|
{markerAddress + sizeof(uint32_t) * (1 + slotIndex), frame},
|
|
{markerAddress, frame},
|
|
};
|
|
D3D12_WRITEBUFFERIMMEDIATE_MODE modes[] = {
|
|
D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT, D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT,
|
|
};
|
|
list2->WriteBufferImmediate(2, markers, modes);
|
|
list2->Release();
|
|
|
|
// Present the most recent finished DLSS result.
|
|
int latest;
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->outputMutex);
|
|
latest = feature->latestOutput.load();
|
|
if (latest >= 0)
|
|
{
|
|
feature->outputs[latest].lastReadFrame = frame;
|
|
if (timing.enabled)
|
|
timing.presentedFrame = feature->outputs[latest].producedFrame.load();
|
|
}
|
|
}
|
|
const auto outputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
|
|
if (p.vram && feature->split)
|
|
p.split = SUCCEEDED(g_vk.split->SplitCommandListForExternalWait(
|
|
list, reinterpret_cast<UINT64>(feature->splitSemaphore), frame));
|
|
if (p.split)
|
|
{
|
|
// Runs in the second half of the list, once frame's result is in place.
|
|
const int target = static_cast<int>(frame % kOutputSlots);
|
|
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
|
|
if (!record_vram_output(list, feature->outputs[target].vram, vramOutput, feature->outputConversion))
|
|
logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
|
|
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
|
|
if (timing.enabled)
|
|
timing.presentedFrame = frame;
|
|
}
|
|
else if (latest >= 0 && p.vram)
|
|
{
|
|
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
|
|
if (!record_vram_output(list, feature->outputs[latest].vram, vramOutput, feature->outputConversion))
|
|
logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
|
|
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
|
|
}
|
|
else if (latest >= 0)
|
|
{
|
|
OutputSlot& outputSlot = feature->outputs[latest];
|
|
transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
|
|
D3D12_TEXTURE_COPY_LOCATION destination = {};
|
|
destination.pResource = output;
|
|
destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
|
|
destination.SubresourceIndex = 0;
|
|
D3D12_TEXTURE_COPY_LOCATION source = {};
|
|
source.pResource = outputSlot.staging.buffer;
|
|
source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
|
|
source.PlacedFootprint = outputSlot.staging.layout;
|
|
list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
|
|
transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
|
|
}
|
|
if (timing.enabled)
|
|
{
|
|
timing.outputRecord = profile_ms(outputRecordStart, ProfileClock::now());
|
|
timing.queueDepth = g.prep.pending() + g.worker.pending() + g.finish.pending() + 1;
|
|
timing.queuedAt = ProfileClock::now();
|
|
timing.gameCall = profile_ms(timing.gameStart, timing.queuedAt);
|
|
}
|
|
|
|
slot.busy = true;
|
|
feature->inFlight.fetch_add(1);
|
|
if (feature->split)
|
|
{
|
|
std::lock_guard<std::mutex> lock(feature->splitMutex);
|
|
feature->splitCompletion.admit(frame);
|
|
}
|
|
g.prep.post([feature, slotIndex, frame, p, timing] { prepare_inputs(feature, slotIndex, frame, p, timing); });
|
|
if (frame <= 3 || frame % 600 == 0 || GetEnvironmentVariableA("D4R_SHIM_TRACE_FRAMES", nullptr, 0) != 0)
|
|
logf("evaluate frame %u queued (slot %d, showing result %d, jitter %.6f,%.6f, reset %d, "
|
|
"mvScale %.3f,%.3f, preExposure %.3f, exposureScale %.3f, frameTime %.3f, sharpness %.3f)",
|
|
frame, slotIndex, latest, p.jitterX, p.jitterY, p.reset, p.mvScaleX, p.mvScaleY,
|
|
p.preExposure, p.exposureScale, p.frameTime, p.sharpness);
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
|
|
void* parameters, void*)
|
|
{
|
|
return evaluate(list, handle, parameters);
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature_C(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
|
|
void* parameters, void*)
|
|
{
|
|
return evaluate(list, handle, parameters);
|
|
}
|
|
|
|
static void release_feature(Feature* feature)
|
|
{
|
|
// Wait for outstanding evaluations, then free CUDA resources on the worker.
|
|
drain_pipeline();
|
|
g.worker.call([feature] {
|
|
if (feature->cudaHandle != nullptr)
|
|
g.ngx.releaseFeature(feature->cudaHandle);
|
|
if (feature->cudaParams != nullptr)
|
|
g.ngx.destroyParameters(feature->cudaParams);
|
|
for (InputSlot& slot : feature->inputs)
|
|
{
|
|
for (CudaObject& texture : slot.linearTexture)
|
|
if (texture != 0)
|
|
g.cu.texObjectDestroy(texture), texture = 0;
|
|
if (slot.dilatedTexture != 0)
|
|
g.cu.texObjectDestroy(slot.dilatedTexture), slot.dilatedTexture = 0;
|
|
if (slot.dilatedMotion != 0)
|
|
g.cu.memFree(slot.dilatedMotion), slot.dilatedMotion = 0;
|
|
}
|
|
for (CudaImage* image : {&feature->color, &feature->depth, &feature->motion, &feature->exposure})
|
|
{
|
|
if (image->object != 0)
|
|
g.cu.texObjectDestroy(image->object);
|
|
if (image->array != nullptr)
|
|
g.cu.arrayDestroy(image->array);
|
|
}
|
|
if (feature->output.object != 0)
|
|
g.cu.surfObjectDestroy(feature->output.object);
|
|
if (feature->output.array != nullptr)
|
|
g.cu.arrayDestroy(feature->output.array);
|
|
if (feature->profileStart != nullptr)
|
|
g.cu.eventDestroy(feature->profileStart);
|
|
if (feature->profileEnd != nullptr)
|
|
g.cu.eventDestroy(feature->profileEnd);
|
|
if (feature->outputReadyEvent != nullptr)
|
|
g.cu.eventDestroy(feature->outputReadyEvent);
|
|
if (feature->scratch != 0)
|
|
g.cu.memFree(feature->scratch);
|
|
for (CudaDevicePtr buffer : feature->planeLinear)
|
|
if (buffer != 0)
|
|
g.cu.memFree(buffer);
|
|
if (feature->outputLinear != 0)
|
|
g.cu.memFree(feature->outputLinear);
|
|
for (InputSlot& slot : feature->inputs)
|
|
for (HostPlane& host : slot.host)
|
|
release_host(host);
|
|
for (HostPlane& host : feature->outputHost)
|
|
release_host(host);
|
|
return 0;
|
|
});
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
g_features.erase(std::remove(g_features.begin(), g_features.end(), feature), g_features.end());
|
|
}
|
|
// Recorded lists retain this owner through their allocator. Reset/discard
|
|
// or GPU completion followed by allocator destruction releases those refs.
|
|
if (feature->resources != nullptr)
|
|
feature->resources->Release();
|
|
else
|
|
delete feature;
|
|
}
|
|
|
|
static void destroy_recorded_resources(Feature* feature)
|
|
{
|
|
for (InputSlot& slot : feature->inputs)
|
|
for (Staging* staging : {&slot.color, &slot.depth, &slot.motion, &slot.exposure})
|
|
staging->release();
|
|
for (OutputSlot& slot : feature->outputs)
|
|
slot.staging.release();
|
|
if (g_vk.ready && feature->vramDecided)
|
|
release_vram(*feature);
|
|
if (feature->marker != nullptr)
|
|
feature->marker->Release();
|
|
logf("feature %u recorded resources retired after allocator completion", feature->handle.Id);
|
|
delete feature;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_ReleaseFeature(NgxHandle* handle)
|
|
{
|
|
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
|
|
Feature* feature = nullptr;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
for (Feature* candidate : g_features)
|
|
if (&candidate->handle == handle && !candidate->retiring.exchange(true))
|
|
{
|
|
feature = candidate;
|
|
break;
|
|
}
|
|
}
|
|
if (feature == nullptr)
|
|
return NGX_FAIL_INVALID_PARAMETER;
|
|
logf("NVSDK_NGX_D3D12_ReleaseFeature(id=%u) after %u completed frames", feature->handle.Id,
|
|
feature->completedFrames.load());
|
|
release_feature(feature);
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
static void finish_shutdown()
|
|
{
|
|
std::lock_guard<std::mutex> lock(g.mutex);
|
|
if (!g.shutdownPending || g.resourceOwners.load() != 0)
|
|
return;
|
|
if (g_vk.ready)
|
|
free_vram_pool();
|
|
if (g.ngxInitialized)
|
|
{
|
|
drain_pipeline();
|
|
g.worker.call([] { return g.ngx.shutdown(); });
|
|
}
|
|
g.ngxInitialized = false;
|
|
g.shutdownPending = false;
|
|
if (g.device != nullptr)
|
|
{
|
|
g.device->Release();
|
|
g.device = nullptr;
|
|
}
|
|
logf("NGX shutdown completed after recorded resources retired");
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown()
|
|
{
|
|
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
|
|
logf("NVSDK_NGX_D3D12_Shutdown");
|
|
std::vector<Feature*> features;
|
|
{
|
|
std::lock_guard<std::mutex> lock(g_featuresMutex);
|
|
features = g_features;
|
|
for (Feature* feature : features)
|
|
feature->retiring = true;
|
|
}
|
|
for (Feature* feature : features)
|
|
release_feature(feature);
|
|
{
|
|
std::lock_guard<std::mutex> lock(g.mutex);
|
|
if (!g.started)
|
|
return NGX_SUCCESS;
|
|
g.shutdownPending = true;
|
|
}
|
|
// Drain already queued cleanup before returning. If command allocators
|
|
// still own resources, finish_shutdown leaves NGX alive until their last
|
|
// reference retires; otherwise shutdown completes synchronously.
|
|
g.cleanup.call([] { finish_shutdown(); return 0; });
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown1(ID3D12Device*)
|
|
{
|
|
return NVSDK_NGX_D3D12_Shutdown();
|
|
}
|
|
|
|
D4R_EXPORT NgxResult NVSDK_NGX_UpdateFeature(const void*, unsigned int)
|
|
{
|
|
return NGX_SUCCESS;
|
|
}
|
|
|
|
BOOL WINAPI DllMain(HINSTANCE instance, DWORD reason, LPVOID)
|
|
{
|
|
if (reason == DLL_PROCESS_ATTACH)
|
|
{
|
|
g_selfModule = instance;
|
|
DisableThreadLibraryCalls(instance);
|
|
}
|
|
return TRUE;
|
|
}
|