Improve GPU compatibility, RDNA2 kernels and startup diagnostics

- Match render/compute adapters and gate RR models by GPU capabilities
- Add RDNA2 native texture kernels and fix FP16 rounding
- Harden model builds, cache validation and packaging
- Add startup tracing, timeouts, documentation and regression tests
This commit is contained in:
ayo committed 2026-10-19 11:36:40 -04:00
1 parent ca93d0fbf7
commit 2558fcbd39
32 files changed
+1131 -81

No files matched your search

+22 -4
View File
@@ -127,14 +127,29 @@ static bool check(HRESULT hr, const char* what)
static void submit_and_wait()
{
g_list->Close();
if (!check(g_list->Close(), "Close command list")) std::_Exit(1);
ID3D12CommandList* lists[] = {g_list};
g_queue->ExecuteCommandLists(1, lists);
g_queue->Signal(g_fence, ++g_fenceValue);
if (!check(g_queue->Signal(g_fence, ++g_fenceValue), "Signal fence")) std::_Exit(1);
if (g_fence->GetCompletedValue() < g_fenceValue)
{
g_fence->SetEventOnCompletion(g_fenceValue, g_event);
WaitForSingleObject(g_event, INFINITE);
if (!check(g_fence->SetEventOnCompletion(g_fenceValue, g_event), "SetEventOnCompletion")) std::_Exit(1);
DWORD timeout = 60000;
if (const char* value = std::getenv("D4R_HARNESS_GPU_TIMEOUT_MS"))
{
char* end = nullptr;
const unsigned long parsed = std::strtoul(value, &end, 10);
if (end != value && *end == '\0' && parsed >= 1 && parsed <= 600000) timeout = parsed;
}
const DWORD waited = WaitForSingleObject(g_event, timeout);
if (waited != WAIT_OBJECT_0)
{
std::fprintf(stderr, "GPU fence wait failed: wait=0x%lx timeout=%lu ms fence=%llu completed=%llu device=0x%08lx\n",
waited, timeout, g_fenceValue, g_fence->GetCompletedValue(), g_device->GetDeviceRemovedReason());
std::fflush(stderr);
// The capture is an isolated diagnostic process. Do not enter teardown with outstanding GPU work.
std::_Exit(waited == WAIT_TIMEOUT ? 124 : 1);
}
}
g_allocator->Reset();
g_list->Reset(g_allocator, nullptr);
@@ -1043,6 +1058,9 @@ static float halton(uint32_t index, uint32_t base)
int main(int argc, char** argv)
{
// The installer redirects output to a file. Keep the last stage visible even if HIP/Wine stalls or crashes.
std::setvbuf(stdout, nullptr, _IONBF, 0);
std::setvbuf(stderr, nullptr, _IONBF, 0);
trace_stage("entered main");
if (argc < 3)
{
+4
View File
@@ -0,0 +1,4 @@
#pragma once
#ifndef D4R_RUNTIME_BUILD_ID
#define D4R_RUNTIME_BUILD_ID "unidentified (built without runtime_build_id.sh)"
#endif
+35
View File
@@ -0,0 +1,35 @@
#pragma once
#include <vulkan/vulkan_core.h>
#include <sstream>
#include <string>
// Inspect metadata before constructing any pipeline. Old default D/E and F models require FP8;
// only the explicitly marked gfx11 D/E models use widened f16 cooperative matrices.
inline bool d4r_rr_model_compatible(const std::string& metadata, bool f16Matrices, bool fp8Matrices)
{
std::istringstream lines(metadata);
std::string line;
if (!std::getline(lines, line)) return false;
if (!line.empty() && line.back() == '\r') line.pop_back();
if (line == "D4RR0002") return fp8Matrices;
if (line != "D4RR0001") return false;
bool widened = false;
while (std::getline(lines, line))
{
std::istringstream fields(line);
std::string key, value; fields >> key >> value;
if (key == "arithmetic")
{
if (value != "gfx11") return false;
widened = true;
}
}
return widened ? f16Matrices : fp8Matrices;
}
inline bool d4r_matrix_tuple(const VkCooperativeMatrixPropertiesKHR& p, VkComponentTypeKHR operand)
{
return p.MSize == 16 && p.NSize == 16 && p.KSize == 16 && p.scope == VK_SCOPE_SUBGROUP_KHR &&
p.AType == operand && p.BType == operand && p.CType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
p.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR && !p.saturatingAccumulation;
}
+7 -1
View File
@@ -73,7 +73,7 @@ inline bool d4r_read_hex_file(const char* path, uint32_t& value)
// The real PCI identity of the GPU behind a vkd3d-proton device: Vulkan's PCI bus address, then that
// device's sysfs vendor/device files.
inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detail)
inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detail, std::string* address = nullptr)
{
D4rPciIdentity identity;
ID3D12DXVKInteropDevice1* interop = nullptr;
@@ -126,6 +126,12 @@ inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detai
return identity;
}
identity.known = true;
if (address != nullptr)
{
char pci[32];
std::snprintf(pci, sizeof(pci), "%04x:%02x:%02x.%x", bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction);
*address = pci;
}
char text[256];
std::snprintf(text, sizeof(text), "PCI %04x:%02x:%02x.%x is %04x:%04x (Vulkan reports %04x:%04x, driver %s)",
bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction, identity.vendor, identity.device,
+26
View File
@@ -0,0 +1,26 @@
/* PCI identity comparisons shared by the Wine bridge and CPU tests. */
#pragma once
#include <stdio.h>
typedef struct D4rPciAddress { unsigned domain, bus, device, function; } D4rPciAddress;
static int d4r_parse_pci(const char* text, D4rPciAddress* out)
{
char extra;
return text != NULL && sscanf(text, "%x:%x:%x.%x%c", &out->domain, &out->bus,
&out->device, &out->function, &extra) == 4 && out->bus <= 255 && out->device <= 31 && out->function <= 7;
}
static int d4r_same_pci(const char* a, const char* b)
{
D4rPciAddress x, y;
return d4r_parse_pci(a, &x) && d4r_parse_pci(b, &y) && x.domain == y.domain &&
x.bus == y.bus && x.device == y.device && x.function == y.function;
}
static int d4r_pci_matches_kfd(const char* pci, unsigned long domain, unsigned long location)
{
D4rPciAddress address;
return d4r_parse_pci(pci, &address) && address.domain == domain &&
((address.bus << 8) | (address.device << 3) | address.function) == location;
}
+59 -3
View File
@@ -101,6 +101,8 @@
#include "d4r_fsr_rr.h"
#include "d4r_fsr_rr_identity.h"
#include "d4r_fsr4_identity.h"
#include "d4r_engine_compat.h"
#include "d4r_build.h"
#include "d4r_fsr_rr_passes.h"
#include "../engine/game.h"
#include "../engine/game_m.h"
@@ -752,7 +754,7 @@ static void load_portable_config()
bool on = false;
if (enabled.empty() || enabled == "auto")
{
for (const wchar_t* model : {L"k", L"k-ldr", L"m", L"m-ldr", L"l", L"l-ldr", L"rr-d", L"rr-e", L"rr-f"})
for (const wchar_t* model : {L"k", L"k-ldr", L"m", L"m-ldr", L"l", L"l-ldr", L"rr-d", L"rr-e", L"rr-f", L"rr-d-gfx11", L"rr-e-gfx11"})
{
const DWORD attributes = GetFileAttributesW((dir + L"\\engine\\" + model).c_str());
if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY))
@@ -956,12 +958,16 @@ public:
return jobs_.size();
}
template <typename Function> auto call(Function function) -> decltype(function())
template <typename Function> auto call(Function function, const char* label = nullptr) -> decltype(function())
{
using Result = decltype(function());
auto task = std::make_shared<std::packaged_task<Result()>>(std::move(function));
auto future = task->get_future();
post([task] { (*task)(); });
const auto started = std::chrono::steady_clock::now();
while (label && future.wait_for(std::chrono::seconds(15)) == std::future_status::timeout)
logf("worker wait: %s still pending after %.0f s (resources retained)", label,
std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count());
return future.get();
}
@@ -1974,6 +1980,7 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
ensure_portable_config();
if (g_portable.active && !g.core)
{
logf("d4r runtime build %s", D4R_RUNTIME_BUILD_ID);
logf("portable install in %ls (settings from d4r.ini, Linux path %s)", g_portable.dir.c_str(),
g_portable.unixDir.c_str());
for (const std::string& note : g_portable.notes)
@@ -1993,6 +2000,11 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
LUID luid;
device->GetAdapterLuid(&luid);
set_cuda_adapter_luid(luid);
std::string adapterDetail, adapterPci;
d4r_true_identity(device, adapterDetail, &adapterPci);
SetEnvironmentVariableA("D4R_RENDER_PCI", adapterPci.empty() ? nullptr : adapterPci.c_str());
logf("CUDA adapter mapping: %s; PCI match %s", adapterDetail.c_str(),
adapterPci.empty() ? "unavailable (multiple compute devices will be refused)" : adapterPci.c_str());
// OptiScaler's FSR 4 option needs the real AMD identity for OptiScaler and AMD's modules (d4r_fsr4_identity.h)
if (env_string("D4R_FSR4_IDENTITY") != "0")
{
@@ -2003,6 +2015,8 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
}
g.paths.clear();
logf("engine selection: D4R_ENGINE=%s D4R_ENGINE_K=%s network=%s (environment overrides d4r.ini)",
env_string("D4R_ENGINE").c_str(), env_string("D4R_ENGINE_K").c_str(), env_string("D4R_ENGINE_NETWORK").c_str());
wchar_t featureDirectory[MAX_PATH] = {};
DWORD length = GetEnvironmentVariableW(L"D4R_NGX_FEATURE_DIR", featureDirectory, MAX_PATH);
if (length > 0 && length < MAX_PATH)
@@ -3404,6 +3418,7 @@ struct VulkanInterop
// shaderBufferInt64Atomics is enabled on vkd3d-proton's device (it enables it when the GPU has it): Ray
// Reconstruction preset F then measures the exposure in one pass, as NGX does, instead of two
bool int64Atomics = false;
bool matrixF16 = false;
// The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects.
bool native = false;
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
@@ -8952,6 +8967,9 @@ static int start_engine_model_rr(const Feature& feature);
// Vulkan-only K features retain just their creation settings until a frame actually needs CUDA.
static NgxResult create_cuda_feature(Feature& feature)
{
const auto createStarted = std::chrono::steady_clock::now();
logf("CUDA_CREATE_BEGIN preset=%u render=%ux%u output=%ux%u flags=0x%x", feature.preset,
feature.width, feature.height, feature.outWidth, feature.outHeight, feature.flags);
static const char* const kSuperSamplingPresets[] = {
"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance",
@@ -8963,6 +8981,7 @@ static NgxResult create_cuda_feature(Feature& feature)
const bool rayReconstruction = feature.rayReconstruction;
const char* const* presetNames = rayReconstruction ? kRayReconstructionPresets : kSuperSamplingPresets;
const NgxResult result = g.worker.call([&]() -> NgxResult {
logf("CUDA_CREATE_WORKER_BEGIN preset=%u", feature.preset);
NgxResult allocation = g.ngx.allocateParameters(&feature.cudaParams);
if (allocation != NGX_SUCCESS)
return allocation;
@@ -9002,8 +9021,10 @@ static NgxResult create_cuda_feature(Feature& feature)
feature.scratch = 0;
// The core owns the denoiser: it loads nvngx_dlssd.dll, knows its kernel map, and
// dispatches feature 13 to it. Only the feature id differs from Super Sampling.
logf("CUDA_CREATE_NGX_BEGIN preset=%u scratch=0x%llx", feature.preset, feature.scratch);
const NgxResult created = g.ngx.createFeature(
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, p, &feature.cudaHandle);
logf("CUDA_CREATE_NGX_END preset=%u result=0x%08x", feature.preset, created);
if (created != NGX_SUCCESS)
{
if (feature.cudaHandle != nullptr) g.ngx.releaseFeature(feature.cudaHandle);
@@ -9062,9 +9083,11 @@ static NgxResult create_cuda_feature(Feature& feature)
}
}
return created;
});
}, "CUDA feature creation");
logf("NVSDK_NGX_CUDA_CreateFeature(feature=%u) -> 0x%08x",
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, result);
logf("CUDA_CREATE_END preset=%u result=0x%08x elapsed_ms=%.1f", feature.preset, result,
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - createStarted).count());
if (result == NGX_SUCCESS) feature.cudaCreationDeferred = false;
return result;
}
@@ -9339,6 +9362,33 @@ static void init_engine_vulkan()
for (uint32_t index = 0; index < count; ++index)
g_vk.float8 |= std::strcmp(extensions[index].extensionName, "VK_EXT_shader_float8") == 0;
}
auto features2 = reinterpret_cast<PFN_vkGetPhysicalDeviceFeatures2>(instanceProc(instance, "vkGetPhysicalDeviceFeatures2"));
auto matrices = reinterpret_cast<PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR>(
instanceProc(instance, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR"));
bool fp8 = false;
if (features2 && matrices)
{
VkPhysicalDeviceShaderFloat8FeaturesEXT f8 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT};
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cm = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR};
cm.pNext = g_vk.float8 ? &f8 : nullptr;
VkPhysicalDeviceFeatures2 features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &cm};
features2(g_vk.physical, &features);
uint32_t n = 0;
if (cm.cooperativeMatrix && matrices(g_vk.physical, &n, nullptr) == VK_SUCCESS)
{
std::vector<VkCooperativeMatrixPropertiesKHR> properties(n, {VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR});
if (matrices(g_vk.physical, &n, properties.data()) == VK_SUCCESS)
for (uint32_t i = 0; i < n; ++i)
{
g_vk.matrixF16 |= d4r_matrix_tuple(properties[i], VK_COMPONENT_TYPE_FLOAT16_KHR);
fp8 |= d4r_matrix_tuple(properties[i], VK_COMPONENT_TYPE_FLOAT8_E4M3_EXT);
}
}
fp8 = fp8 && f8.shaderFloat8 && f8.shaderFloat8CooperativeMatrix;
}
g_vk.float8 = g_vk.float8 && fp8;
logf("engine backend: f16 matrix tuple %s; FP8 matrix tuple %s", g_vk.matrixF16 ? "supported" : "unsupported",
g_vk.float8 ? "supported" : "unsupported");
logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: "
"M and L run on the CUDA backend");
// the features vkd3d-proton created its device with
@@ -10328,6 +10378,8 @@ static std::string rr_model_directory(int slot)
char folder[MAX_PATH * 3] = {};
WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + folders[slot]).c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
directory = folder;
if (slot < 2 && !g_vk.float8)
directory += "-gfx11";
}
return directory[0] == '/' ? "Z:" + directory : directory;
}
@@ -10346,6 +10398,10 @@ static void init_engine_backend_rr(int slot)
const auto start = std::chrono::steady_clock::now();
try
{
std::string metadata;
if (!read_whole_file(widen(path + "\\rr.txt"), metadata) ||
!d4r_rr_model_compatible(metadata, g_vk.matrixF16, g_vk.float8))
throw std::runtime_error("model arithmetic is unsupported by this GPU (no pipelines created)");
auto model = std::make_unique<d4r::RModel>(g_vk.physical, g_vk.device, path, VK_NULL_HANDLE, g_vk.int64Atomics);
if (model->preset() != g_rrPreset[slot])
throw std::runtime_error("the directory holds a model of another preset");
+7
View File
@@ -620,6 +620,13 @@ static NgxResult native_vk_init(unsigned long long id, const wchar_t* path, VkIn
VkPhysicalDeviceIDProperties identity = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES};
VkPhysicalDeviceProperties2 properties = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity};
properties2(physical, &properties);
VkPhysicalDevicePCIBusInfoPropertiesEXT bus = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT};
properties.pNext = &bus;
properties2(physical, &properties);
char pci[32] = {};
if (bus.pciDomain || bus.pciBus || bus.pciDevice || bus.pciFunction)
std::snprintf(pci, sizeof(pci), "%04x:%02x:%02x.%x", bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction);
SetEnvironmentVariableA("D4R_RENDER_PCI", pci[0] ? pci : nullptr);
if (identity.deviceLUIDValid)
{
uint32_t low, high;
+137 -36
View File
@@ -13,6 +13,8 @@
#include <unistd.h>
#include <time.h>
#include "d4r_native_selection.h"
#include "d4r_gpu_identity.h"
#include "d4r_build.h"
#include "d4r_motion_dilation.h"
#include "d4r_native_motion.h"
#include "d4r_wire_history.h"
@@ -430,46 +432,54 @@ static void native_cleanup(void)
remove_directory(native_served);
}
/* The gfx target ("gfx1101") of the GPU DLSS runs on, "" when unknown: D4R_GPU_ARCH if set, else the KFD
topology's GPU with the most SIMDs. With an integrated and a discrete GPU (Ryzen 7000/9000 desktops) the
first KFD GPU node can be the integrated one, whose kernels would not match the discrete GPU. */
/* Use the rendering adapter's PCI address, never the first/largest KFD GPU. Without an authenticated
address a single GPU is unambiguous; multiple GPUs use translated kernels instead of a guessed target. */
static void gpu_architecture(char* out, size_t size)
{
out[0] = '\0';
const char* forced = getenv("D4R_GPU_ARCH");
if (forced != NULL && strncmp(forced, "gfx", 3) == 0)
char pci[64] = {}, only[32] = {};
GetEnvironmentVariableA("D4R_RENDER_PCI", pci, sizeof(pci));
int gpus = 0, matches = 0;
DIR* nodes = opendir("/sys/class/kfd/kfd/topology/nodes");
if (nodes == NULL) return;
for (struct dirent* node; (node = readdir(nodes)) != NULL;)
{
snprintf(out, size, "%s", forced);
return;
}
unsigned long best_simds = 0;
int gpus = 0;
for (int node = 0; node < 16; ++node)
{
char path[128];
snprintf(path, sizeof(path), "/sys/class/kfd/kfd/topology/nodes/%d/properties", node);
if (node->d_name[0] == '.') continue;
char path[512];
snprintf(path, sizeof(path), "/sys/class/kfd/kfd/topology/nodes/%s/properties", node->d_name);
FILE* properties = fopen(path, "r");
if (properties == NULL)
continue;
char line[256];
unsigned long simds = 0, target = 0;
unsigned long simds = 0, target = 0, domain = ~0ul, location = ~0ul;
while (fgets(line, sizeof(line), properties) != NULL)
{
sscanf(line, "simd_count %lu", &simds);
sscanf(line, "gfx_target_version %lu", &target);
sscanf(line, "domain %lu", &domain);
sscanf(line, "location_id %lu", &location);
}
fclose(properties);
if (simds == 0 || target == 0)
continue;
++gpus;
if (simds > best_simds)
snprintf(only, sizeof(only), "gfx%lu%lu%lx", target / 10000, (target / 100) % 100, target % 100);
if (d4r_pci_matches_kfd(pci, domain, location))
{
best_simds = simds;
snprintf(out, size, "gfx%lu%lu%lx", target / 10000, (target / 100) % 100, target % 100);
++matches;
snprintf(out, size, "%s", only);
}
}
if (gpus > 1)
tracef("native kernels: %d GPUs; using the one with the most SIMDs (%s); D4R_GPU_ARCH overrides", gpus, out);
closedir(nodes);
if (!pci[0] && gpus == 1) snprintf(out, size, "%s", only);
else if (matches != 1) out[0] = '\0';
const char* forced = getenv("D4R_GPU_ARCH");
if (forced != NULL && strcmp(forced, out) != 0)
{
tracef("D4R_GPU_ARCH=%s does not match the rendering GPU (%s); native kernels disabled", forced, out);
out[0] = '\0';
}
tracef("GPU target: render PCI=%s KFD GPUs=%d matches=%d architecture=%s", pci, gpus, matches, out);
}
static void prepare_native_kernels(const char* cache_home)
@@ -480,6 +490,11 @@ static void prepare_native_kernels(const char* cache_home)
/* a release directory holds one manifest and set of code objects per GPU target */
char architecture[32], source[1024], path[1200];
gpu_architecture(architecture, sizeof(architecture));
if (!architecture[0])
{
unsetenv("D4R_ZLUDA_NATIVE_DIR");
return;
}
FILE* manifest = NULL;
/* RDNA4 with native FP8 WMMA (D4R_ZLUDA_WMMA_FP8_NATIVE=1, d4r.ini NativeFp8): the <target>-fp8 variant,
whose kernels match ZLUDA's FP8 lowering; no other GPU has such a directory */
@@ -1385,8 +1400,12 @@ static void tracef(const char* format, ...)
vsnprintf(line, sizeof(line), format, args);
va_end(args);
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
const double seconds = (double)now.tv_sec + (double)now.tv_nsec / 1000000000.0;
const unsigned long thread = GetCurrentThreadId();
pthread_mutex_lock(&trace_lock);
fprintf(stderr, "[d4r nvcuda] %s\n", line);
fprintf(stderr, "[d4r nvcuda] [%.6f tid=%lu] %s\n", seconds, thread, line);
fflush(stderr);
char path[MAX_PATH] = {0};
@@ -1396,9 +1415,14 @@ static void tracef(const char* format, ...)
FILE* output = fopen(path, "a");
if (output != NULL)
{
fprintf(output, "%s\n", line);
fprintf(output, "[%.6f tid=%lu] %s\n", seconds, thread, line);
fclose(output);
}
else
{
static int warned;
if (!warned++) fprintf(stderr, "[d4r nvcuda] cannot open D4R_CUDA_TRACE=%s: %s\n", path, strerror(errno));
}
}
pthread_mutex_unlock(&trace_lock);
}
@@ -1473,7 +1497,8 @@ typedef CUresult(__attribute__((sysv_abi)) *CUGETERRORSTRING_FN)(CUresult, const
typedef CUresult(__attribute__((sysv_abi)) *CUGETPROCADDRESS_FN)(const char*, void**, int, uint64_t, int*);
static pthread_once_t context_once = PTHREAD_ONCE_INIT;
static CUresult context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
static CUresult context_bootstrap_result = CUDA_ERROR_NOT_INITIALIZED;
static _Thread_local CUresult context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
static CUcontext cuda_context;
static unsigned int get_process_u32(const char* name, unsigned int fallback)
@@ -1487,6 +1512,36 @@ static unsigned int get_process_u32(const char* name, unsigned int fallback)
return end != value && *end == '\0' ? (unsigned int)parsed : fallback;
}
/* This calls ZLUDA directly: bridge wrappers would recurse through context bootstrap. */
static CUresult selected_cuda_device(CUdevice* selected)
{
CUDEVICEGETCOUNT_FN count_fn = (CUDEVICEGETCOUNT_FN)find_zluda_symbol("cuDeviceGetCount");
CUDEVICEGET_FN get_fn = (CUDEVICEGET_FN)find_zluda_symbol("cuDeviceGet");
typedef CUresult(__attribute__((sysv_abi)) * PCI_FN)(char*, int, CUdevice);
PCI_FN pci_fn = (PCI_FN)find_zluda_symbol("cuDeviceGetPCIBusId");
int count = 0;
if (!count_fn || !get_fn || count_fn(&count) != CUDA_SUCCESS || count < 1)
return CUDA_ERROR_NOT_INITIALIZED;
char expected[64] = {};
GetEnvironmentVariableA("D4R_RENDER_PCI", expected, sizeof(expected));
if (!expected[0] && count == 1) return get_fn(selected, 0);
if (!expected[0] || !pci_fn) return CUDA_ERROR_NOT_SUPPORTED;
int matches = 0;
for (int ordinal = 0; ordinal < count; ++ordinal)
{
CUdevice device;
char actual[64] = {};
if (get_fn(&device, ordinal) == CUDA_SUCCESS && pci_fn(actual, sizeof(actual), device) == CUDA_SUCCESS)
{
tracef("compute adapter ordinal=%d device=%d PCI=%s render=%s", ordinal, device, actual, expected);
if (d4r_same_pci(expected, actual)) { *selected = device; ++matches; }
}
}
if (matches != 1)
tracef("CUDA adapter mapping refused: render PCI=%s matches=%d; no GPU will be guessed", expected, matches);
return matches == 1 ? CUDA_SUCCESS : CUDA_ERROR_NOT_SUPPORTED;
}
static void create_default_zluda_context(void)
{
CUINIT_FN init = (CUINIT_FN)find_zluda_symbol("cuInit");
@@ -1499,30 +1554,30 @@ static void create_default_zluda_context(void)
return;
}
context_setup_result = init(0);
if (context_setup_result != CUDA_SUCCESS)
context_bootstrap_result = init(0);
if (context_bootstrap_result != CUDA_SUCCESS)
{
tracef("cuInit returned %d", context_setup_result);
tracef("cuInit returned %d", context_bootstrap_result);
return;
}
int device_count = 0;
context_setup_result = get_device_count(&device_count);
if (context_setup_result != CUDA_SUCCESS || device_count < 1)
context_bootstrap_result = get_device_count(&device_count);
if (context_bootstrap_result != CUDA_SUCCESS || device_count < 1)
{
tracef("cuDeviceGetCount returned %d, count %d", context_setup_result, device_count);
if (context_setup_result == CUDA_SUCCESS)
context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
tracef("cuDeviceGetCount returned %d, count %d", context_bootstrap_result, device_count);
if (context_bootstrap_result == CUDA_SUCCESS)
context_bootstrap_result = CUDA_ERROR_NOT_INITIALIZED;
return;
}
CUdevice device = 0;
context_setup_result = get_device(&device, 0);
if (context_setup_result == CUDA_SUCCESS)
context_setup_result = create_context(&cuda_context, 0, device);
if (context_setup_result != CUDA_SUCCESS)
context_bootstrap_result = selected_cuda_device(&device);
if (context_bootstrap_result == CUDA_SUCCESS)
context_bootstrap_result = create_context(&cuda_context, 0, device);
if (context_bootstrap_result != CUDA_SUCCESS)
{
tracef("context creation returned %d", context_setup_result);
tracef("context creation returned %d", context_bootstrap_result);
return;
}
@@ -1531,7 +1586,21 @@ static void create_default_zluda_context(void)
static void ensure_context(void)
{
typedef CUresult(__attribute__((sysv_abi)) * CURRENT_FN)(CUcontext*);
CURRENT_FN current = (CURRENT_FN)find_zluda_symbol("cuCtxGetCurrent");
CUcontext active = NULL;
if (current && current(&active) == CUDA_SUCCESS && active != NULL)
{
context_setup_result = CUDA_SUCCESS;
return; /* NGX already selected a context; do not replace it with a second context. */
}
pthread_once(&context_once, create_default_zluda_context);
context_setup_result = context_bootstrap_result;
if (context_setup_result == CUDA_SUCCESS && cuda_context != NULL)
{
CUCTX_CURRENT_FN set = (CUCTX_CURRENT_FN)find_zluda_symbol("cuCtxSetCurrent");
if (set) context_setup_result = set(cuda_context);
}
}
/* D4R_NGX_STREAM_PRIORITY=high|normal (default off): the null-stream work of NGX, the shim and this bridge
@@ -2567,6 +2636,7 @@ static int before_teardown(void)
CUresult WINAPI cuInit(unsigned int flags)
{
tracef("bridge build %s; CUDA_BEGIN api=cuInit flags=%u", D4R_RUNTIME_BUILD_ID, flags);
CUINIT_FN function = (CUINIT_FN)find_zluda_symbol("cuInit");
CUresult result = function != NULL ? function(flags) : CUDA_ERROR_NOT_INITIALIZED;
tracef("cuInit flags=%u result=%d", flags, result);
@@ -2618,6 +2688,14 @@ CUresult WINAPI cuCtxGetDevice(CUdevice* device)
CUresult WINAPI cuCtxCreate_v2(CUcontext* context, unsigned int flags, CUdevice device)
{
CUdevice selected = -1;
CUresult mapping = selected_cuda_device(&selected);
if (mapping != CUDA_SUCCESS || selected != device)
{
if (context) *context = NULL;
tracef("CUDA context refused: device=%d mapped render device=%d status=%d", device, selected, mapping);
return mapping == CUDA_SUCCESS ? CUDA_ERROR_NOT_SUPPORTED : mapping;
}
CUCTXCREATE_FN function = (CUCTXCREATE_FN)find_zluda_symbol("cuCtxCreate_v2");
CUresult result = function != NULL ? function(context, flags, device) : CUDA_ERROR_NOT_INITIALIZED;
tracef("cuCtxCreate_v2 flags=0x%x device=%d result=%d context=%p", flags, device,
@@ -2659,6 +2737,7 @@ static int elide_ngx_sync(void)
CUresult WINAPI d4rCtxSynchronize(void)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=d4rCtxSynchronize");
if (before_teardown())
return CUDA_SUCCESS;
CUCTX_SYNCHRONIZE_FN function = (CUCTX_SYNCHRONIZE_FN)find_zluda_symbol("cuCtxSynchronize");
@@ -2671,6 +2750,7 @@ CUresult WINAPI d4rCtxSynchronize(void)
CUresult WINAPI cuCtxSynchronize(void)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuCtxSynchronize");
if (before_teardown())
return CUDA_SUCCESS;
if (elide_ngx_sync())
@@ -2695,6 +2775,15 @@ CUresult WINAPI cuDeviceGetLuid(char* luid, unsigned int* device_node_mask, CUde
if ((result == CUDA_SUCCESS || result == CUDA_ERROR_NOT_SUPPORTED) &&
luid != NULL && device_node_mask != NULL)
{
CUdevice selected = -1;
CUresult mapping = selected_cuda_device(&selected);
if (mapping != CUDA_SUCCESS || selected != device)
{
if (luid) memset(luid, 0, 8);
if (device_node_mask) *device_node_mask = 0;
tracef("cuDeviceGetLuid device=%d is not the mapped render adapter (selected=%d status=%d)", device, selected, mapping);
return mapping == CUDA_SUCCESS ? CUDA_SUCCESS : mapping;
}
unsigned int low = get_process_u32("D4R_CUDA_LUID_LOW", 0xffffffffu);
unsigned int high = get_process_u32("D4R_CUDA_LUID_HIGH", 0xffffffffu);
unsigned int node_mask = get_process_u32("D4R_CUDA_NODE_MASK", 0xffffffffu);
@@ -2791,6 +2880,7 @@ static const void* replacement_image(const void* image, const char* api)
CUresult WINAPI cuModuleLoadData(CUmodule* module, const void* image)
{
tracef("CUDA_BEGIN api=cuModuleLoadData image=%p", image);
CUMODULELOADDATA_FN function = (CUMODULELOADDATA_FN)find_zluda_symbol("cuModuleLoadData");
verify_native_kernels(image);
capture_ptx_if_present(image, "cuModuleLoadData");
@@ -2804,6 +2894,7 @@ CUresult WINAPI cuModuleLoadData(CUmodule* module, const void* image)
CUresult WINAPI cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int option_count,
int* options, void** option_values)
{
tracef("CUDA_BEGIN api=cuModuleLoadDataEx image=%p options=%u", image, option_count);
CUMODULELOADDATAEX_FN function = (CUMODULELOADDATAEX_FN)find_zluda_symbol("cuModuleLoadDataEx");
verify_native_kernels(image);
capture_ptx_if_present(image, "cuModuleLoadDataEx");
@@ -2817,6 +2908,7 @@ CUresult WINAPI cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned
CUresult WINAPI cuModuleLoad(CUmodule* module, const char* filename)
{
tracef("CUDA_BEGIN api=cuModuleLoad file=%s", filename != NULL ? filename : "<null>");
CUMODULELOAD_FN function = (CUMODULELOAD_FN)find_zluda_symbol("cuModuleLoad");
CUresult result = function != NULL ? function(module, filename) : missing("cuModuleLoad");
tracef("cuModuleLoad file=%s result=%d module=%p", filename != NULL ? filename : "<null>",
@@ -2837,6 +2929,7 @@ CUresult WINAPI cuModuleUnload(CUmodule module)
CUresult WINAPI cuModuleGetFunction(CUfunction* function_out, CUmodule module, const char* name)
{
tracef("CUDA_BEGIN api=cuModuleGetFunction module=%p name=%s", module, name ? name : "<null>");
CUMODULEGETFUNCTION_FN function = (CUMODULEGETFUNCTION_FN)find_zluda_symbol("cuModuleGetFunction");
CUresult result = function != NULL ? function(function_out, module, name) : missing("cuModuleGetFunction");
if (result == CUDA_SUCCESS && function_out != NULL)
@@ -2882,6 +2975,7 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
unsigned int block_z, unsigned int shared_bytes, CUstream stream,
void** kernel_params, void** extra)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuLaunchKernel function=%p", function_handle);
static CULAUNCHKERNEL_FN function;
if (function == NULL)
function = (CULAUNCHKERNEL_FN)find_zluda_symbol("cuLaunchKernel");
@@ -2964,6 +3058,7 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
CUresult WINAPI cuMemAlloc(CUdeviceptr* pointer, size_t bytes)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemAlloc bytes=%zu", bytes);
CUMEMALLOC_FN function = (CUMEMALLOC_FN)find_zluda_symbol("cuMemAlloc_v2");
CUresult result = function != NULL ? function(pointer, bytes) : missing("cuMemAlloc_v2");
if (result == CUDA_SUCCESS && pointer != NULL)
@@ -3022,6 +3117,8 @@ CUresult WINAPI cuMemcpy2D(const void* copy)
CUresult WINAPI cuMemcpyHtoDAsync(CUdeviceptr destination, const void* source, size_t bytes, CUstream stream)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemcpyHtoDAsync bytes=%zu dst=0x%llx stream=%p", bytes,
(unsigned long long)destination, stream);
CUMEMCPYHTODASYNC_FN function = (CUMEMCPYHTODASYNC_FN)find_zluda_symbol("cuMemcpyHtoDAsync_v2");
if (microcuda_k_write) microcuda_k_write(destination, bytes);
stream = d4r_ngx_stream(stream);
@@ -3033,6 +3130,7 @@ CUresult WINAPI cuMemcpyHtoDAsync(CUdeviceptr destination, const void* source, s
CUresult WINAPI cuMemcpyDtoH(void* destination, CUdeviceptr source, size_t bytes)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemcpyDtoH bytes=%zu src=0x%llx", bytes, (unsigned long long)source);
CUMEMCPYDTOH_FN function = (CUMEMCPYDTOH_FN)find_zluda_symbol("cuMemcpyDtoH_v2");
CUresult result = function != NULL ? function(destination, source, bytes) : missing("cuMemcpyDtoH_v2");
TRACE_CALL(result, "cuMemcpyDtoH->v2 dst=%p src=0x%llx bytes=%zu result=%d",
@@ -3378,6 +3476,7 @@ static int d4r_forward_is_teardown(const char* target)
#define D4R_FORWARD(name, target, count, params, args) \
CUresult WINAPI name params \
{ \
if (trace_verbose()) tracef("CUDA_BEGIN api=" #name); \
if (d4r_forward_is_teardown(#target) && before_teardown()) \
return CUDA_SUCCESS; \
typedef CUresult(__attribute__((sysv_abi)) * function_type) params; \
@@ -3394,6 +3493,7 @@ D4R_FORWARD(cuEventRecord, cuEventRecord, 2, (D4rArg a0, D4rArg a1), (a0, (D4rAr
D4R_FORWARD(cuEventRecordWithFlags, cuEventRecordWithFlags, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, (D4rArg)d4r_ngx_stream((CUstream)a1), a2))
CUresult WINAPI cuEventSynchronize(D4rArg a0)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=cuEventSynchronize event=%p", (void*)a0);
if (before_teardown())
return CUDA_SUCCESS;
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
@@ -3411,6 +3511,7 @@ CUresult WINAPI cuEventSynchronize(D4rArg a0)
game's split command list only after this event has actually completed. */
CUresult WINAPI d4rEventSynchronize(D4rArg a0)
{
if (trace_verbose()) tracef("CUDA_BEGIN api=d4rEventSynchronize event=%p", (void*)a0);
if (before_teardown())
return CUDA_SUCCESS;
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);