Improve GPU compatibility, RDNA2 kernels and startup diagnostics
- Match render/compute adapters and gate RR models by GPU capabilities - Add RDNA2 native texture kernels and fix FP16 rounding - Harden model builds, cache validation and packaging - Add startup tracing, timeouts, documentation and regression tests
This commit is contained in:
1 parent
ca93d0fbf7
commit
2558fcbd39
32 files changed
+1131
-81
No files matched your search
@@ -127,14 +127,29 @@ static bool check(HRESULT hr, const char* what)
|
||||
|
||||
static void submit_and_wait()
|
||||
{
|
||||
g_list->Close();
|
||||
if (!check(g_list->Close(), "Close command list")) std::_Exit(1);
|
||||
ID3D12CommandList* lists[] = {g_list};
|
||||
g_queue->ExecuteCommandLists(1, lists);
|
||||
g_queue->Signal(g_fence, ++g_fenceValue);
|
||||
if (!check(g_queue->Signal(g_fence, ++g_fenceValue), "Signal fence")) std::_Exit(1);
|
||||
if (g_fence->GetCompletedValue() < g_fenceValue)
|
||||
{
|
||||
g_fence->SetEventOnCompletion(g_fenceValue, g_event);
|
||||
WaitForSingleObject(g_event, INFINITE);
|
||||
if (!check(g_fence->SetEventOnCompletion(g_fenceValue, g_event), "SetEventOnCompletion")) std::_Exit(1);
|
||||
DWORD timeout = 60000;
|
||||
if (const char* value = std::getenv("D4R_HARNESS_GPU_TIMEOUT_MS"))
|
||||
{
|
||||
char* end = nullptr;
|
||||
const unsigned long parsed = std::strtoul(value, &end, 10);
|
||||
if (end != value && *end == '\0' && parsed >= 1 && parsed <= 600000) timeout = parsed;
|
||||
}
|
||||
const DWORD waited = WaitForSingleObject(g_event, timeout);
|
||||
if (waited != WAIT_OBJECT_0)
|
||||
{
|
||||
std::fprintf(stderr, "GPU fence wait failed: wait=0x%lx timeout=%lu ms fence=%llu completed=%llu device=0x%08lx\n",
|
||||
waited, timeout, g_fenceValue, g_fence->GetCompletedValue(), g_device->GetDeviceRemovedReason());
|
||||
std::fflush(stderr);
|
||||
// The capture is an isolated diagnostic process. Do not enter teardown with outstanding GPU work.
|
||||
std::_Exit(waited == WAIT_TIMEOUT ? 124 : 1);
|
||||
}
|
||||
}
|
||||
g_allocator->Reset();
|
||||
g_list->Reset(g_allocator, nullptr);
|
||||
@@ -1043,6 +1058,9 @@ static float halton(uint32_t index, uint32_t base)
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
// The installer redirects output to a file. Keep the last stage visible even if HIP/Wine stalls or crashes.
|
||||
std::setvbuf(stdout, nullptr, _IONBF, 0);
|
||||
std::setvbuf(stderr, nullptr, _IONBF, 0);
|
||||
trace_stage("entered main");
|
||||
if (argc < 3)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
#pragma once
|
||||
#ifndef D4R_RUNTIME_BUILD_ID
|
||||
#define D4R_RUNTIME_BUILD_ID "unidentified (built without runtime_build_id.sh)"
|
||||
#endif
|
||||
@@ -0,0 +1,35 @@
|
||||
#pragma once
|
||||
#include <vulkan/vulkan_core.h>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
// Inspect metadata before constructing any pipeline. Old default D/E and F models require FP8;
|
||||
// only the explicitly marked gfx11 D/E models use widened f16 cooperative matrices.
|
||||
inline bool d4r_rr_model_compatible(const std::string& metadata, bool f16Matrices, bool fp8Matrices)
|
||||
{
|
||||
std::istringstream lines(metadata);
|
||||
std::string line;
|
||||
if (!std::getline(lines, line)) return false;
|
||||
if (!line.empty() && line.back() == '\r') line.pop_back();
|
||||
if (line == "D4RR0002") return fp8Matrices;
|
||||
if (line != "D4RR0001") return false;
|
||||
bool widened = false;
|
||||
while (std::getline(lines, line))
|
||||
{
|
||||
std::istringstream fields(line);
|
||||
std::string key, value; fields >> key >> value;
|
||||
if (key == "arithmetic")
|
||||
{
|
||||
if (value != "gfx11") return false;
|
||||
widened = true;
|
||||
}
|
||||
}
|
||||
return widened ? f16Matrices : fp8Matrices;
|
||||
}
|
||||
|
||||
inline bool d4r_matrix_tuple(const VkCooperativeMatrixPropertiesKHR& p, VkComponentTypeKHR operand)
|
||||
{
|
||||
return p.MSize == 16 && p.NSize == 16 && p.KSize == 16 && p.scope == VK_SCOPE_SUBGROUP_KHR &&
|
||||
p.AType == operand && p.BType == operand && p.CType == VK_COMPONENT_TYPE_FLOAT32_KHR &&
|
||||
p.ResultType == VK_COMPONENT_TYPE_FLOAT32_KHR && !p.saturatingAccumulation;
|
||||
}
|
||||
@@ -73,7 +73,7 @@ inline bool d4r_read_hex_file(const char* path, uint32_t& value)
|
||||
|
||||
// The real PCI identity of the GPU behind a vkd3d-proton device: Vulkan's PCI bus address, then that
|
||||
// device's sysfs vendor/device files.
|
||||
inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detail)
|
||||
inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detail, std::string* address = nullptr)
|
||||
{
|
||||
D4rPciIdentity identity;
|
||||
ID3D12DXVKInteropDevice1* interop = nullptr;
|
||||
@@ -126,6 +126,12 @@ inline D4rPciIdentity d4r_true_identity(ID3D12Device* device, std::string& detai
|
||||
return identity;
|
||||
}
|
||||
identity.known = true;
|
||||
if (address != nullptr)
|
||||
{
|
||||
char pci[32];
|
||||
std::snprintf(pci, sizeof(pci), "%04x:%02x:%02x.%x", bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction);
|
||||
*address = pci;
|
||||
}
|
||||
char text[256];
|
||||
std::snprintf(text, sizeof(text), "PCI %04x:%02x:%02x.%x is %04x:%04x (Vulkan reports %04x:%04x, driver %s)",
|
||||
bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction, identity.vendor, identity.device,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/* PCI identity comparisons shared by the Wine bridge and CPU tests. */
|
||||
#pragma once
|
||||
#include <stdio.h>
|
||||
|
||||
typedef struct D4rPciAddress { unsigned domain, bus, device, function; } D4rPciAddress;
|
||||
|
||||
static int d4r_parse_pci(const char* text, D4rPciAddress* out)
|
||||
{
|
||||
char extra;
|
||||
return text != NULL && sscanf(text, "%x:%x:%x.%x%c", &out->domain, &out->bus,
|
||||
&out->device, &out->function, &extra) == 4 && out->bus <= 255 && out->device <= 31 && out->function <= 7;
|
||||
}
|
||||
|
||||
static int d4r_same_pci(const char* a, const char* b)
|
||||
{
|
||||
D4rPciAddress x, y;
|
||||
return d4r_parse_pci(a, &x) && d4r_parse_pci(b, &y) && x.domain == y.domain &&
|
||||
x.bus == y.bus && x.device == y.device && x.function == y.function;
|
||||
}
|
||||
|
||||
static int d4r_pci_matches_kfd(const char* pci, unsigned long domain, unsigned long location)
|
||||
{
|
||||
D4rPciAddress address;
|
||||
return d4r_parse_pci(pci, &address) && address.domain == domain &&
|
||||
((address.bus << 8) | (address.device << 3) | address.function) == location;
|
||||
}
|
||||
@@ -101,6 +101,8 @@
|
||||
#include "d4r_fsr_rr.h"
|
||||
#include "d4r_fsr_rr_identity.h"
|
||||
#include "d4r_fsr4_identity.h"
|
||||
#include "d4r_engine_compat.h"
|
||||
#include "d4r_build.h"
|
||||
#include "d4r_fsr_rr_passes.h"
|
||||
#include "../engine/game.h"
|
||||
#include "../engine/game_m.h"
|
||||
@@ -752,7 +754,7 @@ static void load_portable_config()
|
||||
bool on = false;
|
||||
if (enabled.empty() || enabled == "auto")
|
||||
{
|
||||
for (const wchar_t* model : {L"k", L"k-ldr", L"m", L"m-ldr", L"l", L"l-ldr", L"rr-d", L"rr-e", L"rr-f"})
|
||||
for (const wchar_t* model : {L"k", L"k-ldr", L"m", L"m-ldr", L"l", L"l-ldr", L"rr-d", L"rr-e", L"rr-f", L"rr-d-gfx11", L"rr-e-gfx11"})
|
||||
{
|
||||
const DWORD attributes = GetFileAttributesW((dir + L"\\engine\\" + model).c_str());
|
||||
if (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY))
|
||||
@@ -956,12 +958,16 @@ public:
|
||||
return jobs_.size();
|
||||
}
|
||||
|
||||
template <typename Function> auto call(Function function) -> decltype(function())
|
||||
template <typename Function> auto call(Function function, const char* label = nullptr) -> decltype(function())
|
||||
{
|
||||
using Result = decltype(function());
|
||||
auto task = std::make_shared<std::packaged_task<Result()>>(std::move(function));
|
||||
auto future = task->get_future();
|
||||
post([task] { (*task)(); });
|
||||
const auto started = std::chrono::steady_clock::now();
|
||||
while (label && future.wait_for(std::chrono::seconds(15)) == std::future_status::timeout)
|
||||
logf("worker wait: %s still pending after %.0f s (resources retained)", label,
|
||||
std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count());
|
||||
return future.get();
|
||||
}
|
||||
|
||||
@@ -1974,6 +1980,7 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
|
||||
ensure_portable_config();
|
||||
if (g_portable.active && !g.core)
|
||||
{
|
||||
logf("d4r runtime build %s", D4R_RUNTIME_BUILD_ID);
|
||||
logf("portable install in %ls (settings from d4r.ini, Linux path %s)", g_portable.dir.c_str(),
|
||||
g_portable.unixDir.c_str());
|
||||
for (const std::string& note : g_portable.notes)
|
||||
@@ -1993,6 +2000,11 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
|
||||
LUID luid;
|
||||
device->GetAdapterLuid(&luid);
|
||||
set_cuda_adapter_luid(luid);
|
||||
std::string adapterDetail, adapterPci;
|
||||
d4r_true_identity(device, adapterDetail, &adapterPci);
|
||||
SetEnvironmentVariableA("D4R_RENDER_PCI", adapterPci.empty() ? nullptr : adapterPci.c_str());
|
||||
logf("CUDA adapter mapping: %s; PCI match %s", adapterDetail.c_str(),
|
||||
adapterPci.empty() ? "unavailable (multiple compute devices will be refused)" : adapterPci.c_str());
|
||||
// OptiScaler's FSR 4 option needs the real AMD identity for OptiScaler and AMD's modules (d4r_fsr4_identity.h)
|
||||
if (env_string("D4R_FSR4_IDENTITY") != "0")
|
||||
{
|
||||
@@ -2003,6 +2015,8 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat
|
||||
}
|
||||
|
||||
g.paths.clear();
|
||||
logf("engine selection: D4R_ENGINE=%s D4R_ENGINE_K=%s network=%s (environment overrides d4r.ini)",
|
||||
env_string("D4R_ENGINE").c_str(), env_string("D4R_ENGINE_K").c_str(), env_string("D4R_ENGINE_NETWORK").c_str());
|
||||
wchar_t featureDirectory[MAX_PATH] = {};
|
||||
DWORD length = GetEnvironmentVariableW(L"D4R_NGX_FEATURE_DIR", featureDirectory, MAX_PATH);
|
||||
if (length > 0 && length < MAX_PATH)
|
||||
@@ -3404,6 +3418,7 @@ struct VulkanInterop
|
||||
// shaderBufferInt64Atomics is enabled on vkd3d-proton's device (it enables it when the GPU has it): Ray
|
||||
// Reconstruction preset F then measures the exposure in one pass, as NGX does, instead of two
|
||||
bool int64Atomics = false;
|
||||
bool matrixF16 = false;
|
||||
// The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects.
|
||||
bool native = false;
|
||||
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
|
||||
@@ -8952,6 +8967,9 @@ static int start_engine_model_rr(const Feature& feature);
|
||||
// Vulkan-only K features retain just their creation settings until a frame actually needs CUDA.
|
||||
static NgxResult create_cuda_feature(Feature& feature)
|
||||
{
|
||||
const auto createStarted = std::chrono::steady_clock::now();
|
||||
logf("CUDA_CREATE_BEGIN preset=%u render=%ux%u output=%ux%u flags=0x%x", feature.preset,
|
||||
feature.width, feature.height, feature.outWidth, feature.outHeight, feature.flags);
|
||||
static const char* const kSuperSamplingPresets[] = {
|
||||
"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
|
||||
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance",
|
||||
@@ -8963,6 +8981,7 @@ static NgxResult create_cuda_feature(Feature& feature)
|
||||
const bool rayReconstruction = feature.rayReconstruction;
|
||||
const char* const* presetNames = rayReconstruction ? kRayReconstructionPresets : kSuperSamplingPresets;
|
||||
const NgxResult result = g.worker.call([&]() -> NgxResult {
|
||||
logf("CUDA_CREATE_WORKER_BEGIN preset=%u", feature.preset);
|
||||
NgxResult allocation = g.ngx.allocateParameters(&feature.cudaParams);
|
||||
if (allocation != NGX_SUCCESS)
|
||||
return allocation;
|
||||
@@ -9002,8 +9021,10 @@ static NgxResult create_cuda_feature(Feature& feature)
|
||||
feature.scratch = 0;
|
||||
// The core owns the denoiser: it loads nvngx_dlssd.dll, knows its kernel map, and
|
||||
// dispatches feature 13 to it. Only the feature id differs from Super Sampling.
|
||||
logf("CUDA_CREATE_NGX_BEGIN preset=%u scratch=0x%llx", feature.preset, feature.scratch);
|
||||
const NgxResult created = g.ngx.createFeature(
|
||||
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, p, &feature.cudaHandle);
|
||||
logf("CUDA_CREATE_NGX_END preset=%u result=0x%08x", feature.preset, created);
|
||||
if (created != NGX_SUCCESS)
|
||||
{
|
||||
if (feature.cudaHandle != nullptr) g.ngx.releaseFeature(feature.cudaHandle);
|
||||
@@ -9062,9 +9083,11 @@ static NgxResult create_cuda_feature(Feature& feature)
|
||||
}
|
||||
}
|
||||
return created;
|
||||
});
|
||||
}, "CUDA feature creation");
|
||||
logf("NVSDK_NGX_CUDA_CreateFeature(feature=%u) -> 0x%08x",
|
||||
rayReconstruction ? NGX_FEATURE_RAY_RECONSTRUCTION : NGX_FEATURE_SUPER_SAMPLING, result);
|
||||
logf("CUDA_CREATE_END preset=%u result=0x%08x elapsed_ms=%.1f", feature.preset, result,
|
||||
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - createStarted).count());
|
||||
if (result == NGX_SUCCESS) feature.cudaCreationDeferred = false;
|
||||
return result;
|
||||
}
|
||||
@@ -9339,6 +9362,33 @@ static void init_engine_vulkan()
|
||||
for (uint32_t index = 0; index < count; ++index)
|
||||
g_vk.float8 |= std::strcmp(extensions[index].extensionName, "VK_EXT_shader_float8") == 0;
|
||||
}
|
||||
auto features2 = reinterpret_cast<PFN_vkGetPhysicalDeviceFeatures2>(instanceProc(instance, "vkGetPhysicalDeviceFeatures2"));
|
||||
auto matrices = reinterpret_cast<PFN_vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR>(
|
||||
instanceProc(instance, "vkGetPhysicalDeviceCooperativeMatrixPropertiesKHR"));
|
||||
bool fp8 = false;
|
||||
if (features2 && matrices)
|
||||
{
|
||||
VkPhysicalDeviceShaderFloat8FeaturesEXT f8 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT};
|
||||
VkPhysicalDeviceCooperativeMatrixFeaturesKHR cm = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR};
|
||||
cm.pNext = g_vk.float8 ? &f8 : nullptr;
|
||||
VkPhysicalDeviceFeatures2 features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &cm};
|
||||
features2(g_vk.physical, &features);
|
||||
uint32_t n = 0;
|
||||
if (cm.cooperativeMatrix && matrices(g_vk.physical, &n, nullptr) == VK_SUCCESS)
|
||||
{
|
||||
std::vector<VkCooperativeMatrixPropertiesKHR> properties(n, {VK_STRUCTURE_TYPE_COOPERATIVE_MATRIX_PROPERTIES_KHR});
|
||||
if (matrices(g_vk.physical, &n, properties.data()) == VK_SUCCESS)
|
||||
for (uint32_t i = 0; i < n; ++i)
|
||||
{
|
||||
g_vk.matrixF16 |= d4r_matrix_tuple(properties[i], VK_COMPONENT_TYPE_FLOAT16_KHR);
|
||||
fp8 |= d4r_matrix_tuple(properties[i], VK_COMPONENT_TYPE_FLOAT8_E4M3_EXT);
|
||||
}
|
||||
}
|
||||
fp8 = fp8 && f8.shaderFloat8 && f8.shaderFloat8CooperativeMatrix;
|
||||
}
|
||||
g_vk.float8 = g_vk.float8 && fp8;
|
||||
logf("engine backend: f16 matrix tuple %s; FP8 matrix tuple %s", g_vk.matrixF16 ? "supported" : "unsupported",
|
||||
g_vk.float8 ? "supported" : "unsupported");
|
||||
logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: "
|
||||
"M and L run on the CUDA backend");
|
||||
// the features vkd3d-proton created its device with
|
||||
@@ -10328,6 +10378,8 @@ static std::string rr_model_directory(int slot)
|
||||
char folder[MAX_PATH * 3] = {};
|
||||
WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + folders[slot]).c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
|
||||
directory = folder;
|
||||
if (slot < 2 && !g_vk.float8)
|
||||
directory += "-gfx11";
|
||||
}
|
||||
return directory[0] == '/' ? "Z:" + directory : directory;
|
||||
}
|
||||
@@ -10346,6 +10398,10 @@ static void init_engine_backend_rr(int slot)
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
try
|
||||
{
|
||||
std::string metadata;
|
||||
if (!read_whole_file(widen(path + "\\rr.txt"), metadata) ||
|
||||
!d4r_rr_model_compatible(metadata, g_vk.matrixF16, g_vk.float8))
|
||||
throw std::runtime_error("model arithmetic is unsupported by this GPU (no pipelines created)");
|
||||
auto model = std::make_unique<d4r::RModel>(g_vk.physical, g_vk.device, path, VK_NULL_HANDLE, g_vk.int64Atomics);
|
||||
if (model->preset() != g_rrPreset[slot])
|
||||
throw std::runtime_error("the directory holds a model of another preset");
|
||||
|
||||
@@ -620,6 +620,13 @@ static NgxResult native_vk_init(unsigned long long id, const wchar_t* path, VkIn
|
||||
VkPhysicalDeviceIDProperties identity = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES};
|
||||
VkPhysicalDeviceProperties2 properties = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity};
|
||||
properties2(physical, &properties);
|
||||
VkPhysicalDevicePCIBusInfoPropertiesEXT bus = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PCI_BUS_INFO_PROPERTIES_EXT};
|
||||
properties.pNext = &bus;
|
||||
properties2(physical, &properties);
|
||||
char pci[32] = {};
|
||||
if (bus.pciDomain || bus.pciBus || bus.pciDevice || bus.pciFunction)
|
||||
std::snprintf(pci, sizeof(pci), "%04x:%02x:%02x.%x", bus.pciDomain, bus.pciBus, bus.pciDevice, bus.pciFunction);
|
||||
SetEnvironmentVariableA("D4R_RENDER_PCI", pci[0] ? pci : nullptr);
|
||||
if (identity.deviceLUIDValid)
|
||||
{
|
||||
uint32_t low, high;
|
||||
|
||||
+137
-36
@@ -13,6 +13,8 @@
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include "d4r_native_selection.h"
|
||||
#include "d4r_gpu_identity.h"
|
||||
#include "d4r_build.h"
|
||||
#include "d4r_motion_dilation.h"
|
||||
#include "d4r_native_motion.h"
|
||||
#include "d4r_wire_history.h"
|
||||
@@ -430,46 +432,54 @@ static void native_cleanup(void)
|
||||
remove_directory(native_served);
|
||||
}
|
||||
|
||||
/* The gfx target ("gfx1101") of the GPU DLSS runs on, "" when unknown: D4R_GPU_ARCH if set, else the KFD
|
||||
topology's GPU with the most SIMDs. With an integrated and a discrete GPU (Ryzen 7000/9000 desktops) the
|
||||
first KFD GPU node can be the integrated one, whose kernels would not match the discrete GPU. */
|
||||
/* Use the rendering adapter's PCI address, never the first/largest KFD GPU. Without an authenticated
|
||||
address a single GPU is unambiguous; multiple GPUs use translated kernels instead of a guessed target. */
|
||||
static void gpu_architecture(char* out, size_t size)
|
||||
{
|
||||
out[0] = '\0';
|
||||
const char* forced = getenv("D4R_GPU_ARCH");
|
||||
if (forced != NULL && strncmp(forced, "gfx", 3) == 0)
|
||||
char pci[64] = {}, only[32] = {};
|
||||
GetEnvironmentVariableA("D4R_RENDER_PCI", pci, sizeof(pci));
|
||||
int gpus = 0, matches = 0;
|
||||
DIR* nodes = opendir("/sys/class/kfd/kfd/topology/nodes");
|
||||
if (nodes == NULL) return;
|
||||
for (struct dirent* node; (node = readdir(nodes)) != NULL;)
|
||||
{
|
||||
snprintf(out, size, "%s", forced);
|
||||
return;
|
||||
}
|
||||
unsigned long best_simds = 0;
|
||||
int gpus = 0;
|
||||
for (int node = 0; node < 16; ++node)
|
||||
{
|
||||
char path[128];
|
||||
snprintf(path, sizeof(path), "/sys/class/kfd/kfd/topology/nodes/%d/properties", node);
|
||||
if (node->d_name[0] == '.') continue;
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "/sys/class/kfd/kfd/topology/nodes/%s/properties", node->d_name);
|
||||
FILE* properties = fopen(path, "r");
|
||||
if (properties == NULL)
|
||||
continue;
|
||||
char line[256];
|
||||
unsigned long simds = 0, target = 0;
|
||||
unsigned long simds = 0, target = 0, domain = ~0ul, location = ~0ul;
|
||||
while (fgets(line, sizeof(line), properties) != NULL)
|
||||
{
|
||||
sscanf(line, "simd_count %lu", &simds);
|
||||
sscanf(line, "gfx_target_version %lu", &target);
|
||||
sscanf(line, "domain %lu", &domain);
|
||||
sscanf(line, "location_id %lu", &location);
|
||||
}
|
||||
fclose(properties);
|
||||
if (simds == 0 || target == 0)
|
||||
continue;
|
||||
++gpus;
|
||||
if (simds > best_simds)
|
||||
snprintf(only, sizeof(only), "gfx%lu%lu%lx", target / 10000, (target / 100) % 100, target % 100);
|
||||
if (d4r_pci_matches_kfd(pci, domain, location))
|
||||
{
|
||||
best_simds = simds;
|
||||
snprintf(out, size, "gfx%lu%lu%lx", target / 10000, (target / 100) % 100, target % 100);
|
||||
++matches;
|
||||
snprintf(out, size, "%s", only);
|
||||
}
|
||||
}
|
||||
if (gpus > 1)
|
||||
tracef("native kernels: %d GPUs; using the one with the most SIMDs (%s); D4R_GPU_ARCH overrides", gpus, out);
|
||||
closedir(nodes);
|
||||
if (!pci[0] && gpus == 1) snprintf(out, size, "%s", only);
|
||||
else if (matches != 1) out[0] = '\0';
|
||||
const char* forced = getenv("D4R_GPU_ARCH");
|
||||
if (forced != NULL && strcmp(forced, out) != 0)
|
||||
{
|
||||
tracef("D4R_GPU_ARCH=%s does not match the rendering GPU (%s); native kernels disabled", forced, out);
|
||||
out[0] = '\0';
|
||||
}
|
||||
tracef("GPU target: render PCI=%s KFD GPUs=%d matches=%d architecture=%s", pci, gpus, matches, out);
|
||||
}
|
||||
|
||||
static void prepare_native_kernels(const char* cache_home)
|
||||
@@ -480,6 +490,11 @@ static void prepare_native_kernels(const char* cache_home)
|
||||
/* a release directory holds one manifest and set of code objects per GPU target */
|
||||
char architecture[32], source[1024], path[1200];
|
||||
gpu_architecture(architecture, sizeof(architecture));
|
||||
if (!architecture[0])
|
||||
{
|
||||
unsetenv("D4R_ZLUDA_NATIVE_DIR");
|
||||
return;
|
||||
}
|
||||
FILE* manifest = NULL;
|
||||
/* RDNA4 with native FP8 WMMA (D4R_ZLUDA_WMMA_FP8_NATIVE=1, d4r.ini NativeFp8): the <target>-fp8 variant,
|
||||
whose kernels match ZLUDA's FP8 lowering; no other GPU has such a directory */
|
||||
@@ -1385,8 +1400,12 @@ static void tracef(const char* format, ...)
|
||||
vsnprintf(line, sizeof(line), format, args);
|
||||
va_end(args);
|
||||
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
const double seconds = (double)now.tv_sec + (double)now.tv_nsec / 1000000000.0;
|
||||
const unsigned long thread = GetCurrentThreadId();
|
||||
pthread_mutex_lock(&trace_lock);
|
||||
fprintf(stderr, "[d4r nvcuda] %s\n", line);
|
||||
fprintf(stderr, "[d4r nvcuda] [%.6f tid=%lu] %s\n", seconds, thread, line);
|
||||
fflush(stderr);
|
||||
|
||||
char path[MAX_PATH] = {0};
|
||||
@@ -1396,9 +1415,14 @@ static void tracef(const char* format, ...)
|
||||
FILE* output = fopen(path, "a");
|
||||
if (output != NULL)
|
||||
{
|
||||
fprintf(output, "%s\n", line);
|
||||
fprintf(output, "[%.6f tid=%lu] %s\n", seconds, thread, line);
|
||||
fclose(output);
|
||||
}
|
||||
else
|
||||
{
|
||||
static int warned;
|
||||
if (!warned++) fprintf(stderr, "[d4r nvcuda] cannot open D4R_CUDA_TRACE=%s: %s\n", path, strerror(errno));
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&trace_lock);
|
||||
}
|
||||
@@ -1473,7 +1497,8 @@ typedef CUresult(__attribute__((sysv_abi)) *CUGETERRORSTRING_FN)(CUresult, const
|
||||
typedef CUresult(__attribute__((sysv_abi)) *CUGETPROCADDRESS_FN)(const char*, void**, int, uint64_t, int*);
|
||||
|
||||
static pthread_once_t context_once = PTHREAD_ONCE_INIT;
|
||||
static CUresult context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
|
||||
static CUresult context_bootstrap_result = CUDA_ERROR_NOT_INITIALIZED;
|
||||
static _Thread_local CUresult context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
|
||||
static CUcontext cuda_context;
|
||||
|
||||
static unsigned int get_process_u32(const char* name, unsigned int fallback)
|
||||
@@ -1487,6 +1512,36 @@ static unsigned int get_process_u32(const char* name, unsigned int fallback)
|
||||
return end != value && *end == '\0' ? (unsigned int)parsed : fallback;
|
||||
}
|
||||
|
||||
/* This calls ZLUDA directly: bridge wrappers would recurse through context bootstrap. */
|
||||
static CUresult selected_cuda_device(CUdevice* selected)
|
||||
{
|
||||
CUDEVICEGETCOUNT_FN count_fn = (CUDEVICEGETCOUNT_FN)find_zluda_symbol("cuDeviceGetCount");
|
||||
CUDEVICEGET_FN get_fn = (CUDEVICEGET_FN)find_zluda_symbol("cuDeviceGet");
|
||||
typedef CUresult(__attribute__((sysv_abi)) * PCI_FN)(char*, int, CUdevice);
|
||||
PCI_FN pci_fn = (PCI_FN)find_zluda_symbol("cuDeviceGetPCIBusId");
|
||||
int count = 0;
|
||||
if (!count_fn || !get_fn || count_fn(&count) != CUDA_SUCCESS || count < 1)
|
||||
return CUDA_ERROR_NOT_INITIALIZED;
|
||||
char expected[64] = {};
|
||||
GetEnvironmentVariableA("D4R_RENDER_PCI", expected, sizeof(expected));
|
||||
if (!expected[0] && count == 1) return get_fn(selected, 0);
|
||||
if (!expected[0] || !pci_fn) return CUDA_ERROR_NOT_SUPPORTED;
|
||||
int matches = 0;
|
||||
for (int ordinal = 0; ordinal < count; ++ordinal)
|
||||
{
|
||||
CUdevice device;
|
||||
char actual[64] = {};
|
||||
if (get_fn(&device, ordinal) == CUDA_SUCCESS && pci_fn(actual, sizeof(actual), device) == CUDA_SUCCESS)
|
||||
{
|
||||
tracef("compute adapter ordinal=%d device=%d PCI=%s render=%s", ordinal, device, actual, expected);
|
||||
if (d4r_same_pci(expected, actual)) { *selected = device; ++matches; }
|
||||
}
|
||||
}
|
||||
if (matches != 1)
|
||||
tracef("CUDA adapter mapping refused: render PCI=%s matches=%d; no GPU will be guessed", expected, matches);
|
||||
return matches == 1 ? CUDA_SUCCESS : CUDA_ERROR_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
static void create_default_zluda_context(void)
|
||||
{
|
||||
CUINIT_FN init = (CUINIT_FN)find_zluda_symbol("cuInit");
|
||||
@@ -1499,30 +1554,30 @@ static void create_default_zluda_context(void)
|
||||
return;
|
||||
}
|
||||
|
||||
context_setup_result = init(0);
|
||||
if (context_setup_result != CUDA_SUCCESS)
|
||||
context_bootstrap_result = init(0);
|
||||
if (context_bootstrap_result != CUDA_SUCCESS)
|
||||
{
|
||||
tracef("cuInit returned %d", context_setup_result);
|
||||
tracef("cuInit returned %d", context_bootstrap_result);
|
||||
return;
|
||||
}
|
||||
|
||||
int device_count = 0;
|
||||
context_setup_result = get_device_count(&device_count);
|
||||
if (context_setup_result != CUDA_SUCCESS || device_count < 1)
|
||||
context_bootstrap_result = get_device_count(&device_count);
|
||||
if (context_bootstrap_result != CUDA_SUCCESS || device_count < 1)
|
||||
{
|
||||
tracef("cuDeviceGetCount returned %d, count %d", context_setup_result, device_count);
|
||||
if (context_setup_result == CUDA_SUCCESS)
|
||||
context_setup_result = CUDA_ERROR_NOT_INITIALIZED;
|
||||
tracef("cuDeviceGetCount returned %d, count %d", context_bootstrap_result, device_count);
|
||||
if (context_bootstrap_result == CUDA_SUCCESS)
|
||||
context_bootstrap_result = CUDA_ERROR_NOT_INITIALIZED;
|
||||
return;
|
||||
}
|
||||
|
||||
CUdevice device = 0;
|
||||
context_setup_result = get_device(&device, 0);
|
||||
if (context_setup_result == CUDA_SUCCESS)
|
||||
context_setup_result = create_context(&cuda_context, 0, device);
|
||||
if (context_setup_result != CUDA_SUCCESS)
|
||||
context_bootstrap_result = selected_cuda_device(&device);
|
||||
if (context_bootstrap_result == CUDA_SUCCESS)
|
||||
context_bootstrap_result = create_context(&cuda_context, 0, device);
|
||||
if (context_bootstrap_result != CUDA_SUCCESS)
|
||||
{
|
||||
tracef("context creation returned %d", context_setup_result);
|
||||
tracef("context creation returned %d", context_bootstrap_result);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1531,7 +1586,21 @@ static void create_default_zluda_context(void)
|
||||
|
||||
static void ensure_context(void)
|
||||
{
|
||||
typedef CUresult(__attribute__((sysv_abi)) * CURRENT_FN)(CUcontext*);
|
||||
CURRENT_FN current = (CURRENT_FN)find_zluda_symbol("cuCtxGetCurrent");
|
||||
CUcontext active = NULL;
|
||||
if (current && current(&active) == CUDA_SUCCESS && active != NULL)
|
||||
{
|
||||
context_setup_result = CUDA_SUCCESS;
|
||||
return; /* NGX already selected a context; do not replace it with a second context. */
|
||||
}
|
||||
pthread_once(&context_once, create_default_zluda_context);
|
||||
context_setup_result = context_bootstrap_result;
|
||||
if (context_setup_result == CUDA_SUCCESS && cuda_context != NULL)
|
||||
{
|
||||
CUCTX_CURRENT_FN set = (CUCTX_CURRENT_FN)find_zluda_symbol("cuCtxSetCurrent");
|
||||
if (set) context_setup_result = set(cuda_context);
|
||||
}
|
||||
}
|
||||
|
||||
/* D4R_NGX_STREAM_PRIORITY=high|normal (default off): the null-stream work of NGX, the shim and this bridge
|
||||
@@ -2567,6 +2636,7 @@ static int before_teardown(void)
|
||||
|
||||
CUresult WINAPI cuInit(unsigned int flags)
|
||||
{
|
||||
tracef("bridge build %s; CUDA_BEGIN api=cuInit flags=%u", D4R_RUNTIME_BUILD_ID, flags);
|
||||
CUINIT_FN function = (CUINIT_FN)find_zluda_symbol("cuInit");
|
||||
CUresult result = function != NULL ? function(flags) : CUDA_ERROR_NOT_INITIALIZED;
|
||||
tracef("cuInit flags=%u result=%d", flags, result);
|
||||
@@ -2618,6 +2688,14 @@ CUresult WINAPI cuCtxGetDevice(CUdevice* device)
|
||||
|
||||
CUresult WINAPI cuCtxCreate_v2(CUcontext* context, unsigned int flags, CUdevice device)
|
||||
{
|
||||
CUdevice selected = -1;
|
||||
CUresult mapping = selected_cuda_device(&selected);
|
||||
if (mapping != CUDA_SUCCESS || selected != device)
|
||||
{
|
||||
if (context) *context = NULL;
|
||||
tracef("CUDA context refused: device=%d mapped render device=%d status=%d", device, selected, mapping);
|
||||
return mapping == CUDA_SUCCESS ? CUDA_ERROR_NOT_SUPPORTED : mapping;
|
||||
}
|
||||
CUCTXCREATE_FN function = (CUCTXCREATE_FN)find_zluda_symbol("cuCtxCreate_v2");
|
||||
CUresult result = function != NULL ? function(context, flags, device) : CUDA_ERROR_NOT_INITIALIZED;
|
||||
tracef("cuCtxCreate_v2 flags=0x%x device=%d result=%d context=%p", flags, device,
|
||||
@@ -2659,6 +2737,7 @@ static int elide_ngx_sync(void)
|
||||
|
||||
CUresult WINAPI d4rCtxSynchronize(void)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=d4rCtxSynchronize");
|
||||
if (before_teardown())
|
||||
return CUDA_SUCCESS;
|
||||
CUCTX_SYNCHRONIZE_FN function = (CUCTX_SYNCHRONIZE_FN)find_zluda_symbol("cuCtxSynchronize");
|
||||
@@ -2671,6 +2750,7 @@ CUresult WINAPI d4rCtxSynchronize(void)
|
||||
|
||||
CUresult WINAPI cuCtxSynchronize(void)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuCtxSynchronize");
|
||||
if (before_teardown())
|
||||
return CUDA_SUCCESS;
|
||||
if (elide_ngx_sync())
|
||||
@@ -2695,6 +2775,15 @@ CUresult WINAPI cuDeviceGetLuid(char* luid, unsigned int* device_node_mask, CUde
|
||||
if ((result == CUDA_SUCCESS || result == CUDA_ERROR_NOT_SUPPORTED) &&
|
||||
luid != NULL && device_node_mask != NULL)
|
||||
{
|
||||
CUdevice selected = -1;
|
||||
CUresult mapping = selected_cuda_device(&selected);
|
||||
if (mapping != CUDA_SUCCESS || selected != device)
|
||||
{
|
||||
if (luid) memset(luid, 0, 8);
|
||||
if (device_node_mask) *device_node_mask = 0;
|
||||
tracef("cuDeviceGetLuid device=%d is not the mapped render adapter (selected=%d status=%d)", device, selected, mapping);
|
||||
return mapping == CUDA_SUCCESS ? CUDA_SUCCESS : mapping;
|
||||
}
|
||||
unsigned int low = get_process_u32("D4R_CUDA_LUID_LOW", 0xffffffffu);
|
||||
unsigned int high = get_process_u32("D4R_CUDA_LUID_HIGH", 0xffffffffu);
|
||||
unsigned int node_mask = get_process_u32("D4R_CUDA_NODE_MASK", 0xffffffffu);
|
||||
@@ -2791,6 +2880,7 @@ static const void* replacement_image(const void* image, const char* api)
|
||||
|
||||
CUresult WINAPI cuModuleLoadData(CUmodule* module, const void* image)
|
||||
{
|
||||
tracef("CUDA_BEGIN api=cuModuleLoadData image=%p", image);
|
||||
CUMODULELOADDATA_FN function = (CUMODULELOADDATA_FN)find_zluda_symbol("cuModuleLoadData");
|
||||
verify_native_kernels(image);
|
||||
capture_ptx_if_present(image, "cuModuleLoadData");
|
||||
@@ -2804,6 +2894,7 @@ CUresult WINAPI cuModuleLoadData(CUmodule* module, const void* image)
|
||||
CUresult WINAPI cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int option_count,
|
||||
int* options, void** option_values)
|
||||
{
|
||||
tracef("CUDA_BEGIN api=cuModuleLoadDataEx image=%p options=%u", image, option_count);
|
||||
CUMODULELOADDATAEX_FN function = (CUMODULELOADDATAEX_FN)find_zluda_symbol("cuModuleLoadDataEx");
|
||||
verify_native_kernels(image);
|
||||
capture_ptx_if_present(image, "cuModuleLoadDataEx");
|
||||
@@ -2817,6 +2908,7 @@ CUresult WINAPI cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned
|
||||
|
||||
CUresult WINAPI cuModuleLoad(CUmodule* module, const char* filename)
|
||||
{
|
||||
tracef("CUDA_BEGIN api=cuModuleLoad file=%s", filename != NULL ? filename : "<null>");
|
||||
CUMODULELOAD_FN function = (CUMODULELOAD_FN)find_zluda_symbol("cuModuleLoad");
|
||||
CUresult result = function != NULL ? function(module, filename) : missing("cuModuleLoad");
|
||||
tracef("cuModuleLoad file=%s result=%d module=%p", filename != NULL ? filename : "<null>",
|
||||
@@ -2837,6 +2929,7 @@ CUresult WINAPI cuModuleUnload(CUmodule module)
|
||||
|
||||
CUresult WINAPI cuModuleGetFunction(CUfunction* function_out, CUmodule module, const char* name)
|
||||
{
|
||||
tracef("CUDA_BEGIN api=cuModuleGetFunction module=%p name=%s", module, name ? name : "<null>");
|
||||
CUMODULEGETFUNCTION_FN function = (CUMODULEGETFUNCTION_FN)find_zluda_symbol("cuModuleGetFunction");
|
||||
CUresult result = function != NULL ? function(function_out, module, name) : missing("cuModuleGetFunction");
|
||||
if (result == CUDA_SUCCESS && function_out != NULL)
|
||||
@@ -2882,6 +2975,7 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
|
||||
unsigned int block_z, unsigned int shared_bytes, CUstream stream,
|
||||
void** kernel_params, void** extra)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuLaunchKernel function=%p", function_handle);
|
||||
static CULAUNCHKERNEL_FN function;
|
||||
if (function == NULL)
|
||||
function = (CULAUNCHKERNEL_FN)find_zluda_symbol("cuLaunchKernel");
|
||||
@@ -2964,6 +3058,7 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
|
||||
|
||||
CUresult WINAPI cuMemAlloc(CUdeviceptr* pointer, size_t bytes)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemAlloc bytes=%zu", bytes);
|
||||
CUMEMALLOC_FN function = (CUMEMALLOC_FN)find_zluda_symbol("cuMemAlloc_v2");
|
||||
CUresult result = function != NULL ? function(pointer, bytes) : missing("cuMemAlloc_v2");
|
||||
if (result == CUDA_SUCCESS && pointer != NULL)
|
||||
@@ -3022,6 +3117,8 @@ CUresult WINAPI cuMemcpy2D(const void* copy)
|
||||
|
||||
CUresult WINAPI cuMemcpyHtoDAsync(CUdeviceptr destination, const void* source, size_t bytes, CUstream stream)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemcpyHtoDAsync bytes=%zu dst=0x%llx stream=%p", bytes,
|
||||
(unsigned long long)destination, stream);
|
||||
CUMEMCPYHTODASYNC_FN function = (CUMEMCPYHTODASYNC_FN)find_zluda_symbol("cuMemcpyHtoDAsync_v2");
|
||||
if (microcuda_k_write) microcuda_k_write(destination, bytes);
|
||||
stream = d4r_ngx_stream(stream);
|
||||
@@ -3033,6 +3130,7 @@ CUresult WINAPI cuMemcpyHtoDAsync(CUdeviceptr destination, const void* source, s
|
||||
|
||||
CUresult WINAPI cuMemcpyDtoH(void* destination, CUdeviceptr source, size_t bytes)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuMemcpyDtoH bytes=%zu src=0x%llx", bytes, (unsigned long long)source);
|
||||
CUMEMCPYDTOH_FN function = (CUMEMCPYDTOH_FN)find_zluda_symbol("cuMemcpyDtoH_v2");
|
||||
CUresult result = function != NULL ? function(destination, source, bytes) : missing("cuMemcpyDtoH_v2");
|
||||
TRACE_CALL(result, "cuMemcpyDtoH->v2 dst=%p src=0x%llx bytes=%zu result=%d",
|
||||
@@ -3378,6 +3476,7 @@ static int d4r_forward_is_teardown(const char* target)
|
||||
#define D4R_FORWARD(name, target, count, params, args) \
|
||||
CUresult WINAPI name params \
|
||||
{ \
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=" #name); \
|
||||
if (d4r_forward_is_teardown(#target) && before_teardown()) \
|
||||
return CUDA_SUCCESS; \
|
||||
typedef CUresult(__attribute__((sysv_abi)) * function_type) params; \
|
||||
@@ -3394,6 +3493,7 @@ D4R_FORWARD(cuEventRecord, cuEventRecord, 2, (D4rArg a0, D4rArg a1), (a0, (D4rAr
|
||||
D4R_FORWARD(cuEventRecordWithFlags, cuEventRecordWithFlags, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, (D4rArg)d4r_ngx_stream((CUstream)a1), a2))
|
||||
CUresult WINAPI cuEventSynchronize(D4rArg a0)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=cuEventSynchronize event=%p", (void*)a0);
|
||||
if (before_teardown())
|
||||
return CUDA_SUCCESS;
|
||||
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
|
||||
@@ -3411,6 +3511,7 @@ CUresult WINAPI cuEventSynchronize(D4rArg a0)
|
||||
game's split command list only after this event has actually completed. */
|
||||
CUresult WINAPI d4rEventSynchronize(D4rArg a0)
|
||||
{
|
||||
if (trace_verbose()) tracef("CUDA_BEGIN api=d4rEventSynchronize event=%p", (void*)a0);
|
||||
if (before_teardown())
|
||||
return CUDA_SUCCESS;
|
||||
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
|
||||
|
||||
Reference in new issue
Block a user