wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add experimental native Vulkan DLSS frontend

Browse Source 
Implement NGX's Vulkan API over the existing CUDA backend. Compute shaders
pack the game's sampled images into buffers shared with CUDA (device-local
memory exported through the Wine bridge), and a device event in the game's
command buffer holds its queue until the result is written back, keeping
frame age at zero. DLSS reads inputs through pitch-linear textures and its
native output kernel writes the shared buffer directly; input and
completion waits are adaptive.

Make it work across engines: shared buffers are concurrent over every
compute-capable queue family (Indiana Jones records DLSS on async compute),
depth kept in an R32F colour image is accepted under a depth aspect, and the
extension-requirement query allows a null list. Spare buffer sets are
imported on the first frame and never while a frame is pending, since an
import can block on GPU work that is waiting for us; host staging covers any
shortfall.

Fix two exit hazards. A game quitting without NGX shutdown left the GPU
waiting on an event that only a killed thread would set, resetting the ring
10 s later: the bridge now runs a shim hook before teardown and skips all
CUDA destroy/sync calls once Wine is shutting the process down. The shim's
own entry point skips CRT static destructors at process termination, where
locks held by killed threads made the exit spin forever.

Add a Vulkan DLSS harness (dynamic, recreate, discard, invalid, exit,
compute-queue and in-flight modes), staging/depth probes, ABI/config tests,
and an OptiScaler settings overlay for Vulkan games.
This commit is contained in:
ayo committed 2026-10-09 02:57:23 -04:00
1 parent 18c694eecb
commit 694a7a0126
24 files changed
+2215 -20
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/d4r_nvngx_shim.cpp
+62 -11
View File
@@ -1,3 +1,4 @@
// Experimental native Vulkan NGX frontend: d4r_vulkan_frontend.inl.
// d4r_nvngx.dll: an NGX core replacement that implements the D3D12 DLSS
// entry points on top of the official NGX core's CUDA path (which runs on AMD
// through the Wine nvcuda bridge and ZLUDA). OptiScaler loads it through its
@@ -1342,6 +1343,18 @@ static bool load_libraries()
logf("loading D4R_NGX_CORE=%ls failed: %lu", corePath, GetLastError());
return false;
}
// A successful LoadLibrary can still be OptiScaler's interception proxy.
// Diagnose that before trying to bind the rest of the CUDA NGX API.
if (GetProcAddress(g.core, "NVSDK_NGX_CUDA_Init") == nullptr)
{
wchar_t loadedPath[MAX_PATH] = {};
GetModuleFileNameW(g.core, loadedPath, MAX_PATH);
logf("NGX core load returned an incompatible/intercepted module: %ls; "
"expected %ls. Set [Hooks] HookOriginalNvngxOnly=true in OptiScaler.ini",
loadedPath, corePath);
g.core = nullptr;
return false;
}
bool ok = true;
ok &= load_export(g.core, "NVSDK_NGX_CUDA_Init", g.ngx.init);
if (!load_export(g.core, "NVSDK_NGX_CUDA_Init_ProjectID", g.ngx.initProjectId))
@@ -1686,6 +1699,7 @@ struct OutputSlot
struct Feature
{
NgxHandle handle = {}; // returned to the caller
bool nativeVulkan = false; // distinct resource/lifetime contract
std::atomic<bool> retiring{false};
IUnknown* resources = nullptr; // main reference plus command-allocator references
NgxHandle* cudaHandle = nullptr;
@@ -1701,6 +1715,8 @@ struct Feature
// output wait pacing (D4R_SHIM_OUTPUT_POLL_US): smoothed time from the event record to its completion, kept
// separately for each wait site (they wait for very different amounts of GPU work)
double publishWaitEmaUs = 0.0, evalWaitEmaUs = 0.0;
// native Vulkan input wait: smoothed time from recording to the input event (waiters may overlap)
std::atomic<double> vkInputWaitEmaUs{0.0};
bool profileEventsReady = false;
CudaImage color, depth, motion, exposure, output;
@@ -4766,15 +4782,17 @@ static float get_float_or(void* parameters, const char* name, float fallback)
return d4r_ngx_get_float(parameters, name, &value) == NGX_SUCCESS ? value : fallback;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned int featureId, void* parameters,
NgxHandle** handle)
static NgxResult create_feature(unsigned int featureId, void* parameters, NgxHandle** handle, bool nativeVulkan)
{
std::lock_guard<std::mutex> apiLock(g_featureApiMutex);
std::lock_guard<std::mutex> stateLock(g.mutex);
if (!g.ngxInitialized)
return NGX_FAIL_NOT_INITIALIZED;
std::call_once(g_vk.once, init_vram_interop);
if (g_vk.lifetime == nullptr)
if (!nativeVulkan && g.device == nullptr)
return NGX_FAIL_NOT_INITIALIZED;
if (!nativeVulkan)
std::call_once(g_vk.once, init_vram_interop);
if (!nativeVulkan && g_vk.lifetime == nullptr)
{
logf("DLSS requires lifetime-aware d4r d3d12.dll/d3d12core.dll; update both files with the shim");
return NGX_FAIL_PLATFORM_ERROR;
@@ -4785,6 +4803,7 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, u
return NGX_FAIL_FEATURE_NOT_SUPPORTED;
}
auto* feature = new Feature();
feature->nativeVulkan = nativeVulkan;
feature->width = get_uint_or(parameters, "Width", 0);
feature->height = get_uint_or(parameters, "Height", 0);
feature->outWidth = get_uint_or(parameters, "OutWidth", 0);
@@ -4895,15 +4914,18 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, u
release_feature(feature);
return result;
}
if (!create_buffer(D3D12_HEAP_TYPE_READBACK, 256, &feature->marker,
if (!nativeVulkan && !create_buffer(D3D12_HEAP_TYPE_READBACK, 256, &feature->marker,
reinterpret_cast<uint8_t**>(const_cast<uint32_t**>(&feature->markerValue))))
{
release_feature(feature);
return NGX_FAIL_PLATFORM_ERROR;
}
for (int index = 0; index <= kSlots; ++index)
feature->markerValue[index] = 0;
feature->resources = new DeferredResources(feature);
if (!nativeVulkan)
{
for (int index = 0; index <= kSlots; ++index)
feature->markerValue[index] = 0;
feature->resources = new DeferredResources(feature);
}
feature->handle.Id = g.nextHandleId++;
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
@@ -4913,6 +4935,12 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, u
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned int featureId,
void* parameters, NgxHandle** handle)
{
return create_feature(featureId, parameters, handle, false);
}
static Feature* find_feature(const NgxHandle* handle)
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
@@ -4947,7 +4975,7 @@ static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* hand
lastGameStart = timing.gameStart;
}
Feature* feature = find_feature(handle);
if (feature == nullptr || list == nullptr || parameters == nullptr)
if (feature == nullptr || feature->nativeVulkan || list == nullptr || parameters == nullptr)
return NGX_FAIL_INVALID_PARAMETER;
ID3D12Resource* color = get_resource(parameters, "Color");
ID3D12Resource* depth = get_resource(parameters, "Depth");
@@ -5426,13 +5454,13 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_ReleaseFeature(NgxHandle* handle)
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
for (Feature* candidate : g_features)
if (&candidate->handle == handle && !candidate->retiring.exchange(true))
if (&candidate->handle == handle && !candidate->nativeVulkan && !candidate->retiring.exchange(true))
{
feature = candidate;
break;
}
}
if (feature == nullptr)
if (feature == nullptr || feature->nativeVulkan)
return NGX_FAIL_INVALID_PARAMETER;
logf("NVSDK_NGX_D3D12_ReleaseFeature(id=%u) after %u completed frames", feature->handle.Id,
feature->completedFrames.load());
@@ -5470,6 +5498,9 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown()
{
std::lock_guard<std::mutex> lock(g_featuresMutex);
features = g_features;
for (Feature* feature : features)
if (feature->nativeVulkan)
return NGX_FAIL_INVALID_PARAMETER;
for (Feature* feature : features)
feature->retiring = true;
}
@@ -5498,6 +5529,8 @@ D4R_EXPORT NgxResult NVSDK_NGX_UpdateFeature(const void*, unsigned int)
return NGX_SUCCESS;
}
#include "d4r_vulkan_frontend.inl"
BOOL WINAPI DllMain(HINSTANCE instance, DWORD reason, LPVOID)
{
if (reason == DLL_PROCESS_ATTACH)
@@ -5505,5 +5538,23 @@ BOOL WINAPI DllMain(HINSTANCE instance, DWORD reason, LPVOID)
g_selfModule = instance;
DisableThreadLibraryCalls(instance);
}
else if (reason == DLL_PROCESS_DETACH)
native_vk_process_exit();
return TRUE;
}
// The DLL entry point (linked with --entry). At process termination (reserved != NULL) ExitProcess
// has already killed the worker and waiter threads, possibly inside winpthreads' global condition
// variable spinlock, so the CRT's static destructors (mutexes, condition variables) can spin forever
// and the game never exits. Release the GPU waits and skip all CRT teardown then, as Windows advises;
// the system reclaims everything. Every other notification goes through the CRT as usual.
extern "C" BOOL WINAPI DllMainCRTStartup(HANDLE, DWORD, LPVOID);
extern "C" BOOL WINAPI d4r_dll_entry(HANDLE instance, DWORD reason, LPVOID reserved)
{
if (reason == DLL_PROCESS_DETACH && reserved != nullptr)
{
native_vk_process_exit();
return TRUE;
}
return DllMainCRTStartup(instance, reason, reserved);
}
tools/d4r_vulkan_depth.h
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include <vulkan/vulkan_core.h>
#include <cstddef>
// Image-to-buffer depth copies use the depth aspect's scalar representation,
// not the size of a packed depth+stencil texel.
constexpr size_t d4r_vk_depth_bytes(VkFormat format)
{
switch (format)
{
case VK_FORMAT_D16_UNORM:
case VK_FORMAT_D16_UNORM_S8_UINT: return 2;
case VK_FORMAT_X8_D24_UNORM_PACK32:
case VK_FORMAT_D24_UNORM_S8_UINT:
case VK_FORMAT_D32_SFLOAT:
case VK_FORMAT_D32_SFLOAT_S8_UINT: return 4;
default: return 0;
}
}
constexpr bool d4r_vk_has_stencil(VkFormat format)
{
return format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT ||
format == VK_FORMAT_D32_SFLOAT_S8_UINT;
}
constexpr VkImageAspectFlags d4r_vk_copy_aspect(VkFormat format, VkImageAspectFlags requested)
{
if (d4r_vk_depth_bytes(format))
{
const auto allowed = VK_IMAGE_ASPECT_DEPTH_BIT |
(d4r_vk_has_stencil(format) ? VK_IMAGE_ASPECT_STENCIL_BIT : 0);
return (requested & VK_IMAGE_ASPECT_DEPTH_BIT) && !(requested & ~allowed)
? static_cast<VkImageAspectFlags>(VK_IMAGE_ASPECT_DEPTH_BIT) : 0;
}
// Depth kept in a colour image (R32_SFLOAT in Indiana Jones) arrives labelled with the depth
// aspect by Streamline/OptiScaler; the caller's view is still a colour view, so sample it as one.
return requested == VK_IMAGE_ASPECT_COLOR_BIT || requested == VK_IMAGE_ASPECT_DEPTH_BIT
? static_cast<VkImageAspectFlags>(VK_IMAGE_ASPECT_COLOR_BIT) : 0;
}
constexpr VkImageAspectFlags d4r_vk_barrier_aspect(VkFormat format, VkImageAspectFlags requested)
{
return d4r_vk_has_stencil(format) ? VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT : requested;
}
tools/d4r_vulkan_frontend.inl
+938
View File
@@ -0,0 +1,938 @@
// Native Vulkan NGX frontend. Included after the shared CUDA NGX implementation.
// Resource ABI follows NVIDIA/DLSS include/nvsdk_ngx_defs_vk.h. No SDK dependency.
// CPU staging is intentionally conservative: events keep the result in the
// evaluating command buffer without ending/submitting an application-owned list.
#include "d4r_vulkan_ngx.h"
#include "d4r_vulkan_depth.h"
#include "d4r_vulkan_staging.h"
struct NativeVkApi
{
VkDevice device = VK_NULL_HANDLE;
VkPhysicalDeviceMemoryProperties memory = {};
PFN_vkCreateBuffer createBuffer = nullptr;
PFN_vkDestroyBuffer destroyBuffer = nullptr;
PFN_vkGetBufferMemoryRequirements requirements = nullptr;
PFN_vkAllocateMemory allocate = nullptr;
PFN_vkFreeMemory free = nullptr;
PFN_vkBindBufferMemory bind = nullptr;
PFN_vkMapMemory map = nullptr;
PFN_vkUnmapMemory unmap = nullptr;
PFN_vkCreateEvent createEvent = nullptr;
PFN_vkDestroyEvent destroyEvent = nullptr;
PFN_vkGetEventStatus eventStatus = nullptr;
PFN_vkSetEvent setEvent = nullptr;
PFN_vkResetEvent resetEvent = nullptr;
PFN_vkCmdSetEvent cmdSetEvent = nullptr;
PFN_vkCmdWaitEvents waitEvents = nullptr;
PFN_vkCmdPipelineBarrier barrier = nullptr;
PFN_vkCmdFillBuffer fill = nullptr;
PFN_vkDeviceWaitIdle idle = nullptr;
int(WINAPI* import)(VkDevice, uint64_t, uint64_t, CudaDevicePtr*, void**) = nullptr;
int(WINAPI* release)(void*) = nullptr;
uint32_t queueFamily = UINT32_MAX; // exclusive owner when the device has one compute-capable family
// Every compute-capable family: shared buffers are CONCURRENT across them, since a game may record
// DLSS on its async compute queue (Indiana Jones) as well as on the graphics queue (DOOM Eternal).
uint32_t families[8] = {};
uint32_t familyCount = 0;
bool external = false;
};
static NativeVkApi g_nativeVk;
static D4rVkStaging g_nativeVkStaging;
static std::mutex g_nativeVkMutex; // all native API calls, including init/teardown
struct NativeVkPlane
{
VkBuffer buffer = VK_NULL_HANDLE;
VkDeviceMemory memory = VK_NULL_HANDLE;
uint8_t* mapped = nullptr;
NgxVkImage image = {};
DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
size_t rowBytes = 0;
size_t capacity = 0;
CudaDevicePtr cuda = 0;
void* external = nullptr;
bool externalOwned = false;
D4rVkStagingSet staging;
// Pitch-linear texture on `cuda`, handed to NGX without a buffer->array copy
// (worker only). Pooled buffers keep it; the key detects a geometry change.
CudaObject texture = 0;
uint64_t textureKey = 0;
};
struct NativeVkFrame
{
NativeVkPlane planes[5]; // color, depth, motion, optional exposure, output
VkEvent inputReady = VK_NULL_HANDLE, outputReady = VK_NULL_HANDLE, consumed = VK_NULL_HANDLE;
std::atomic<bool> cancel{false}, done{false};
std::atomic<NgxResult> result{NGX_SUCCESS};
std::thread waiter; // submission waits must not serialize across command buffers
FrameParams params;
uint32_t number = 0;
VkCommandBuffer commandBuffer = VK_NULL_HANDLE;
};
struct NativeVkFeature
{
Feature* feature;
std::vector<NativeVkFrame*> frames;
std::vector<NativeVkFrame*> reusable;
};
static std::vector<NativeVkFeature> g_nativeVkFeatures;
// Release can happen while recordings are still pending. Their GPU objects
// outlive CUDA feature state and are reclaimed at completion or shutdown.
static std::vector<NativeVkFrame*> g_nativeVkRetired;
// ROCm retains imported mappings; reuse across feature recreation rather than
// exporting and importing the same-sized allocations again on each mode change.
static std::vector<NativeVkPlane> g_nativeVkBufferPool;
static unsigned g_nativeVkExternalAllocations = 0;
static DXGI_FORMAT native_vk_format(VkFormat format, size_t& bytes)
{
bytes = 4;
switch (format)
{
case VK_FORMAT_R16G16B16A16_SFLOAT: bytes = 8; return DXGI_FORMAT_R16G16B16A16_FLOAT;
case VK_FORMAT_R32G32B32A32_SFLOAT: bytes = 16; return DXGI_FORMAT_R32G32B32A32_FLOAT;
case VK_FORMAT_R16G16_SFLOAT: return DXGI_FORMAT_R16G16_FLOAT;
case VK_FORMAT_R32G32_SFLOAT: bytes = 8; return DXGI_FORMAT_R32G32_FLOAT;
case VK_FORMAT_R32_SFLOAT: return DXGI_FORMAT_R32_FLOAT;
case VK_FORMAT_D32_SFLOAT:
case VK_FORMAT_D32_SFLOAT_S8_UINT: return DXGI_FORMAT_D32_FLOAT; // depth-only buffer copy is 4 bytes
case VK_FORMAT_X8_D24_UNORM_PACK32:
case VK_FORMAT_D24_UNORM_S8_UINT: return DXGI_FORMAT_D24_UNORM_S8_UINT;
case VK_FORMAT_R16_SFLOAT: bytes = 2; return DXGI_FORMAT_R16_FLOAT;
case VK_FORMAT_R16_UNORM: bytes = 2; return DXGI_FORMAT_R16_UNORM;
case VK_FORMAT_D16_UNORM:
case VK_FORMAT_D16_UNORM_S8_UINT: bytes = 2; return DXGI_FORMAT_D16_UNORM;
case VK_FORMAT_R8G8B8A8_UNORM: return DXGI_FORMAT_R8G8B8A8_UNORM;
case VK_FORMAT_R8G8B8A8_SRGB: return DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
case VK_FORMAT_B8G8R8A8_UNORM: return DXGI_FORMAT_B8G8R8A8_UNORM;
case VK_FORMAT_B8G8R8A8_SRGB: return DXGI_FORMAT_B8G8R8A8_UNORM_SRGB;
case VK_FORMAT_A2B10G10R10_UNORM_PACK32: return DXGI_FORMAT_R10G10B10A2_UNORM;
case VK_FORMAT_B10G11R11_UFLOAT_PACK32: return DXGI_FORMAT_R11G11B10_FLOAT;
case VK_FORMAT_E5B9G9R9_UFLOAT_PACK32: return DXGI_FORMAT_R9G9B9E5_SHAREDEXP;
default: bytes = 0; return DXGI_FORMAT_UNKNOWN;
}
}
static void native_vk_free_plane(NativeVkPlane& p, bool final = false)
{
g_nativeVkStaging.free(p.staging);
if (p.external && !final) {
g_nativeVkBufferPool.push_back(p); p = {}; return;
}
if (p.external) g.worker.call([&] {
if (p.texture) g.cu.texObjectDestroy(p.texture);
g.cu.memFree(p.cuda); return g_nativeVk.release(p.external);
});
if (p.mapped) g_nativeVk.unmap(g_nativeVk.device, p.memory);
if (p.buffer) g_nativeVk.destroyBuffer(g_nativeVk.device, p.buffer, nullptr);
if (p.memory) g_nativeVk.free(g_nativeVk.device, p.memory, nullptr);
p = {};
}
// Exports a new device-local buffer and imports it into CUDA. The import can block on GPU work of
// the game's queues, which may in turn wait for one of our pending frames: callers make sure none is.
static bool native_vk_import(NativeVkPlane& p, VkDeviceSize bytes)
{
if (g_nativeVkExternalAllocations >= 64) return false;
VkExternalMemoryBufferCreateInfo external = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
info.size = bytes; info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
info.sharingMode = g_nativeVk.familyCount > 1 ? VK_SHARING_MODE_CONCURRENT : VK_SHARING_MODE_EXCLUSIVE;
info.queueFamilyIndexCount = g_nativeVk.familyCount > 1 ? g_nativeVk.familyCount : 0;
info.pQueueFamilyIndices = g_nativeVk.familyCount > 1 ? g_nativeVk.families : nullptr;
VkResult result = g_nativeVk.createBuffer(g_nativeVk.device, &info, nullptr, &p.buffer);
VkMemoryRequirements requirements = {};
if (result == VK_SUCCESS) g_nativeVk.requirements(g_nativeVk.device, p.buffer, &requirements);
uint32_t type = UINT32_MAX;
for (int pass = 0; pass < 2 && type == UINT32_MAX; ++pass)
for (uint32_t i = 0; i < g_nativeVk.memory.memoryTypeCount; ++i) {
const auto flags = g_nativeVk.memory.memoryTypes[i].propertyFlags;
if ((requirements.memoryTypeBits & (1u << i)) && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
(pass || !(flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT))) { type = i; break; }
}
VkMemoryDedicatedAllocateInfo dedicated = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
dedicated.buffer = p.buffer;
VkExportMemoryAllocateInfo exportInfo = {VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, &dedicated};
exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo};
allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = type;
if (result == VK_SUCCESS && type != UINT32_MAX) result = g_nativeVk.allocate(g_nativeVk.device, &allocation, nullptr, &p.memory);
else result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
if (result == VK_SUCCESS) result = g_nativeVk.bind(g_nativeVk.device, p.buffer, p.memory, 0);
int imported = -1;
if (result == VK_SUCCESS) imported = g.worker.call([&] {
return g_nativeVk.import(g_nativeVk.device, reinterpret_cast<uint64_t>(p.memory), requirements.size, &p.cuda, &p.external);
});
if (result == VK_SUCCESS && imported == 0) {
p.capacity = bytes; ++g_nativeVkExternalAllocations;
logf("native Vulkan VRAM buffer: %llu bytes, type=%u", static_cast<unsigned long long>(bytes), type);
return true;
}
logf("native Vulkan VRAM export/import unavailable: Vulkan=%d CUDA=%d; using host staging", result, imported);
const auto image = p.image; const auto format = p.format; const auto row = p.rowBytes;
native_vk_free_plane(p, true); p.image = image; p.format = format; p.rowBytes = row;
// Avoid repeating an unsupported import for every subsequent plane/frame.
g_nativeVk.external = false;
return false;
}
static bool native_vk_vram_plane(NativeVkPlane& p, VkDeviceSize bytes, bool allowImport)
{
if (!g_nativeVk.external) return false;
auto best = g_nativeVkBufferPool.end();
for (auto it = g_nativeVkBufferPool.begin(); it != g_nativeVkBufferPool.end(); ++it)
if (it->capacity >= bytes && (best == g_nativeVkBufferPool.end() || it->capacity < best->capacity)) best = it;
if (best != g_nativeVkBufferPool.end()) {
const auto image = p.image; const auto format = p.format; const auto row = p.rowBytes;
p = *best; p.image = image; p.format = format; p.rowBytes = row;
g_nativeVkBufferPool.erase(best); return true;
}
return allowImport && native_vk_import(p, bytes); // otherwise host staging for this frame
}
// With no frame pending, import spare buffers for the frames that will be in flight later
// (D4R_VULKAN_SPARE_SETS, default 3), so steady state never imports while the GPU waits on us.
static void native_vk_prefill(const NativeVkFrame& frame)
{
static const unsigned spares = env_uint("D4R_VULKAN_SPARE_SETS", 3);
std::vector<VkDeviceSize> sizes;
for (const auto& p : frame.planes) if (p.cuda) sizes.push_back(p.rowBytes * p.image.height);
std::sort(sizes.begin(), sizes.end(), std::greater<>());
std::vector<bool> claimed(g_nativeVkBufferPool.size(), false);
for (const auto bytes : sizes)
for (unsigned k = 0; k < spares && g_nativeVk.external; ++k) {
size_t best = claimed.size();
for (size_t i = 0; i < claimed.size(); ++i)
if (!claimed[i] && g_nativeVkBufferPool[i].capacity >= bytes &&
(best == claimed.size() || g_nativeVkBufferPool[i].capacity < g_nativeVkBufferPool[best].capacity)) best = i;
if (best != claimed.size()) { claimed[best] = true; continue; }
NativeVkPlane spare;
if (!native_vk_import(spare, bytes)) return;
g_nativeVkBufferPool.push_back(spare); claimed.push_back(true);
}
}
static void native_vk_free_frame(NativeVkFrame* frame)
{
if (frame->waiter.joinable()) frame->waiter.join();
for (auto& p : frame->planes)
{
native_vk_free_plane(p);
}
if (frame->inputReady) g_nativeVk.destroyEvent(g_nativeVk.device, frame->inputReady, nullptr);
if (frame->outputReady) g_nativeVk.destroyEvent(g_nativeVk.device, frame->outputReady, nullptr);
if (frame->consumed) g_nativeVk.destroyEvent(g_nativeVk.device, frame->consumed, nullptr);
delete frame;
}
static bool native_vk_plane(NativeVkPlane& plane, const NgxVkResource* resource, int index, bool allowImport)
{
if (!resource || resource->type != 0) return false;
plane.image = resource->resource.image;
const auto& image = plane.image;
const VkImageAspectFlags aspect = image.range.aspectMask;
if (!image.image || !image.view || !image.width || !image.height ||
image.width > 16384 || image.height > 16384 || image.range.levelCount != 1 ||
image.range.layerCount != 1 || !d4r_vk_copy_aspect(image.format, aspect))
return false;
if (d4r_vk_depth_bytes(image.format) && aspect != VK_IMAGE_ASPECT_DEPTH_BIT)
return false; // a sampled depth view must not include the stencil aspect
size_t bytes;
plane.format = native_vk_format(image.format, bytes);
if (!bytes || (index == 4 ? !D4rVkStaging::supported_output(image.format) :
!supported_input(static_cast<Plane>(index), plane.format))) return false;
// Pack directly into CUDA's canonical layouts on the GPU. Sampling still
// avoids packed depth/stencil copies and requires no transfer image usage.
plane.format = index == 0 || index == 4 ? DXGI_FORMAT_R16G16B16A16_FLOAT :
index == 2 ? DXGI_FORMAT_R16G16_FLOAT : DXGI_FORMAT_R32_FLOAT;
plane.rowBytes = (index == 0 || index == 4 ? 8 : 4) * image.width;
VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
info.size = plane.rowBytes * image.height;
info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
const auto inputLayout = env_uint("D4R_VULKAN_INPUT_GENERAL", 0) != 0
? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
if (plane.buffer && plane.capacity >= info.size)
return g_nativeVkStaging.set(plane.staging, image.view, image.format, plane.buffer,
info.size, index == 4, inputLayout);
if (plane.buffer)
{
const auto descriptor = plane.image;
const auto format = plane.format;
const auto pitch = plane.rowBytes;
native_vk_free_plane(plane);
plane.image = descriptor; plane.format = format; plane.rowBytes = pitch;
}
if (native_vk_vram_plane(plane, info.size, allowImport))
return g_nativeVkStaging.set(plane.staging, image.view, image.format, plane.buffer,
info.size, index == 4, inputLayout);
if (g_nativeVk.createBuffer(g_nativeVk.device, &info, nullptr, &plane.buffer) != VK_SUCCESS) return false;
VkMemoryRequirements requirements;
g_nativeVk.requirements(g_nativeVk.device, plane.buffer, &requirements);
uint32_t type = UINT32_MAX;
const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
// Reading float4 pixels from uncached mapped GTT/BAR memory makes every
// scalar conversion pay uncached access latency. Prefer cached, coherent
// host memory, retaining the coherent fallback on devices without it.
for (int cached = 1; cached >= 0 && type == UINT32_MAX; --cached)
for (uint32_t i = 0; i < g_nativeVk.memory.memoryTypeCount; ++i)
if ((requirements.memoryTypeBits & (1u << i)) &&
(g_nativeVk.memory.memoryTypes[i].propertyFlags & flags) == flags &&
(!cached || (g_nativeVk.memory.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)))
{ type = i; break; }
if (type == UINT32_MAX) return false;
VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = requirements.size;
allocation.memoryTypeIndex = type;
static unsigned memoryReports = 0;
if ((index == 0 || index == 4) && memoryReports < 6)
{
++memoryReports;
logf("Vulkan staging %s memory: type=%u flags=0x%x bytes=%llu",
index == 4 ? "output" : "input", type,
g_nativeVk.memory.memoryTypes[type].propertyFlags,
static_cast<unsigned long long>(info.size));
}
if (!(g_nativeVk.allocate(g_nativeVk.device, &allocation, nullptr, &plane.memory) == VK_SUCCESS &&
g_nativeVk.bind(g_nativeVk.device, plane.buffer, plane.memory, 0) == VK_SUCCESS &&
g_nativeVk.map(g_nativeVk.device, plane.memory, 0, VK_WHOLE_SIZE, 0,
reinterpret_cast<void**>(&plane.mapped)) == VK_SUCCESS)) return false;
plane.capacity = info.size;
return g_nativeVkStaging.set(plane.staging, image.view, image.format, plane.buffer,
info.size, index == 4, inputLayout);
}
static bool native_vk_consumed(const NativeVkFrame& frame)
{
return frame.done.load() && g_nativeVk.eventStatus(g_nativeVk.device, frame.consumed) == VK_EVENT_SET;
}
static void native_vk_reap(std::vector<NativeVkFrame*>& frames)
{
for (auto it = frames.begin(); it != frames.end();)
if (native_vk_consumed(**it)) { native_vk_free_frame(*it); it = frames.erase(it); }
else ++it;
}
static NativeVkFrame* native_vk_reuse(NativeVkFeature& owner, VkCommandBuffer cmd)
{
for (auto it = owner.frames.begin(); it != owner.frames.end();)
if (native_vk_consumed(**it)) {
if ((*it)->waiter.joinable()) (*it)->waiter.join();
owner.reusable.push_back(*it); it = owner.frames.erase(it);
} else ++it;
if (owner.frames.size() >= 8) return nullptr;
// Re-recording this same command buffer proves its previous recording was
// reset. Reuse only after CPU production AND GPU consumption completed.
for (auto it = owner.reusable.begin(); it != owner.reusable.end(); ++it)
if ((*it)->commandBuffer == cmd) {
auto* frame = *it; owner.reusable.erase(it);
if (g_nativeVk.resetEvent(g_nativeVk.device, frame->inputReady) != VK_SUCCESS ||
g_nativeVk.resetEvent(g_nativeVk.device, frame->outputReady) != VK_SUCCESS ||
g_nativeVk.resetEvent(g_nativeVk.device, frame->consumed) != VK_SUCCESS) {
native_vk_free_frame(frame); return nullptr;
}
frame->cancel = false; frame->done = false; frame->result = NGX_SUCCESS; frame->params = {};
return frame;
}
while (owner.reusable.size() > 8) {
native_vk_free_frame(owner.reusable.front()); owner.reusable.erase(owner.reusable.begin());
}
return nullptr;
}
// Worker only. D4R_SHIM_LINEAR_INPUTS: NGX samples the shared VRAM buffer through a
// pitch-linear texture (reported to NGX as an array by the bridge), skipping the
// buffer->array copy. Rows must meet the texture pitch alignment; others copy.
static CudaObject native_vk_linear_texture(NativeVkPlane& plane, int index)
{
static const bool enabled = env_uint("D4R_SHIM_LINEAR_INPUTS", 0) != 0;
if (!enabled || !plane.cuda || g.cu.registerLinearTexture == nullptr || plane.rowBytes % 256 != 0)
return 0;
const uint64_t key = (static_cast<uint64_t>(index) << 60) ^ (static_cast<uint64_t>(plane.rowBytes) << 32) ^
(static_cast<uint64_t>(plane.image.width) << 16) ^ plane.image.height;
if (plane.texture && plane.textureKey == key) return plane.texture;
if (plane.texture) g.cu.texObjectDestroy(plane.texture);
plane.texture = 0;
const Plane kind = static_cast<Plane>(index);
struct Pitch2D { CudaDevicePtr pointer; uint32_t format, channels; size_t width, height, pitch; } pitch2D =
{plane.cuda, plane_format(kind), plane_channels(kind), plane.image.width, plane.image.height, plane.rowBytes};
static_assert(sizeof(Pitch2D) == 40, "CUDA_RESOURCE_DESC pitch2D");
CudaResourceDesc resource = {};
resource.resType = 3; // CU_RESOURCE_TYPE_PITCH2D
memcpy(&resource.res, &pitch2D, sizeof(pitch2D));
const CudaTextureDesc sampler = input_sampler(plane.image.width, pitch2D.channels, kind == Plane::Color);
CudaObject object = 0;
int result = g.cu.texObjectCreate(&object, &resource, &sampler, nullptr);
if (result == 0)
result = g.cu.registerLinearTexture(object, plane.image.width, plane.image.height, pitch2D.format, pitch2D.channels);
if (result != 0) {
logf("Vulkan linear input %d (%ux%u, pitch %zu) failed: %d; copying", index, plane.image.width,
plane.image.height, plane.rowBytes, result);
if (object) g.cu.texObjectDestroy(object);
return 0;
}
plane.texture = object; plane.textureKey = key;
return object;
}
// Polls the input event. D4R_VULKAN_INPUT_POLL_US=N polls every N us; unset is
// adaptive like the shim's output wait: sleep in chunks until shortly before the
// predicted readiness (smoothed over earlier frames), then yield, backing off
// when the game runs late. Detection latency is on the frame's critical path.
static VkResult native_vk_wait_input(Feature* feature, NativeVkFrame* frame)
{
const auto start = std::chrono::steady_clock::now();
const auto deadline = start + std::chrono::milliseconds(env_uint("D4R_VULKAN_SUBMIT_TIMEOUT_MS", 30000));
static const int fixedPoll = [] {
const char* v = std::getenv("D4R_VULKAN_INPUT_POLL_US");
return v != nullptr && *v != '\0' ? static_cast<int>(env_uint("D4R_VULKAN_INPUT_POLL_US", 200)) : -1;
}();
constexpr double kMarginUs = 120.0, kStepUs = 20.0, kChunkUs = 200.0, kSpinAfterUs = 150.0;
const double predicted = feature->vkInputWaitEmaUs.load();
double lastNotReadyUs = -1.0;
VkResult ready = VK_EVENT_RESET;
while (!frame->cancel && (ready = g_nativeVk.eventStatus(g_nativeVk.device, frame->inputReady)) == VK_EVENT_RESET &&
std::chrono::steady_clock::now() < deadline)
{
if (fixedPoll >= 0) { d4r_sleep_us(static_cast<unsigned>(fixedPoll)); continue; }
const double elapsed = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - start).count();
lastNotReadyUs = elapsed;
const double remaining = predicted - kMarginUs - elapsed;
if (remaining > kStepUs) d4r_sleep_us(static_cast<unsigned>(std::min(remaining, kChunkUs)));
else if (elapsed < predicted + kSpinAfterUs) d4r_sleep_us(0);
else d4r_sleep_us(static_cast<unsigned>(std::min(kStepUs * (1.0 + (elapsed - predicted - kSpinAfterUs) / 160.0), kChunkUs)));
}
if (ready == VK_EVENT_SET && fixedPoll < 0) {
const double waited = std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now() - start).count();
// Learn the readiness time (between the last NOT_READY and the successful query), not our own wait.
double sample = std::min(lastNotReadyUs >= 0.0 ? 0.5 * (lastNotReadyUs + waited) : waited, 20000.0);
feature->vkInputWaitEmaUs.store(predicted == 0.0 ? sample : 0.8 * predicted + 0.2 * sample);
}
return ready;
}
static void native_vk_run(Feature* feature, NativeVkFrame* frame)
{
const auto waitStarted = ProfileClock::now();
// Independent submission waiter: recording order need not match submit
// order. Never block the CUDA worker on an unsubmitted command buffer.
const VkResult ready = native_vk_wait_input(feature, frame);
NgxResult result = NGX_FAIL_PLATFORM_ERROR;
const auto inputsReadyAt = ProfileClock::now();
if (!frame->cancel && ready == VK_EVENT_SET)
{
result = g.worker.call([&]() -> NgxResult {
const auto stageStart = ProfileClock::now();
InputSlot& slot = feature->inputs[0];
for (int i = 0; i < 4; ++i)
{
auto& p = frame->planes[i];
if (!p.buffer) continue;
if (p.cuda) {
slot.host[i].width = p.image.width; slot.host[i].height = p.image.height;
slot.host[i].rowBytes = p.rowBytes;
continue;
}
Staging staging;
staging.mapped = p.mapped;
staging.width = p.image.width; staging.height = p.image.height;
staging.format = p.format;
staging.layout.Footprint.RowPitch = static_cast<UINT>(p.rowBytes);
stage_plane(static_cast<Plane>(i), staging, slot.host[i], nullptr, i);
}
auto& p = frame->params;
const auto staged = ProfileClock::now();
bool ok = true, copied = false;
CudaImage* images[] = {&feature->color, &feature->depth, &feature->motion, &feature->exposure};
CudaObject linear[4] = {};
for (int i = 0; i < 4 && ok; ++i) {
auto& plane = frame->planes[i];
if (!plane.buffer) continue;
copied = copied || !(linear[i] = native_vk_linear_texture(plane, i));
if (linear[i]) continue;
if (!plane.cuda) {
ok = upload_plane(static_cast<Plane>(i), slot.host[i], *images[i], frame->number, nullptr, i);
continue;
}
ok = ensure_cuda_image(*images[i], plane.image.width, plane.image.height,
plane_format(static_cast<Plane>(i)), plane_channels(static_cast<Plane>(i)), false, i == 0);
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_DEVICE; copy.srcDevice = plane.cuda; copy.srcPitch = plane.rowBytes;
copy.dstMemoryType = CUDA_MEMORY_ARRAY; copy.dstArray = images[i]->array;
copy.WidthInBytes = plane.rowBytes; copy.Height = plane.image.height;
if (ok) ok = copy_2d(copy) == 0;
}
ok = ok && ensure_cuda_image(feature->output, feature->outWidth, feature->outHeight, CUDA_FORMAT_HALF, 4, true, false);
CudaObject dilated = ok ? prepare_dilated_motion(*feature, slot, p) : 0;
// NGX does not evaluate on the copies' stream: they must finish first. Linear
// inputs were completed by the game's GPU before the input event.
if (!ok || ((copied || dilated) && g.cu.ctxSynchronize() != 0)) return NGX_FAIL_PLATFORM_ERROR;
const auto uploaded = ProfileClock::now();
if (feature->evaluatedFrames++ == 0) p.reset = 1;
set_evaluation_parameters(*feature, p);
CudaObject* handles[] = {&feature->colorHandle, &feature->depthHandle, &feature->motionHandle, &feature->exposureHandle};
for (int i = 0; i < 4; ++i)
if (linear[i]) *handles[i] = linear[i]; // the parameters point at these variables
if (dilated) feature->motionHandle = dilated;
const auto& destination = frame->planes[4];
// D4R_SHIM_OUTPUT_DIRECT: the native output kernel stores straight into the
// shared output buffer. Set while the previous output kernel was native,
// undone below when this frame's was not (it then wrote the array).
static const bool outputDirect = env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0;
const bool nativeOutput = g.cu.outputKernelNative == nullptr || g.cu.outputKernelNative() == 1;
if (outputDirect && feature->outputDirectAllowed && g.cu.setArrayRedirect != nullptr &&
feature->output.array != nullptr && (nativeOutput || feature->outputRedirected)) {
const bool redirect = nativeOutput && destination.cuda &&
g.cu.setArrayRedirect(feature->output.array, destination.cuda, static_cast<uint32_t>(destination.rowBytes)) == 0;
if (!redirect && feature->outputRedirected) g.cu.setArrayRedirect(feature->output.array, 0, 0);
static bool logged = false;
if (redirect && !logged) { logf("Vulkan frame %u: direct output (no array copy)", frame->number); logged = true; }
feature->outputRedirected = redirect;
}
NgxResult evaluated = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
if (evaluated == NGX_FAIL_INVALID_PARAMETER && recreate_for_render_size(*feature, p, frame->number))
evaluated = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
if (feature->outputRedirected && g.cu.outputKernelNative != nullptr && g.cu.outputKernelNative() != 1) {
g.cu.setArrayRedirect(feature->output.array, 0, 0);
feature->outputRedirected = false;
}
if (synchronize_default_stream(*feature, nullptr, &feature->evalWaitEmaUs) != 0) return NGX_FAIL_PLATFORM_ERROR;
if (evaluated != NGX_SUCCESS) return evaluated;
const auto evaluatedAt = ProfileClock::now();
if (destination.cuda && feature->outputRedirected) {
// already in the shared buffer
} else if (destination.cuda) {
CudaMemcpy2D copy = {};
copy.srcMemoryType = CUDA_MEMORY_ARRAY; copy.srcArray = feature->output.array;
copy.dstMemoryType = CUDA_MEMORY_DEVICE; copy.dstDevice = destination.cuda; copy.dstPitch = destination.rowBytes;
copy.WidthInBytes = destination.rowBytes; copy.Height = destination.image.height;
if (copy_2d(copy) != 0 || g.cu.ctxSynchronize() != 0) return NGX_FAIL_PLATFORM_ERROR;
} else if (!download_output(*feature, feature->outputHost[0], frame->number, nullptr)) return NGX_FAIL_PLATFORM_ERROR;
const auto downloaded = ProfileClock::now();
if (!destination.cuda) std::memcpy(destination.mapped, feature->outputHost[0].bytes, feature->outputHost[0].size());
feature->completedFrames++;
if (frame->number <= 3 || frame->number % 120 == 0 || profile_enabled())
logf("D4R_VULKAN_PROFILE frame=%u stage=%.3f upload=%.3f evaluate=%.3f download=%.3f encode=%.3f ms",
frame->number, profile_ms(stageStart, staged), profile_ms(staged, uploaded),
profile_ms(uploaded, evaluatedAt), profile_ms(evaluatedAt, downloaded),
profile_ms(downloaded, ProfileClock::now()));
if (frame->number <= 3) logf("native Vulkan frame %u CUDA output ready (%ux%u)",
frame->number, destination.image.width, destination.image.height);
return NGX_SUCCESS;
});
}
if (result != NGX_SUCCESS)
{
// Never leave the GPU permanently waiting on an abandoned/failed job.
// Signal a cleared output and expose failure on the next API call.
auto& output = frame->planes[4];
if (output.mapped) std::memset(output.mapped, 0, output.rowBytes * output.image.height);
// VRAM output is cleared by the recorded GPU commands before releasing
// it to CUDA. A cancelled, unsubmitted job must not write unowned memory.
logf("Vulkan frame %u failed/cancelled (0x%08x, input event %d)", frame->number, result, ready);
}
frame->result = result;
const auto signalStarted = ProfileClock::now();
if (g_nativeVk.setEvent(g_nativeVk.device, frame->outputReady) != VK_SUCCESS)
frame->result = NGX_FAIL_PLATFORM_ERROR;
frame->done = true;
if (frame->number <= 3 || frame->number % 120 == 0 || profile_enabled())
logf("D4R_VULKAN_WAIT frame=%u input_ready=%.3f signal=%.3f total=%.3f ms",
frame->number, profile_ms(waitStarted, inputsReadyAt),
profile_ms(signalStarted, ProfileClock::now()), profile_ms(waitStarted, ProfileClock::now()));
}
static void native_vk_process_exit();
static void WINAPI native_vk_shutdown_hook() { native_vk_process_exit(); }
static NgxResult native_vk_init(unsigned long long id, const wchar_t* path, VkInstance instance,
VkPhysicalDevice physical, VkDevice device, PFN_vkGetInstanceProcAddr instanceProc,
PFN_vkGetDeviceProcAddr deviceProc, const NgxFeatureCommonInfo* info,
unsigned int version, const ProjectIdentity* project = nullptr)
{
std::lock_guard<std::mutex> lock(g_nativeVkMutex);
if (!instance || !physical || !device || g.device) return NGX_FAIL_INVALID_PARAMETER;
if (g_nativeVk.device) return g_nativeVk.device == device ? NGX_SUCCESS : NGX_FAIL_INVALID_PARAMETER;
if (!instanceProc)
{
HMODULE module = LoadLibraryA("vulkan-1.dll");
if (!module || !load_export(module, "vkGetInstanceProcAddr", instanceProc)) return NGX_FAIL_PLATFORM_ERROR;
}
if (!deviceProc) deviceProc = reinterpret_cast<PFN_vkGetDeviceProcAddr>(instanceProc(instance, "vkGetDeviceProcAddr"));
auto memory = reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(instanceProc(instance, "vkGetPhysicalDeviceMemoryProperties"));
if (!deviceProc || !memory) return NGX_FAIL_PLATFORM_ERROR;
NativeVkApi api;
api.device = device;
memory(physical, &api.memory);
bool ok = true;
auto load = [&](auto& fn, const char* name) {
fn = reinterpret_cast<std::remove_reference_t<decltype(fn)>>(deviceProc(device, name));
ok &= fn != nullptr;
};
load(api.createBuffer, "vkCreateBuffer"); load(api.destroyBuffer, "vkDestroyBuffer");
load(api.requirements, "vkGetBufferMemoryRequirements"); load(api.allocate, "vkAllocateMemory");
load(api.free, "vkFreeMemory"); load(api.bind, "vkBindBufferMemory");
load(api.map, "vkMapMemory"); load(api.unmap, "vkUnmapMemory");
load(api.createEvent, "vkCreateEvent"); load(api.destroyEvent, "vkDestroyEvent");
load(api.eventStatus, "vkGetEventStatus"); load(api.setEvent, "vkSetEvent");
load(api.resetEvent, "vkResetEvent");
load(api.cmdSetEvent, "vkCmdSetEvent"); load(api.waitEvents, "vkCmdWaitEvents");
load(api.barrier, "vkCmdPipelineBarrier"); load(api.idle, "vkDeviceWaitIdle");
load(api.fill, "vkCmdFillBuffer");
if (!ok) return NGX_FAIL_PLATFORM_ERROR;
// Match NGX's CUDA adapter identity to the supplied Vulkan physical device.
auto properties2 = reinterpret_cast<PFN_vkGetPhysicalDeviceProperties2>(instanceProc(instance, "vkGetPhysicalDeviceProperties2"));
if (!properties2) properties2 = reinterpret_cast<PFN_vkGetPhysicalDeviceProperties2>(instanceProc(instance, "vkGetPhysicalDeviceProperties2KHR"));
if (properties2)
{
VkPhysicalDeviceIDProperties identity = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES};
VkPhysicalDeviceProperties2 properties = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity};
properties2(physical, &properties);
if (identity.deviceLUIDValid)
{
uint32_t low, high;
std::memcpy(&low, identity.deviceLUID, 4); std::memcpy(&high, identity.deviceLUID + 4, 4);
char value[24];
std::snprintf(value, sizeof(value), "0x%08x", low); SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", value);
std::snprintf(value, sizeof(value), "0x%08x", high); SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", value);
std::snprintf(value, sizeof(value), "%u", identity.deviceNodeMask); SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", value);
}
}
D4rVkStaging staging;
if (!staging.init(device, deviceProc)) { staging.destroy(); return NGX_FAIL_PLATFORM_ERROR; }
const auto result = initialize(id, path, nullptr, info, version, project);
if (result == NGX_SUCCESS)
{
auto families = reinterpret_cast<PFN_vkGetPhysicalDeviceQueueFamilyProperties>(instanceProc(instance, "vkGetPhysicalDeviceQueueFamilyProperties"));
if (families) {
uint32_t count = 0; families(physical, &count, nullptr);
std::vector<VkQueueFamilyProperties> properties(count); families(physical, &count, properties.data());
for (uint32_t i = 0; i < count && api.familyCount < 8; ++i)
if (properties[i].queueFlags & VK_QUEUE_COMPUTE_BIT) api.families[api.familyCount++] = i;
if (api.familyCount == 1) api.queueFamily = api.families[0];
}
load_export(g.cuda, "d4rImportVulkanMemory", api.import);
load_export(g.cuda, "d4rReleaseVulkanMemory", api.release);
// NGX's own detach tears CUDA down before ours runs; the bridge calls this first.
void(WINAPI* setShutdownHook)(void(WINAPI*)()) = nullptr;
if (load_export(g.cuda, "d4rSetShutdownHook", setShutdownHook)) setShutdownHook(native_vk_shutdown_hook);
else logf("CUDA bridge lacks d4rSetShutdownHook: a game exit during DLSS can still reset the GPU ring");
api.external = env_uint("D4R_VULKAN_VRAM_INTEROP", 0) != 0 && api.import && api.release &&
api.familyCount > 0 && deviceProc(device, "vkGetMemoryWin32HandleKHR") != nullptr;
g_nativeVk = api;
g_nativeVkStaging = staging;
logf("native Vulkan NGX ready: device=%p, sampled/storage compute staging (experimental), VRAM interop=%d, "
"%u compute-capable queue famil%s", static_cast<void*>(device), api.external, api.familyCount,
api.familyCount == 1 ? "y" : "ies (concurrent sharing)");
}
else staging.destroy();
return result;
}
static void native_vk_ownership(VkCommandBuffer cmd, NativeVkPlane& p, bool external)
{
if (!p.cuda || p.externalOwned == external) return;
VkBufferMemoryBarrier b = {VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER};
b.buffer = p.buffer; b.offset = 0; b.size = VK_WHOLE_SIZE;
// Concurrent buffers are shared by every family the game may record on (RADV keeps buffers
// uncompressed, so CUDA's view needs no transfer); an exclusive one moves to/from EXTERNAL.
const bool concurrent = g_nativeVk.familyCount > 1;
b.srcQueueFamilyIndex = concurrent ? VK_QUEUE_FAMILY_IGNORED : external ? g_nativeVk.queueFamily : VK_QUEUE_FAMILY_EXTERNAL;
b.dstQueueFamilyIndex = concurrent ? VK_QUEUE_FAMILY_IGNORED : external ? VK_QUEUE_FAMILY_EXTERNAL : g_nativeVk.queueFamily;
b.srcAccessMask = external ? VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT : 0;
b.dstAccessMask = external ? 0 : VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
g_nativeVk.barrier(cmd, external ? VK_PIPELINE_STAGE_ALL_COMMANDS_BIT : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
external ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT : VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
0, 0, nullptr, 1, &b, 0, nullptr);
p.externalOwned = external;
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init(unsigned long long id, const wchar_t* path, VkInstance instance,
VkPhysicalDevice physical, VkDevice device, PFN_vkGetInstanceProcAddr ip, PFN_vkGetDeviceProcAddr dp,
const NgxFeatureCommonInfo* info, unsigned int version)
{ return native_vk_init(id, path, instance, physical, device, ip, dp, info, version); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init_Ext(unsigned long long id, const wchar_t* path, VkInstance instance,
VkPhysicalDevice physical, VkDevice device, unsigned int version, const NgxFeatureCommonInfo* info)
{ return native_vk_init(id, path, instance, physical, device, nullptr, nullptr, info, version); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init_Ext2(unsigned long long id, const wchar_t* path, VkInstance instance,
VkPhysicalDevice physical, VkDevice device, PFN_vkGetInstanceProcAddr ip, PFN_vkGetDeviceProcAddr dp,
unsigned int version, const NgxFeatureCommonInfo* info)
{ return native_vk_init(id, path, instance, physical, device, ip, dp, info, version); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init_with_ProjectID(const char* id, int engine, const char* engineVersion,
const wchar_t* path, VkInstance instance, VkPhysicalDevice physical, VkDevice device,
PFN_vkGetInstanceProcAddr ip, PFN_vkGetDeviceProcAddr dp, const NgxFeatureCommonInfo* info, unsigned int version)
{
ProjectIdentity project{id ? id : "", engine, engineVersion ? engineVersion : ""};
return native_vk_init(0, path, instance, physical, device, ip, dp, info, version, &project);
}
// The NGX core/OptiScaler project-ID exports put SDKVersion before FeatureInfo.
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init_ProjectID(const char* id, int engine, const char* engineVersion,
const wchar_t* path, VkInstance instance, VkPhysicalDevice physical, VkDevice device,
PFN_vkGetInstanceProcAddr ip, PFN_vkGetDeviceProcAddr dp, unsigned int version, const NgxFeatureCommonInfo* info)
{ return NVSDK_NGX_VULKAN_Init_with_ProjectID(id, engine, engineVersion, path, instance, physical, device, ip, dp, info, version); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Init_ProjectID_Ext(const char* id, int engine, const char* engineVersion,
const wchar_t* path, VkInstance instance, VkPhysicalDevice physical, VkDevice device,
PFN_vkGetInstanceProcAddr ip, PFN_vkGetDeviceProcAddr dp, unsigned int version, const NgxFeatureCommonInfo* info)
{ return NVSDK_NGX_VULKAN_Init_ProjectID(id, engine, engineVersion, path, instance, physical, device, ip, dp, version, info); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetParameters(void** p)
{ return p ? NVSDK_NGX_D3D12_GetParameters(p) : NGX_FAIL_INVALID_PARAMETER; }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_AllocateParameters(void** p)
{ return p ? NVSDK_NGX_D3D12_AllocateParameters(p) : NGX_FAIL_INVALID_PARAMETER; }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetCapabilityParameters(void** p)
{ return p ? NVSDK_NGX_D3D12_GetCapabilityParameters(p) : NGX_FAIL_INVALID_PARAMETER; }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_DestroyParameters(void* p)
{ return NVSDK_NGX_D3D12_DestroyParameters(p); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetScratchBufferSize(unsigned int feature, const void* p, size_t* size)
{ return size ? NVSDK_NGX_D3D12_GetScratchBufferSize(feature, p, size) : NGX_FAIL_INVALID_PARAMETER; }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_RequiredExtensions(unsigned int* ic, const char*** ie, unsigned int* dc, const char*** de)
{
if (!ic || !ie || !dc || !de) return NGX_FAIL_INVALID_PARAMETER;
*ic = *dc = 0; *ie = *de = nullptr;
return NGX_SUCCESS; // CPU staging requires no NVIDIA or external-memory extensions
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureInstanceExtensionRequirements(const void*, uint32_t* count, VkExtensionProperties** p)
{
// Callers query the count with a null list first (Streamline does).
if (!count) return NGX_FAIL_INVALID_PARAMETER;
*count = 0;
if (p) *p = nullptr;
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureDeviceExtensionRequirements(VkInstance, VkPhysicalDevice,
const void* info, uint32_t* count, VkExtensionProperties** p)
{ return NVSDK_NGX_VULKAN_GetFeatureInstanceExtensionRequirements(info, count, p); }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureRequirements(VkInstance, VkPhysicalDevice, const void*, NgxFeatureRequirement* p)
{ return p ? NVSDK_NGX_D3D12_GetFeatureRequirements(nullptr, nullptr, p) : NGX_FAIL_INVALID_PARAMETER; }
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_CreateFeature(VkCommandBuffer, unsigned int id, void* p, NgxHandle** handle)
{
std::lock_guard<std::mutex> lock(g_nativeVkMutex);
if (!g_nativeVk.device) return NGX_FAIL_NOT_INITIALIZED;
const auto result = create_feature(id, p, handle, true);
if (result == NGX_SUCCESS) g_nativeVkFeatures.push_back({find_feature(*handle), {}});
return result;
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_CreateFeature1(VkDevice device, VkCommandBuffer cmd, unsigned int id, void* p, NgxHandle** handle)
{
if (device != g_nativeVk.device) return NGX_FAIL_INVALID_PARAMETER;
return NVSDK_NGX_VULKAN_CreateFeature(cmd, id, p, handle);
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_EvaluateFeature(VkCommandBuffer cmd, const NgxHandle* handle, void* p, void*)
{
std::lock_guard<std::mutex> lock(g_nativeVkMutex);
const auto apiStarted = ProfileClock::now();
if (!g_nativeVk.device || !g.ngxInitialized) return NGX_FAIL_NOT_INITIALIZED;
auto it = std::find_if(g_nativeVkFeatures.begin(), g_nativeVkFeatures.end(),
[&](const auto& f) { return &f.feature->handle == handle; });
if (!cmd || !p || it == g_nativeVkFeatures.end()) return NGX_FAIL_INVALID_PARAMETER;
auto& owner = *it;
if (owner.feature->retiring) return NGX_FAIL_INVALID_PARAMETER;
for (auto* frame : owner.frames)
if (frame->done && frame->result != NGX_SUCCESS) return frame->result;
native_vk_reap(g_nativeVkRetired);
auto* reusable = native_vk_reuse(owner, cmd);
const auto reapedAt = ProfileClock::now();
if (owner.frames.size() >= 8) return NGX_FAIL_PLATFORM_ERROR;
auto* frame = reusable ? reusable : new NativeVkFrame;
frame->commandBuffer = cmd;
// An import while any recording of ours may be waiting on the GPU can deadlock against it
// (Indiana Jones: 10 s compute ring timeout). Then only pooled buffers or host staging are used.
bool pending = !g_nativeVkRetired.empty();
for (const auto& f : g_nativeVkFeatures)
for (const auto* other : f.frames) pending = pending || !native_vk_consumed(*other);
const char* names[] = {"Color", "Depth", "MotionVectors", "ExposureTexture", "Output"};
for (int i = 0; i < 5; ++i)
{
void* resource = nullptr;
d4r_ngx_get_void(p, names[i], &resource);
if (i == 3 && !resource) { native_vk_free_plane(frame->planes[3]); continue; }
if (!native_vk_plane(frame->planes[i], static_cast<NgxVkResource*>(resource), i, !pending))
{
if (resource)
{
const auto* descriptor = static_cast<NgxVkResource*>(resource);
const auto& image = descriptor->resource.image;
logf("Vulkan rejected resource %s: type=%d image=%p view=%p format=%d "
"size=%ux%u aspect=0x%x mip=%u+%u layer=%u+%u", names[i], descriptor->type,
reinterpret_cast<void*>(image.image), reinterpret_cast<void*>(image.view), image.format,
image.width, image.height, image.range.aspectMask, image.range.baseMipLevel,
image.range.levelCount, image.range.baseArrayLayer, image.range.layerCount);
}
else logf("Vulkan missing resource %s", names[i]);
native_vk_free_frame(frame); return NGX_FAIL_UNSUPPORTED_FORMAT;
}
}
if (!pending && g_nativeVk.external) native_vk_prefill(*frame);
auto* feature = owner.feature;
const auto allocatedAt = ProfileClock::now();
if (feature->frame == 0)
for (int i = 0; i < 5; ++i)
{
const auto& plane = frame->planes[i];
if (plane.buffer) logf("Vulkan resource %s: format=%d size=%ux%u aspect=0x%x copyAspect=0x%x rowBytes=%zu",
names[i], plane.image.format, plane.image.width, plane.image.height,
plane.image.range.aspectMask, d4r_vk_copy_aspect(plane.image.format, plane.image.range.aspectMask),
plane.rowBytes);
}
if (frame->planes[4].image.width != feature->outWidth || frame->planes[4].image.height != feature->outHeight)
{ native_vk_free_frame(frame); return NGX_FAIL_INVALID_PARAMETER; }
auto& params = frame->params;
params.jitterX = get_float_or(p, "Jitter.Offset.X", 0); params.jitterY = get_float_or(p, "Jitter.Offset.Y", 0);
params.mvScaleX = get_float_or(p, "MV.Scale.X", 1); params.mvScaleY = get_float_or(p, "MV.Scale.Y", 1);
params.sharpness = get_float_or(p, "Sharpness", 0); params.reset = get_int_or(p, "Reset", 0);
params.preExposure = get_float_or(p, "DLSS.Pre.Exposure", 1); params.exposureScale = get_float_or(p, "DLSS.Exposure.Scale", 1);
params.frameTime = get_float_or(p, "FrameTimeDeltaInMsec", 16.6f);
params.renderWidth = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Width", feature->width);
params.renderHeight = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Height", feature->height);
if (!params.renderWidth) params.renderWidth = feature->width;
if (!params.renderHeight) params.renderHeight = feature->height;
params.colorBaseX = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.X", 0);
params.colorBaseY = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.Y", 0);
params.depthBaseX = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.X", 0);
params.depthBaseY = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.Y", 0);
params.mvBaseX = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.X", 0);
params.mvBaseY = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.Y", 0);
params.outputBaseX = get_uint_or(p, "DLSS.Output.Subrect.Base.X", 0);
params.outputBaseY = get_uint_or(p, "DLSS.Output.Subrect.Base.Y", 0);
params.hasExposure = frame->planes[3].buffer != VK_NULL_HANDLE;
params.invertX = get_int_or(p, "DLSS.Indicator.Invert.X.Axis", 0);
params.invertY = get_int_or(p, "DLSS.Indicator.Invert.Y.Axis", 0);
// Output subrect conversion and auxiliary masks are not implemented yet.
void* mask = nullptr;
d4r_ngx_get_void(p, "DLSS.Input.Bias.Current.Color.Mask", &mask);
void* transparency = nullptr; d4r_ngx_get_void(p, "TransparencyMask", &transparency);
if (params.outputBaseX || params.outputBaseY || mask || transparency ||
params.colorBaseX > frame->planes[0].image.width || params.colorBaseY > frame->planes[0].image.height ||
params.renderWidth > frame->planes[0].image.width - params.colorBaseX ||
params.renderHeight > frame->planes[0].image.height - params.colorBaseY)
{ native_vk_free_frame(frame); return NGX_FAIL_INVALID_PARAMETER; }
VkEventCreateInfo event = {VK_STRUCTURE_TYPE_EVENT_CREATE_INFO};
if (!frame->inputReady && (g_nativeVk.createEvent(g_nativeVk.device, &event, nullptr, &frame->inputReady) != VK_SUCCESS ||
g_nativeVk.createEvent(g_nativeVk.device, &event, nullptr, &frame->outputReady) != VK_SUCCESS ||
g_nativeVk.createEvent(g_nativeVk.device, &event, nullptr, &frame->consumed) != VK_SUCCESS))
{ native_vk_free_frame(frame); return NGX_FAIL_PLATFORM_ERROR; }
frame->number = ++feature->frame;
try { frame->waiter = std::thread([feature, frame] { native_vk_run(feature, frame); }); }
catch (...) { native_vk_free_frame(frame); return NGX_FAIL_PLATFORM_ERROR; }
// All validation/allocation precedes the first recorded command.
VkMemoryBarrier readable = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
readable.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
readable.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT;
g_nativeVk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
0, 1, &readable, 0, nullptr, 0, nullptr);
for (int i = 0; i < 4; ++i)
{
auto& plane = frame->planes[i];
if (!plane.buffer) continue;
native_vk_ownership(cmd, plane, false);
g_nativeVkStaging.record(cmd, plane.staging, plane.image.width, plane.image.height, i + 1);
}
auto& resultPlane = frame->planes[4];
if (resultPlane.cuda) {
native_vk_ownership(cmd, resultPlane, false);
g_nativeVk.fill(cmd, resultPlane.buffer, 0, resultPlane.rowBytes * resultPlane.image.height, 0);
}
for (auto& plane : frame->planes) native_vk_ownership(cmd, plane, true);
VkMemoryBarrier readback = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
readback.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; readback.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
g_nativeVk.barrier(cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &readback, 0, nullptr, 0, nullptr);
g_nativeVk.cmdSetEvent(cmd, frame->inputReady, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
VkMemoryBarrier upload = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
upload.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; upload.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
g_nativeVk.waitEvents(cmd, 1, &frame->outputReady, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
1, &upload, 0, nullptr, 0, nullptr);
auto& output = frame->planes[4];
native_vk_ownership(cmd, output, false);
g_nativeVkStaging.record(cmd, output.staging, output.image.width, output.image.height);
VkMemoryBarrier visible = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
visible.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
visible.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
g_nativeVk.barrier(cmd, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
0, 1, &visible, 0, nullptr, 0, nullptr);
// CPU production is not GPU consumption. This marker follows the image
// write and all earlier commands which reference the staging objects.
g_nativeVk.cmdSetEvent(cmd, frame->consumed, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
owner.frames.push_back(frame);
if (frame->number <= 3 || frame->number % 120 == 0 || profile_enabled())
logf("D4R_VULKAN_API frame=%u reclaim=%.3f allocate=%.3f record=%.3f total=%.3f ms",
frame->number, profile_ms(apiStarted, reapedAt), profile_ms(reapedAt, allocatedAt),
profile_ms(allocatedAt, ProfileClock::now()), profile_ms(apiStarted, ProfileClock::now()));
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_EvaluateFeature_C(VkCommandBuffer cmd, const NgxHandle* handle, void* p, void* callback)
{ return NVSDK_NGX_VULKAN_EvaluateFeature(cmd, handle, p, callback); }
static NgxResult native_vk_release(std::vector<NativeVkFeature>::iterator it)
{
for (auto* frame : it->frames) frame->cancel = true;
// Every waiter signals output before CUDA state disappears.
for (auto* frame : it->frames)
if (frame->waiter.joinable()) frame->waiter.join();
for (auto* frame : it->frames)
if (native_vk_consumed(*frame)) native_vk_free_frame(frame);
else g_nativeVkRetired.push_back(frame);
for (auto* frame : it->reusable) native_vk_free_frame(frame);
logf("native Vulkan feature %u released after %u completed frames",
it->feature->handle.Id, it->feature->completedFrames.load());
it->feature->retiring = true;
release_feature(it->feature);
g_nativeVkFeatures.erase(it);
return NGX_SUCCESS;
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_ReleaseFeature(NgxHandle* handle)
{
std::lock_guard<std::mutex> lock(g_nativeVkMutex);
auto it = std::find_if(g_nativeVkFeatures.begin(), g_nativeVkFeatures.end(),
[&](const auto& f) { return &f.feature->handle == handle; });
return it == g_nativeVkFeatures.end() ? NGX_FAIL_INVALID_PARAMETER : native_vk_release(it);
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Shutdown()
{
std::lock_guard<std::mutex> lock(g_nativeVkMutex);
NgxResult result = NGX_SUCCESS;
while (!g_nativeVkFeatures.empty())
if (native_vk_release(g_nativeVkFeatures.begin()) != NGX_SUCCESS) result = NGX_FAIL_PLATFORM_ERROR;
if (g_nativeVk.device)
{
// Shutdown is the application's teardown boundary: it must externally
// synchronize its queues and discard unsubmitted recordings first.
if (g_nativeVk.idle(g_nativeVk.device) != VK_SUCCESS) return NGX_FAIL_PLATFORM_ERROR;
for (auto* frame : g_nativeVkRetired) native_vk_free_frame(frame);
g_nativeVkRetired.clear();
for (auto& plane : g_nativeVkBufferPool) native_vk_free_plane(plane, true);
g_nativeVkBufferPool.clear(); g_nativeVkExternalAllocations = 0;
g_nativeVkStaging.destroy();
NVSDK_NGX_D3D12_Shutdown();
g_nativeVk = {};
}
return result;
}
// DLL_PROCESS_DETACH. A game may exit without NGX shutdown while a submitted
// command buffer still waits in vkCmdWaitEvents for an output event that only the
// (now killed) waiter thread would set. The wait sits inside the GPU ring, so the
// kernel cannot cancel it: the ring times out and resets 10 s after the exit.
// Release every such wait. No locking: other threads are gone, and one may have
// died holding the mutex.
static void native_vk_process_exit()
{
static std::atomic<bool> ran{false}; // the bridge's hook, then DLL_PROCESS_DETACH
if (!g_nativeVk.device || !g_nativeVk.setEvent || ran.exchange(true)) return;
unsigned released = 0;
auto release = [&](NativeVkFrame* frame) {
if (frame && !frame->done.load() && frame->outputReady &&
g_nativeVk.setEvent(g_nativeVk.device, frame->outputReady) == VK_SUCCESS) ++released;
};
for (auto& owner : g_nativeVkFeatures)
for (auto* frame : owner.frames) release(frame);
for (auto* frame : g_nativeVkRetired) release(frame);
if (released) logf("process exit: released %u pending Vulkan output wait(s)", released);
}
D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Shutdown1(VkDevice device)
{
if (device != g_nativeVk.device) return NGX_FAIL_INVALID_PARAMETER;
return NVSDK_NGX_VULKAN_Shutdown();
}
tools/d4r_vulkan_ngx.h
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include <vulkan/vulkan_core.h>
// ABI-compatible subset of NVIDIA nvsdk_ngx_defs_vk.h.
struct NgxVkImage
{
VkImageView view;
VkImage image;
VkImageSubresourceRange range;
VkFormat format;
unsigned int width, height;
};
struct NgxVkResource
{
union { NgxVkImage image; struct { VkBuffer buffer; unsigned int size; } buffer; } resource;
int type; // 0 = image view, 1 = buffer
bool readWrite;
};
tools/d4r_vulkan_staging.h
+160
View File
@@ -0,0 +1,160 @@
#pragma once
#include <vulkan/vulkan_core.h>
#include "d4r_vulkan_staging_spv.h"
// Uses the actual NGX contract: sampled input views and storage output views.
// The game need not create its images with either TRANSFER usage bit.
struct D4rVkStagingSet {
VkDescriptorPool pool = VK_NULL_HANDLE;
VkDescriptorSet set = VK_NULL_HANDLE;
VkPipeline pipeline = VK_NULL_HANDLE;
bool output = false;
};
struct D4rVkStaging {
VkDevice device = VK_NULL_HANDLE;
VkSampler sampler = VK_NULL_HANDLE;
VkDescriptorSetLayout readSet = VK_NULL_HANDLE, writeSet = VK_NULL_HANDLE;
VkPipelineLayout readLayout = VK_NULL_HANDLE, writeLayout = VK_NULL_HANDLE;
VkPipeline readPipeline = VK_NULL_HANDLE;
struct OutputPipeline { VkFormat format; VkPipeline pipeline; } outputs[5] = {};
PFN_vkCreateSampler createSampler = nullptr;
PFN_vkDestroySampler destroySampler = nullptr;
PFN_vkCreateDescriptorSetLayout createSetLayout = nullptr;
PFN_vkDestroyDescriptorSetLayout destroySetLayout = nullptr;
PFN_vkCreatePipelineLayout createLayout = nullptr;
PFN_vkDestroyPipelineLayout destroyLayout = nullptr;
PFN_vkCreateShaderModule createShader = nullptr;
PFN_vkDestroyShaderModule destroyShader = nullptr;
PFN_vkCreateComputePipelines createPipelines = nullptr;
PFN_vkDestroyPipeline destroyPipeline = nullptr;
PFN_vkCreateDescriptorPool createPool = nullptr;
PFN_vkDestroyDescriptorPool destroyPool = nullptr;
PFN_vkAllocateDescriptorSets allocateSets = nullptr;
PFN_vkUpdateDescriptorSets updateSets = nullptr;
PFN_vkCmdBindPipeline bindPipeline = nullptr;
PFN_vkCmdBindDescriptorSets bindSets = nullptr;
PFN_vkCmdPushConstants push = nullptr;
PFN_vkCmdDispatch dispatch = nullptr;
static bool supported_output(VkFormat format) {
return format == VK_FORMAT_R16G16B16A16_SFLOAT || format == VK_FORMAT_R32G32B32A32_SFLOAT ||
format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 || format == VK_FORMAT_A2B10G10R10_UNORM_PACK32 ||
format == VK_FORMAT_R8G8B8A8_UNORM;
}
bool pipeline(const uint32_t* code, size_t bytes, VkPipelineLayout layout, VkPipeline& result) {
VkShaderModuleCreateInfo shader = {VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
shader.codeSize = bytes; shader.pCode = code;
VkShaderModule module = VK_NULL_HANDLE;
if (createShader(device, &shader, nullptr, &module) != VK_SUCCESS) return false;
VkComputePipelineCreateInfo info = {VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
info.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO};
info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; info.stage.module = module; info.stage.pName = "main";
info.layout = layout;
const VkResult status = createPipelines(device, VK_NULL_HANDLE, 1, &info, nullptr, &result);
destroyShader(device, module, nullptr);
return status == VK_SUCCESS;
}
bool init(VkDevice d, PFN_vkGetDeviceProcAddr proc) {
device = d;
#define D4R_LOAD(member, name) member = reinterpret_cast<decltype(member)>(proc(d, "vk" #name)); if (!member) return false
D4R_LOAD(createSampler, CreateSampler); D4R_LOAD(destroySampler, DestroySampler);
D4R_LOAD(createSetLayout, CreateDescriptorSetLayout); D4R_LOAD(destroySetLayout, DestroyDescriptorSetLayout);
D4R_LOAD(createLayout, CreatePipelineLayout); D4R_LOAD(destroyLayout, DestroyPipelineLayout);
D4R_LOAD(createShader, CreateShaderModule); D4R_LOAD(destroyShader, DestroyShaderModule);
D4R_LOAD(createPipelines, CreateComputePipelines); D4R_LOAD(destroyPipeline, DestroyPipeline);
D4R_LOAD(createPool, CreateDescriptorPool); D4R_LOAD(destroyPool, DestroyDescriptorPool);
D4R_LOAD(allocateSets, AllocateDescriptorSets); D4R_LOAD(updateSets, UpdateDescriptorSets);
D4R_LOAD(bindPipeline, CmdBindPipeline); D4R_LOAD(bindSets, CmdBindDescriptorSets);
D4R_LOAD(push, CmdPushConstants); D4R_LOAD(dispatch, CmdDispatch);
#undef D4R_LOAD
VkSamplerCreateInfo sample = {VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
sample.magFilter = sample.minFilter = VK_FILTER_NEAREST;
sample.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
sample.addressModeU = sample.addressModeV = sample.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
if (createSampler(d, &sample, nullptr, &sampler) != VK_SUCCESS) return false;
for (int output = 0; output < 2; ++output) {
VkDescriptorSetLayoutBinding bindings[2] = {};
bindings[0] = {0, output ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
bindings[1] = {1, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
VkDescriptorSetLayoutCreateInfo set = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
set.bindingCount = 2; set.pBindings = bindings;
auto& setLayout = output ? writeSet : readSet;
if (createSetLayout(d, &set, nullptr, &setLayout) != VK_SUCCESS) return false;
VkPushConstantRange range = {VK_SHADER_STAGE_COMPUTE_BIT, 0, output ? 8u : 12u};
VkPipelineLayoutCreateInfo layout = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
layout.setLayoutCount = 1; layout.pSetLayouts = &setLayout;
layout.pushConstantRangeCount = 1; layout.pPushConstantRanges = &range;
if (createLayout(d, &layout, nullptr, output ? &writeLayout : &readLayout) != VK_SUCCESS) return false;
}
return pipeline(d4r_vk_read, sizeof(d4r_vk_read), readLayout, readPipeline);
}
VkPipeline output_pipeline(VkFormat format) {
for (auto& item : outputs) if (item.pipeline && item.format == format) return item.pipeline;
const uint32_t* code = nullptr; size_t bytes = 0;
#define D4R_CODE(name) code = d4r_vk_write_##name; bytes = sizeof(d4r_vk_write_##name)
switch (format) {
case VK_FORMAT_R16G16B16A16_SFLOAT: D4R_CODE(rgba16f); break;
case VK_FORMAT_R32G32B32A32_SFLOAT: D4R_CODE(rgba32f); break;
case VK_FORMAT_B10G11R11_UFLOAT_PACK32: D4R_CODE(r11f_g11f_b10f); break;
case VK_FORMAT_A2B10G10R10_UNORM_PACK32: D4R_CODE(rgb10_a2); break;
case VK_FORMAT_R8G8B8A8_UNORM: D4R_CODE(rgba8); break;
default: return VK_NULL_HANDLE;
}
#undef D4R_CODE
for (auto& item : outputs) if (!item.pipeline) {
if (!pipeline(code, bytes, writeLayout, item.pipeline)) return VK_NULL_HANDLE;
item.format = format; return item.pipeline;
}
return VK_NULL_HANDLE;
}
bool set(D4rVkStagingSet& result, VkImageView image, VkFormat format, VkBuffer buffer,
VkDeviceSize bytes, bool output, VkImageLayout inputLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) {
result.output = output;
result.pipeline = output ? output_pipeline(format) : readPipeline;
if (!result.pipeline) return false;
VkDescriptorPoolSize sizes[] = {{output ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1},
{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}};
VkDescriptorPoolCreateInfo pool = {VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
pool.maxSets = 1; pool.poolSizeCount = 2; pool.pPoolSizes = sizes;
if (!result.pool && createPool(device, &pool, nullptr, &result.pool) != VK_SUCCESS) return false;
const auto layout = output ? writeSet : readSet;
VkDescriptorSetAllocateInfo alloc = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
alloc.descriptorPool = result.pool; alloc.descriptorSetCount = 1; alloc.pSetLayouts = &layout;
if (!result.set && allocateSets(device, &alloc, &result.set) != VK_SUCCESS) return false;
VkDescriptorImageInfo texture = {output ? VK_NULL_HANDLE : sampler, image,
output ? VK_IMAGE_LAYOUT_GENERAL : inputLayout};
VkDescriptorBufferInfo storage = {buffer, 0, bytes};
VkWriteDescriptorSet writes[2] = {};
writes[0] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
writes[0].dstSet = result.set; writes[0].dstBinding = 0; writes[0].descriptorCount = 1;
writes[0].descriptorType = sizes[0].type; writes[0].pImageInfo = &texture;
writes[1] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
writes[1].dstSet = result.set; writes[1].dstBinding = 1; writes[1].descriptorCount = 1;
writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; writes[1].pBufferInfo = &storage;
updateSets(device, 2, writes, 0, nullptr);
return true;
}
void record(VkCommandBuffer cmd, const D4rVkStagingSet& set, uint32_t width, uint32_t height, uint32_t inputMode = 0) {
const auto layout = set.output ? writeLayout : readLayout;
bindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, set.pipeline);
bindSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &set.set, 0, nullptr);
const uint32_t dimensions[] = {width, height, inputMode};
push(cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, set.output ? 8 : 12, dimensions);
dispatch(cmd, (width+7)/8, (height+7)/8, 1);
}
void free(D4rVkStagingSet& set) {
if (set.pool) destroyPool(device, set.pool, nullptr);
set = {};
}
void destroy() {
if (readPipeline) destroyPipeline(device, readPipeline, nullptr);
for (auto& item : outputs) if (item.pipeline) destroyPipeline(device, item.pipeline, nullptr);
if (readLayout) destroyLayout(device, readLayout, nullptr);
if (writeLayout) destroyLayout(device, writeLayout, nullptr);
if (readSet) destroySetLayout(device, readSet, nullptr);
if (writeSet) destroySetLayout(device, writeSet, nullptr);
if (sampler) destroySampler(device, sampler, nullptr);
*this = {};
}
};
tools/vulkan_depth_copy_probe.cpp
+83
View File
@@ -0,0 +1,83 @@
// Real-GPU check of the native NGX depth-aspect copy contract (no CUDA/NGX).
#include "d4r_vulkan_depth.h"
#include <cassert>
#include <cmath>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
static void check(VkResult r) { if (r != VK_SUCCESS) { std::fprintf(stderr,"Vulkan error %d\n",r); std::abort(); } }
int main()
{
assert(d4r_vk_copy_aspect(VK_FORMAT_D32_SFLOAT_S8_UINT, VK_IMAGE_ASPECT_STENCIL_BIT)==0);
assert(d4r_vk_copy_aspect(VK_FORMAT_D32_SFLOAT_S8_UINT, VK_IMAGE_ASPECT_DEPTH_BIT|VK_IMAGE_ASPECT_COLOR_BIT)==0);
VkApplicationInfo app={VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion=VK_API_VERSION_1_1;
VkInstanceCreateInfo ic={VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; ic.pApplicationInfo=&app;
VkInstance instance; check(vkCreateInstance(&ic,nullptr,&instance));
uint32_t count=0;check(vkEnumeratePhysicalDevices(instance,&count,nullptr));
std::vector<VkPhysicalDevice> devices(count);check(vkEnumeratePhysicalDevices(instance,&count,devices.data()));
VkPhysicalDevice physical=VK_NULL_HANDLE;
for(auto candidate:devices) { VkPhysicalDeviceProperties p;vkGetPhysicalDeviceProperties(candidate,&p);if(p.vendorID==0x1002){physical=candidate;std::printf("GPU: %s\n",p.deviceName);break;} }
assert(physical);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,nullptr);
std::vector<VkQueueFamilyProperties> families(count);vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,families.data());
uint32_t family=0;while(family<count && !(families[family].queueFlags&VK_QUEUE_GRAPHICS_BIT))++family;assert(family<count);
float priority=1;
VkDeviceQueueCreateInfo qi={VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};qi.queueFamilyIndex=family;qi.queueCount=1;qi.pQueuePriorities=&priority;
VkDeviceCreateInfo dc={VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};dc.queueCreateInfoCount=1;dc.pQueueCreateInfos=&qi;
VkDevice device;check(vkCreateDevice(physical,&dc,nullptr,&device));
VkQueue queue;vkGetDeviceQueue(device,family,0,&queue);
VkPhysicalDeviceMemoryProperties memory;vkGetPhysicalDeviceMemoryProperties(physical,&memory);
auto allocate=[&](VkMemoryRequirements req,VkMemoryPropertyFlags flags){
uint32_t type=0;while(type<memory.memoryTypeCount && (!(req.memoryTypeBits&(1u<<type)) || (memory.memoryTypes[type].propertyFlags&flags)!=flags))++type;assert(type<memory.memoryTypeCount);
VkMemoryAllocateInfo info={VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};info.allocationSize=req.size;info.memoryTypeIndex=type;
VkDeviceMemory result;check(vkAllocateMemory(device,&info,nullptr,&result));return result;
};
VkCommandPoolCreateInfo pi={VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};pi.queueFamilyIndex=family;
VkCommandPool pool;check(vkCreateCommandPool(device,&pi,nullptr,&pool));
VkCommandBufferAllocateInfo ca={VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};ca.commandPool=pool;ca.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY;ca.commandBufferCount=1;
VkCommandBuffer cmd;check(vkAllocateCommandBuffers(device,&ca,&cmd));
unsigned verified=0;
for(auto format:{VK_FORMAT_D16_UNORM_S8_UINT,VK_FORMAT_D24_UNORM_S8_UINT,VK_FORMAT_D32_SFLOAT_S8_UINT})
{
VkImageFormatProperties properties;
VkResult support=vkGetPhysicalDeviceImageFormatProperties(physical,format,VK_IMAGE_TYPE_2D,VK_IMAGE_TILING_OPTIMAL,VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT,0,&properties);
if(support==VK_ERROR_FORMAT_NOT_SUPPORTED){std::printf("format %d: unsupported by GPU (skipped)\n",format);continue;}check(support);
for(auto viewAspect:{static_cast<VkImageAspectFlags>(VK_IMAGE_ASPECT_DEPTH_BIT),static_cast<VkImageAspectFlags>(VK_IMAGE_ASPECT_DEPTH_BIT|VK_IMAGE_ASPECT_STENCIL_BIT)})
{
constexpr unsigned W=7,H=5; // odd widths detect accidental packed-depth strides
const size_t bytes=d4r_vk_depth_bytes(format);assert(bytes==2 || bytes==4);
VkImageCreateInfo ii={VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};ii.imageType=VK_IMAGE_TYPE_2D;ii.format=format;ii.extent={W,H,1};ii.mipLevels=ii.arrayLayers=1;ii.samples=VK_SAMPLE_COUNT_1_BIT;ii.tiling=VK_IMAGE_TILING_OPTIMAL;ii.usage=VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT;
VkImage image;check(vkCreateImage(device,&ii,nullptr,&image));VkMemoryRequirements req;vkGetImageMemoryRequirements(device,image,&req);auto im=allocate(req,0);check(vkBindImageMemory(device,image,im,0));
VkBufferCreateInfo bi={VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};bi.size=W*H*bytes;bi.usage=VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBuffer buffer;check(vkCreateBuffer(device,&bi,nullptr,&buffer));vkGetBufferMemoryRequirements(device,buffer,&req);auto bm=allocate(req,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);check(vkBindBufferMemory(device,buffer,bm,0));
check(vkResetCommandPool(device,pool,0));VkCommandBufferBeginInfo begin={VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};check(vkBeginCommandBuffer(cmd,&begin));
VkImageMemoryBarrier barrier={VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};barrier.image=image;barrier.subresourceRange={d4r_vk_barrier_aspect(format,viewAspect),0,1,0,1};barrier.srcQueueFamilyIndex=barrier.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED;barrier.oldLayout=VK_IMAGE_LAYOUT_UNDEFINED;barrier.newLayout=VK_IMAGE_LAYOUT_GENERAL;barrier.dstAccessMask=VK_ACCESS_TRANSFER_WRITE_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,VK_PIPELINE_STAGE_TRANSFER_BIT,0,0,nullptr,0,nullptr,1,&barrier);
VkClearDepthStencilValue clear={0.25f,0xab};vkCmdClearDepthStencilImage(cmd,image,VK_IMAGE_LAYOUT_GENERAL,&clear,1,&barrier.subresourceRange);
barrier.oldLayout=VK_IMAGE_LAYOUT_GENERAL;barrier.newLayout=VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;barrier.srcAccessMask=VK_ACCESS_TRANSFER_WRITE_BIT;barrier.dstAccessMask=VK_ACCESS_TRANSFER_READ_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_TRANSFER_BIT,0,0,nullptr,0,nullptr,1,&barrier);
VkBufferImageCopy copy={};copy.imageSubresource={d4r_vk_copy_aspect(format,viewAspect),0,0,1};copy.imageExtent={W,H,1};assert(copy.imageSubresource.aspectMask==VK_IMAGE_ASPECT_DEPTH_BIT);
vkCmdCopyImageToBuffer(cmd,image,VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,buffer,1,&copy);
barrier.oldLayout=VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;barrier.newLayout=VK_IMAGE_LAYOUT_GENERAL;barrier.srcAccessMask=VK_ACCESS_TRANSFER_READ_BIT;barrier.dstAccessMask=VK_ACCESS_MEMORY_READ_BIT|VK_ACCESS_MEMORY_WRITE_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,0,0,nullptr,0,nullptr,1,&barrier);
VkMemoryBarrier host={VK_STRUCTURE_TYPE_MEMORY_BARRIER};host.srcAccessMask=VK_ACCESS_TRANSFER_WRITE_BIT;host.dstAccessMask=VK_ACCESS_HOST_READ_BIT;vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_HOST_BIT,0,1,&host,0,nullptr,0,nullptr);
check(vkEndCommandBuffer(cmd));VkSubmitInfo si={VK_STRUCTURE_TYPE_SUBMIT_INFO};si.commandBufferCount=1;si.pCommandBuffers=&cmd;check(vkQueueSubmit(queue,1,&si,VK_NULL_HANDLE));check(vkQueueWaitIdle(queue));
void* mapped;check(vkMapMemory(device,bm,0,VK_WHOLE_SIZE,0,&mapped));
for(unsigned n=0;n<W*H;++n)
{
float value=0;
const auto* source=static_cast<const uint8_t*>(mapped)+n*bytes;
if(format==VK_FORMAT_D32_SFLOAT_S8_UINT)std::memcpy(&value,source,4);
else if(bytes==2){uint16_t v;std::memcpy(&v,source,2);value=static_cast<float>(v)/65535.f;}
else{uint32_t v;std::memcpy(&v,source,4);value=static_cast<float>(v&0xffffffu)/16777215.f;}
assert(std::isfinite(value) && std::abs(value-0.25f)<0.00002f);
}
std::printf("format %d viewAspect=0x%x: %zu-byte depth copy, 35 correct samples\n",format,viewAspect,bytes);++verified;
vkUnmapMemory(device,bm);vkDestroyBuffer(device,buffer,nullptr);vkFreeMemory(device,bm,nullptr);vkDestroyImage(device,image,nullptr);vkFreeMemory(device,im,nullptr);
}
}
assert(verified>=4); // at least two supported combined formats, each with both view masks
vkDestroyCommandPool(device,pool,nullptr);vkDestroyDevice(device,nullptr);vkDestroyInstance(instance,nullptr);
}
tools/vulkan_dlss_harness.cpp
+260
View File
@@ -0,0 +1,260 @@
// Synthetic native-Vulkan NGX integration test. No SDK or Vulkan import library.
// usage: vulkan_dlss_harness.exe SHIM_DLL OUTPUT_RAW [FRAMES [discard|invalid|dynamic|recreate|exit]]
// exit: the last frame is submitted and the process exits at once, without NGX shutdown.
#define VK_NO_PROTOTYPES
#include <windows.h>
#include "d4r_vulkan_ngx.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
extern "C" {
void d4r_ngx_set_uint(void*, const char*, unsigned int);
void d4r_ngx_set_int(void*, const char*, int);
void d4r_ngx_set_float(void*, const char*, float);
void d4r_ngx_set_void(void*, const char*, void*);
}
using Result = unsigned int;
struct Handle { unsigned int id; };
using Init = Result (*)(unsigned long long, const wchar_t*, VkInstance, VkPhysicalDevice, VkDevice,
PFN_vkGetInstanceProcAddr, PFN_vkGetDeviceProcAddr, const void*, unsigned int);
using Params = Result (*)(void**);
using Destroy = Result (*)(void*);
using Create = Result (*)(VkCommandBuffer, unsigned int, void*, Handle**);
using Eval = Result (*)(VkCommandBuffer, const Handle*, void*, void*);
using Release = Result (*)(Handle*);
using Shutdown = Result (*)();
#define VK_FUNCTIONS(X) \
X(CreateInstance) X(EnumeratePhysicalDevices) X(GetPhysicalDeviceProperties) X(EnumerateDeviceExtensionProperties) \
X(GetPhysicalDeviceQueueFamilyProperties) X(GetPhysicalDeviceMemoryProperties) X(CreateDevice) \
X(GetDeviceProcAddr) X(GetDeviceQueue) X(CreateCommandPool) X(AllocateCommandBuffers) \
X(BeginCommandBuffer) X(EndCommandBuffer) X(ResetCommandPool) X(QueueSubmit) X(QueueWaitIdle) X(CreateFence) X(WaitForFences) X(ResetFences) X(DestroyFence) \
X(CreateImage) X(GetImageMemoryRequirements) X(AllocateMemory) X(BindImageMemory) X(CreateImageView) \
X(CreateBuffer) X(GetBufferMemoryRequirements) X(BindBufferMemory) X(MapMemory) X(UnmapMemory) \
X(CmdPipelineBarrier) X(CmdClearColorImage) X(CmdCopyImageToBuffer) X(DestroyImageView) X(DestroyImage) \
X(DestroyBuffer) X(FreeMemory) X(DestroyCommandPool) X(DestroyDevice) X(DestroyInstance)
#define DECLARE(name) static PFN_vk##name vk##name;
VK_FUNCTIONS(DECLARE)
#undef DECLARE
// -DD4R_HARNESS_SCALE=2 runs DOOM's Performance size (1280x720 -> 2560x1440) for timing.
#ifndef D4R_HARNESS_SCALE
#define D4R_HARNESS_SCALE 1
#endif
constexpr uint32_t W = 640 * D4R_HARNESS_SCALE, H = 360 * D4R_HARNESS_SCALE, OW = 2 * W, OH = 2 * H;
static VkDevice device;
static VkPhysicalDeviceMemoryProperties memory;
static void check(VkResult r, const char* name) { if (r != VK_SUCCESS) { std::fprintf(stderr, "%s: %d\n", name, r); std::exit(1); } }
static void ngx(Result r, const char* name) { if (r != 1) { std::fprintf(stderr, "%s: 0x%08x\n", name, r); std::exit(1); } }
static VkDeviceMemory allocate(VkMemoryRequirements requirements, VkMemoryPropertyFlags flags)
{
uint32_t type = UINT32_MAX;
for (int cached = (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) ? 1 : 0; cached >= 0 && type == UINT32_MAX; --cached)
for (uint32_t i = 0; i < memory.memoryTypeCount; ++i)
if ((requirements.memoryTypeBits & (1u << i)) && (memory.memoryTypes[i].propertyFlags & flags) == flags &&
(!cached || (memory.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT))) { type = i; break; }
if (type == UINT32_MAX) { std::fprintf(stderr, "no memory type\n"); std::exit(1); }
VkMemoryAllocateInfo info = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
info.allocationSize = requirements.size; info.memoryTypeIndex = type;
VkDeviceMemory result;
check(vkAllocateMemory(device, &info, nullptr, &result), "allocate"); return result;
}
static NgxVkResource make_image(uint32_t w, uint32_t h, VkFormat format, VkDeviceMemory& allocation)
{
VkImageCreateInfo info = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
info.imageType = VK_IMAGE_TYPE_2D; info.format = format; info.extent = {w,h,1};
info.mipLevels = info.arrayLayers = 1; info.samples = VK_SAMPLE_COUNT_1_BIT;
info.tiling = VK_IMAGE_TILING_OPTIMAL;
info.usage = w == OW && h == OH ? VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT :
VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
NgxVkResource resource = {};
auto& image = resource.resource.image;
check(vkCreateImage(device, &info, nullptr, &image.image), "create image");
VkMemoryRequirements requirements;
vkGetImageMemoryRequirements(device, image.image, &requirements);
allocation = allocate(requirements, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
check(vkBindImageMemory(device, image.image, allocation, 0), "bind image");
image.range = {VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1}; image.format = format; image.width = w; image.height = h;
VkImageViewCreateInfo view = {VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
view.image = image.image; view.viewType = VK_IMAGE_VIEW_TYPE_2D; view.format = format; view.subresourceRange = image.range;
check(vkCreateImageView(device, &view, nullptr, &image.view), "image view");
resource.readWrite = true; return resource;
}
int main(int argc, char** argv)
{
if (argc < 3) return 2;
if (const char* log = std::getenv("D4R_HARNESS_LOG")) {
if (!std::freopen(log, "w", stdout)) return 2;
if (!std::freopen(log, "a", stderr)) return 2;
}
std::setvbuf(stdout, nullptr, _IONBF, 0);
std::puts("loading Vulkan and NGX shim");
const unsigned frames = argc > 3 ? std::strtoul(argv[3], nullptr, 10) : 3;
const bool discard = argc > 4 && !std::strcmp(argv[4], "discard");
const bool invalid = argc > 4 && !std::strcmp(argv[4], "invalid");
const bool dynamic = argc > 4 && !std::strcmp(argv[4], "dynamic");
const bool recreate = argc > 4 && !std::strcmp(argv[4], "recreate");
const bool exitEarly = argc > 4 && !std::strcmp(argv[4], "exit");
HMODULE vulkan = LoadLibraryA("vulkan-1.dll"), shim = LoadLibraryA(argv[1]);
if (!vulkan || !shim) { std::fprintf(stderr,"DLL load: %lu\n",GetLastError()); return 1; }
auto ip = reinterpret_cast<PFN_vkGetInstanceProcAddr>(reinterpret_cast<void*>(GetProcAddress(vulkan,"vkGetInstanceProcAddr")));
#define LOAD(name) vk##name = reinterpret_cast<PFN_vk##name>(ip(VK_NULL_HANDLE, "vk" #name));
LOAD(CreateInstance)
#undef LOAD
VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion = VK_API_VERSION_1_1;
VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; ci.pApplicationInfo = &app;
VkInstance instance;
check(vkCreateInstance(&ci,nullptr,&instance),"instance");
#define LOAD(name) vk##name = reinterpret_cast<PFN_vk##name>(ip(instance, "vk" #name)); if (!vk##name) return 1;
VK_FUNCTIONS(LOAD)
#undef LOAD
uint32_t count = 0;
check(vkEnumeratePhysicalDevices(instance,&count,nullptr),"devices");
std::vector<VkPhysicalDevice> devices(count);
check(vkEnumeratePhysicalDevices(instance,&count,devices.data()),"devices");
VkPhysicalDevice physical = VK_NULL_HANDLE;
for (auto d : devices) { VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(d,&p); std::printf("Adapter: %s vendor=%04x\n",p.deviceName,p.vendorID); if (p.vendorID == 0x1002 || p.vendorID == 0x10de) { physical=d; std::printf("GPU: %s\n",p.deviceName); break; } }
if (!physical) return 1;
vkGetPhysicalDeviceMemoryProperties(physical,&memory);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,nullptr);
std::vector<VkQueueFamilyProperties> families(count);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,families.data());
uint32_t family = UINT32_MAX;
// D4R_HARNESS_COMPUTE_QUEUE=1 records on a compute-only family, as games with async compute DLSS do.
const bool computeQueue = std::getenv("D4R_HARNESS_COMPUTE_QUEUE") != nullptr;
for (uint32_t i=0;i<count;++i)
if (computeQueue ? (families[i].queueFlags & VK_QUEUE_COMPUTE_BIT) && !(families[i].queueFlags & VK_QUEUE_GRAPHICS_BIT)
: (families[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0) { family=i; break; }
std::printf("queue family %u (%s)\n", family, computeQueue ? "compute" : "graphics");
if (family == UINT32_MAX) return 1;
float priority = 1;
VkDeviceQueueCreateInfo queueInfo = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; queueInfo.queueFamilyIndex=family; queueInfo.queueCount=1; queueInfo.pQueuePriorities=&priority;
VkDeviceCreateInfo di = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; di.queueCreateInfoCount=1; di.pQueueCreateInfos=&queueInfo;
const char* externalExtension = "VK_KHR_external_memory_win32";
if (std::getenv("D4R_HARNESS_EXTERNAL_MEMORY")) {
check(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,nullptr),"extensions");
std::vector<VkExtensionProperties> extensions(count);
check(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,extensions.data()),"extensions");
bool supported = false;
for (const auto& e : extensions) supported |= std::strcmp(e.extensionName,externalExtension)==0;
if (!supported) { std::fputs("external memory extension unavailable\n",stderr); return 1; }
di.enabledExtensionCount=1; di.ppEnabledExtensionNames=&externalExtension;
std::puts("enabling external-memory VRAM interop");
}
check(vkCreateDevice(physical,&di,nullptr,&device),"device");
auto dp = vkGetDeviceProcAddr;
#define LOAD(name) vk##name = reinterpret_cast<PFN_vk##name>(dp(device, "vk" #name));
// Instance entry points above remain valid; only load device entry points here.
LOAD(GetDeviceQueue) LOAD(QueueSubmit) LOAD(QueueWaitIdle)
#undef LOAD
#define NGX(name, type) auto name = reinterpret_cast<type>(reinterpret_cast<void*>(GetProcAddress(shim,"NVSDK_NGX_VULKAN_" #name))); if (!name) return 1;
NGX(Init, ::Init) NGX(GetCapabilityParameters, Params) NGX(DestroyParameters, Destroy)
NGX(CreateFeature, Create) NGX(EvaluateFeature, Eval) NGX(ReleaseFeature, Release) NGX(Shutdown, ::Shutdown)
#undef NGX
std::puts("initializing CUDA NGX through Vulkan frontend");
ngx(Init(100899111ULL,L"Z:\\tmp",instance,physical,device,ip,dp,nullptr,0x14),"init");
void* params = nullptr; ngx(GetCapabilityParameters(&params),"capability parameters");
d4r_ngx_set_uint(params,"Width",W); d4r_ngx_set_uint(params,"Height",H);
d4r_ngx_set_uint(params,"OutWidth",OW); d4r_ngx_set_uint(params,"OutHeight",OH);
d4r_ngx_set_int(params,"PerfQualityValue",2); d4r_ngx_set_int(params,"DLSS.Feature.Create.Flags",0x6b);
Handle* handle = nullptr; ngx(CreateFeature(VK_NULL_HANDLE,1,params,&handle),"create");
VkDeviceMemory allocations[4];
NgxVkResource images[] = {make_image(W,H,VK_FORMAT_R16G16B16A16_SFLOAT,allocations[0]),
make_image(W,H,VK_FORMAT_R32_SFLOAT,allocations[1]), make_image(W,H,VK_FORMAT_R16G16_SFLOAT,allocations[2]),
make_image(OW,OH,VK_FORMAT_R16G16B16A16_SFLOAT,allocations[3])};
const char* names[] = {"Color","Depth","MotionVectors","Output"};
// D4R_HARNESS_DEPTH_ASPECT=1: label the R32F depth image with the depth aspect, as Indiana Jones
// (through Streamline/OptiScaler) does; the harness keeps the real colour range for its own barriers.
NgxVkResource described[4] = {images[0], images[1], images[2], images[3]};
if (std::getenv("D4R_HARNESS_DEPTH_ASPECT")) described[1].resource.image.range.aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;
for (int i=0;i<4;++i) d4r_ngx_set_void(params,names[i],&described[i]);
// D4R_HARNESS_IN_FLIGHT=N: up to N submitted frames at once (own pool, command buffer and fence each),
// as a game does; only the final frame is read back.
const unsigned inFlight = std::max(1u, std::min(8u, static_cast<unsigned>(std::getenv("D4R_HARNESS_IN_FLIGHT") ? std::atoi(std::getenv("D4R_HARNESS_IN_FLIGHT")) : 1)));
VkCommandPool pools[8]; VkCommandBuffer commands[8]; VkFence fences[8];
for (unsigned i=0;i<inFlight;++i) {
VkCommandPoolCreateInfo poolInfo = {VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.queueFamilyIndex=family;
check(vkCreateCommandPool(device,&poolInfo,nullptr,&pools[i]),"pool");
VkCommandBufferAllocateInfo ai = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; ai.commandPool=pools[i]; ai.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY; ai.commandBufferCount=1;
check(vkAllocateCommandBuffers(device,&ai,&commands[i]),"command buffer");
VkFenceCreateInfo fi = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; fi.flags=VK_FENCE_CREATE_SIGNALED_BIT;
check(vkCreateFence(device,&fi,nullptr,&fences[i]),"fence");
}
VkBufferCreateInfo bi = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bi.size=OW*OH*8; bi.usage=VK_BUFFER_USAGE_TRANSFER_DST_BIT;
VkBuffer readback; check(vkCreateBuffer(device,&bi,nullptr,&readback),"readback");
VkMemoryRequirements req; vkGetBufferMemoryRequirements(device,readback,&req);
VkDeviceMemory readMemory=allocate(req,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
check(vkBindBufferMemory(device,readback,readMemory,0),"bind readback");
VkQueue queue; vkGetDeviceQueue(device,family,0,&queue);
for (unsigned n=0;n<frames;++n)
{
if (dynamic) {
const unsigned phase=(n/4)%3;
d4r_ngx_set_uint(params,"DLSS.Render.Subrect.Dimensions.Width",W*(8-phase)/8);
d4r_ngx_set_uint(params,"DLSS.Render.Subrect.Dimensions.Height",H*(8-phase)/8);
}
const unsigned slot = n % inFlight; VkCommandBuffer cmd = commands[slot];
check(vkWaitForFences(device,1,&fences[slot],VK_TRUE,UINT64_MAX),"fence wait"); check(vkResetFences(device,1,&fences[slot]),"fence reset");
check(vkResetCommandPool(device,pools[slot],0),"reset pool");
if (recreate && n && n % 4 == 0) {
ngx(ReleaseFeature(handle),"release before recreation");
const unsigned phase=(n/4)%3;
d4r_ngx_set_uint(params,"Width",W*(8-phase)/8);
d4r_ngx_set_uint(params,"Height",H*(8-phase)/8);
d4r_ngx_set_int(params,"PerfQualityValue",phase);
ngx(CreateFeature(VK_NULL_HANDLE,1,params,&handle),"recreate");
std::printf("recreated feature: quality=%u render=%ux%u\n",phase,W*(8-phase)/8,H*(8-phase)/8);
}
VkCommandBufferBeginInfo begin={VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
check(vkBeginCommandBuffer(cmd,&begin),"begin");
for (int i=0;i<4;++i)
{
VkImageMemoryBarrier b={VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; b.oldLayout=VK_IMAGE_LAYOUT_UNDEFINED;
b.newLayout=i==3 ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
b.srcQueueFamilyIndex=b.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED; b.image=images[i].resource.image.image; b.subresourceRange=images[i].resource.image.range;
b.dstAccessMask=i==3 ? VK_ACCESS_SHADER_WRITE_BIT : VK_ACCESS_TRANSFER_WRITE_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,i==3 ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : VK_PIPELINE_STAGE_TRANSFER_BIT,0,0,nullptr,0,nullptr,1,&b);
if(i==3) continue;
VkClearColorValue clear={};
if (i==0) { clear.float32[0]=0.25f; clear.float32[1]=0.5f; clear.float32[2]=0.75f; clear.float32[3]=1; }
if (i==1) clear.float32[0]=0.5f;
vkCmdClearColorImage(cmd,b.image,VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,&clear,1,&b.subresourceRange);
b.oldLayout=VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; b.newLayout=VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
b.srcAccessMask=VK_ACCESS_TRANSFER_WRITE_BIT; b.dstAccessMask=VK_ACCESS_SHADER_READ_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,0,0,nullptr,0,nullptr,1,&b);
}
if (invalid) described[2].type=1;
const Result evaluated=EvaluateFeature(cmd,handle,params,nullptr);
if (invalid) { if (evaluated != 0xBAD0000e) return 1; std::puts("invalid resource rejected"); break; }
ngx(evaluated,"evaluate");
if (discard) { std::puts("discarded command buffer; releasing pending feature"); break; }
auto& image=images[3].resource.image;
VkBufferImageCopy copy={}; copy.imageSubresource={VK_IMAGE_ASPECT_COLOR_BIT,0,0,1}; copy.imageExtent={OW,OH,1};
VkMemoryBarrier output={VK_STRUCTURE_TYPE_MEMORY_BARRIER}; output.srcAccessMask=VK_ACCESS_MEMORY_WRITE_BIT; output.dstAccessMask=VK_ACCESS_TRANSFER_READ_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_TRANSFER_BIT,0,1,&output,0,nullptr,0,nullptr);
vkCmdCopyImageToBuffer(cmd,image.image,VK_IMAGE_LAYOUT_GENERAL,readback,1,&copy);
VkMemoryBarrier host={VK_STRUCTURE_TYPE_MEMORY_BARRIER}; host.srcAccessMask=VK_ACCESS_TRANSFER_WRITE_BIT; host.dstAccessMask=VK_ACCESS_HOST_READ_BIT;
vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_HOST_BIT,0,1,&host,0,nullptr,0,nullptr);
check(vkEndCommandBuffer(cmd),"end");
VkSubmitInfo submit={VK_STRUCTURE_TYPE_SUBMIT_INFO}; submit.commandBufferCount=1; submit.pCommandBuffers=&cmd;
check(vkQueueSubmit(queue,1,&submit,fences[slot]),"submit");
if (exitEarly && n+1==frames) { std::puts("exiting with the last frame in flight"); std::fflush(stdout); ExitProcess(0); }
if (inFlight > 1 && n+1 < frames) continue; // pipelined: only the last frame is checked
check(vkQueueWaitIdle(queue),"wait");
void* data; check(vkMapMemory(device,readMemory,0,VK_WHOLE_SIZE,0,&data),"map");
const auto* values=static_cast<const uint16_t*>(data);
unsigned nonzero=0, nonfinite=0;
for (unsigned j=0;j<OW*OH*4;++j) { nonzero += (values[j]&0x7fff)!=0; nonfinite += (values[j]&0x7c00)==0x7c00; }
std::printf("frame %u: nonzero=%u nonfinite=%u center=%04x,%04x,%04x,%04x\n",n,nonzero,nonfinite,values[((OH/2)*OW+OW/2)*4],values[((OH/2)*OW+OW/2)*4+1],values[((OH/2)*OW+OW/2)*4+2],values[((OH/2)*OW+OW/2)*4+3]);
if (!nonzero || nonfinite) return 1;
if (n+1==frames) { FILE* out=std::fopen(argv[2],"wb"); if (!out) return 1; const auto written=std::fwrite(data,1,OW*OH*8,out); std::fclose(out); if (written!=OW*OH*8) return 1; }
vkUnmapMemory(device,readMemory);
}
ngx(ReleaseFeature(handle),"release"); ngx(DestroyParameters(params),"destroy params"); ngx(Shutdown(),"shutdown");
check(vkQueueWaitIdle(queue),"final wait");
for (unsigned i=0;i<inFlight;++i) { vkDestroyCommandPool(device,pools[i],nullptr); vkDestroyFence(device,fences[i],nullptr); } vkDestroyBuffer(device,readback,nullptr); vkFreeMemory(device,readMemory,nullptr);
for (int i=0;i<4;++i) { vkDestroyImageView(device,images[i].resource.image.view,nullptr); vkDestroyImage(device,images[i].resource.image.image,nullptr); vkFreeMemory(device,allocations[i],nullptr); }
vkDestroyDevice(device,nullptr); vkDestroyInstance(instance,nullptr);
return 0;
}
tools/vulkan_staging_probe.cpp
+126
View File
@@ -0,0 +1,126 @@
// Real GPU regression for sampled-only inputs and storage-only output writes.
#include "d4r_vulkan_staging.h"
#include <cassert>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
static void check(VkResult r) { if (r != VK_SUCCESS) { std::fprintf(stderr,"Vulkan error %d\n",r); std::exit(1); } }
struct Image { VkImage image; VkImageView view; VkDeviceMemory memory; VkFormat format; bool depth; };
int main() {
constexpr uint32_t W=7, H=5;
constexpr size_t BYTES=W*H*16;
VkApplicationInfo app={VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion=VK_API_VERSION_1_1;
VkInstanceCreateInfo ic={VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};ic.pApplicationInfo=&app;
VkInstance instance;check(vkCreateInstance(&ic,nullptr,&instance));
uint32_t count=0;check(vkEnumeratePhysicalDevices(instance,&count,nullptr));
std::vector<VkPhysicalDevice> devices(count);check(vkEnumeratePhysicalDevices(instance,&count,devices.data()));
VkPhysicalDevice physical=VK_NULL_HANDLE;
for(auto d:devices){VkPhysicalDeviceProperties p;vkGetPhysicalDeviceProperties(d,&p);if(p.vendorID==0x1002){physical=d;std::printf("GPU: %s\n",p.deviceName);break;}}
assert(physical);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,nullptr);std::vector<VkQueueFamilyProperties> families(count);
vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,families.data());
uint32_t family=0;while(family<count && !(families[family].queueFlags&VK_QUEUE_COMPUTE_BIT))++family;assert(family<count);
float priority=1;VkDeviceQueueCreateInfo qi={VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};qi.queueFamilyIndex=family;qi.queueCount=1;qi.pQueuePriorities=&priority;
VkPhysicalDeviceFeatures supported, enabled={};vkGetPhysicalDeviceFeatures(physical,&supported);
enabled.shaderStorageImageExtendedFormats=supported.shaderStorageImageExtendedFormats;
VkDeviceCreateInfo dc={VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};dc.queueCreateInfoCount=1;dc.pQueueCreateInfos=&qi;dc.pEnabledFeatures=&enabled;
VkDevice device;check(vkCreateDevice(physical,&dc,nullptr,&device));VkQueue queue;vkGetDeviceQueue(device,family,0,&queue);
VkPhysicalDeviceMemoryProperties memory;vkGetPhysicalDeviceMemoryProperties(physical,&memory);
auto allocate=[&](VkMemoryRequirements req,VkMemoryPropertyFlags flags){
uint32_t type=0;while(type<memory.memoryTypeCount && (!(req.memoryTypeBits&(1u<<type)) || (memory.memoryTypes[type].propertyFlags&flags)!=flags))++type;assert(type<memory.memoryTypeCount);
VkMemoryAllocateInfo a={VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};a.allocationSize=req.size;a.memoryTypeIndex=type;VkDeviceMemory m;check(vkAllocateMemory(device,&a,nullptr,&m));return m;
};
auto image=[&](VkFormat format,bool output){
Image result={};result.format=format;result.depth=format==VK_FORMAT_D32_SFLOAT_S8_UINT;
VkImageCreateInfo info={VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};info.imageType=VK_IMAGE_TYPE_2D;info.format=format;info.extent={W,H,1};info.mipLevels=info.arrayLayers=1;info.samples=VK_SAMPLE_COUNT_1_BIT;info.tiling=VK_IMAGE_TILING_OPTIMAL;
// Input initialization uses only TRANSFER_DST. Outputs have neither
// TRANSFER bit: all writes and verification reads use the new shaders.
info.usage=VK_IMAGE_USAGE_SAMPLED_BIT | (output ? VK_IMAGE_USAGE_STORAGE_BIT : VK_IMAGE_USAGE_TRANSFER_DST_BIT);
check(vkCreateImage(device,&info,nullptr,&result.image));VkMemoryRequirements req;vkGetImageMemoryRequirements(device,result.image,&req);
result.memory=allocate(req,0);check(vkBindImageMemory(device,result.image,result.memory,0));
VkImageViewCreateInfo view={VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};view.image=result.image;view.viewType=VK_IMAGE_VIEW_TYPE_2D;view.format=format;
view.subresourceRange={result.depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1};
check(vkCreateImageView(device,&view,nullptr,&result.view));return result;
};
auto destroyImage=[&](Image& i){vkDestroyImageView(device,i.view,nullptr);vkDestroyImage(device,i.image,nullptr);vkFreeMemory(device,i.memory,nullptr);};
VkBufferCreateInfo bi={VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};bi.size=BYTES;bi.usage=VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
VkBuffer buffers[2];VkDeviceMemory allocations[2];float* mapped[2];
for(int i=0;i<2;++i){check(vkCreateBuffer(device,&bi,nullptr,&buffers[i]));VkMemoryRequirements req;vkGetBufferMemoryRequirements(device,buffers[i],&req);allocations[i]=allocate(req,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);check(vkBindBufferMemory(device,buffers[i],allocations[i],0));check(vkMapMemory(device,allocations[i],0,VK_WHOLE_SIZE,0,reinterpret_cast<void**>(&mapped[i])));}
D4rVkStaging staging;assert(staging.init(device,vkGetDeviceProcAddr));
VkCommandPoolCreateInfo pi={VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};pi.queueFamilyIndex=family;VkCommandPool pool;check(vkCreateCommandPool(device,&pi,nullptr,&pool));
VkCommandBufferAllocateInfo ca={VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};ca.commandPool=pool;ca.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY;ca.commandBufferCount=1;VkCommandBuffer cmd;check(vkAllocateCommandBuffers(device,&ca,&cmd));
auto begin=[&](){check(vkResetCommandPool(device,pool,0));VkCommandBufferBeginInfo b={VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};b.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;check(vkBeginCommandBuffer(cmd,&b));};
auto submit=[&](){VkMemoryBarrier host={VK_STRUCTURE_TYPE_MEMORY_BARRIER};host.srcAccessMask=VK_ACCESS_SHADER_WRITE_BIT;host.dstAccessMask=VK_ACCESS_HOST_READ_BIT;vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,VK_PIPELINE_STAGE_HOST_BIT,0,1,&host,0,nullptr,0,nullptr);check(vkEndCommandBuffer(cmd));VkSubmitInfo s={VK_STRUCTURE_TYPE_SUBMIT_INFO};s.commandBufferCount=1;s.pCommandBuffers=&cmd;check(vkQueueSubmit(queue,1,&s,VK_NULL_HANDLE));check(vkQueueWaitIdle(queue));};
auto barrier=[&](const Image& i,VkImageLayout from,VkImageLayout to,VkAccessFlags src,VkAccessFlags dst){
VkImageMemoryBarrier b={VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};b.image=i.image;b.oldLayout=from;b.newLayout=to;b.srcAccessMask=src;b.dstAccessMask=dst;b.srcQueueFamilyIndex=b.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED;
b.subresourceRange={i.depth ? VK_IMAGE_ASPECT_DEPTH_BIT|VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1};vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,0,0,nullptr,0,nullptr,1,&b);
};
unsigned tests=0;
for(auto format:{VK_FORMAT_R16G16B16A16_SFLOAT,VK_FORMAT_B10G11R11_UFLOAT_PACK32,VK_FORMAT_R16G16_SFLOAT,VK_FORMAT_R16_SFLOAT,VK_FORMAT_R32_SFLOAT,VK_FORMAT_D32_SFLOAT_S8_UINT}) {
auto source=image(format,false);D4rVkStagingSet read;assert(staging.set(read,source.view,format,buffers[0],BYTES,false));
std::memset(mapped[0],0xa5,BYTES);begin();barrier(source,VK_IMAGE_LAYOUT_UNDEFINED,VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,0,VK_ACCESS_TRANSFER_WRITE_BIT);
VkImageSubresourceRange range={source.depth ? VK_IMAGE_ASPECT_DEPTH_BIT|VK_IMAGE_ASPECT_STENCIL_BIT : VK_IMAGE_ASPECT_COLOR_BIT,0,1,0,1};
if(source.depth){VkClearDepthStencilValue c={0.25f,0xab};vkCmdClearDepthStencilImage(cmd,source.image,VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,&c,1,&range);}
else{VkClearColorValue c={{0.25f,0.5f,0.75f,1.f}};vkCmdClearColorImage(cmd,source.image,VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,&c,1,&range);}
barrier(source,VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,VK_ACCESS_TRANSFER_WRITE_BIT,VK_ACCESS_SHADER_READ_BIT);
staging.record(cmd,read,W,H);submit();
const bool two=format==VK_FORMAT_R16G16_SFLOAT, scalar=source.depth||format==VK_FORMAT_R32_SFLOAT||format==VK_FORMAT_R16_SFLOAT;
for(unsigned p=0;p<W*H;++p){assert(mapped[0][p*4]==0.25f);assert(mapped[0][p*4+1]==(scalar?0.f:0.5f));assert(mapped[0][p*4+2]==(scalar||two?0.f:0.75f));assert(mapped[0][p*4+3]==1.f);}
for(unsigned mode=1;mode<=4;++mode) {
std::memset(mapped[0],0xa5,BYTES);begin();staging.record(cmd,read,W,H,mode);submit();
auto* words=reinterpret_cast<uint32_t*>(mapped[0]);
const unsigned stride=mode==1?2:1;
for(unsigned p=0;p<W*H;++p) {
const uint32_t expected=mode==1||mode==3 ? 0x3400u | ((scalar?0u:0x3800u)<<16) : 0x3e800000u;
assert(words[p*stride]==expected);
if(mode==1)assert(words[p*stride+1]==(scalar||two?0u:0x3a00u) + (0x3c00u<<16));
}
for(unsigned j=W*H*stride;j<BYTES/4;++j)assert(words[j]==0xa5a5a5a5u);
}
std::puts("canonical RGBA16F/R32F/RG16F input modes: values and bounds correct");
std::printf("sample input format %d: 35 correct pixels; no TRANSFER_SRC\n",format);++tests;
check(vkResetCommandPool(device,pool,0));staging.free(read);destroyImage(source);
}
for(auto format:{VK_FORMAT_R16G16B16A16_SFLOAT,VK_FORMAT_R32G32B32A32_SFLOAT,VK_FORMAT_B10G11R11_UFLOAT_PACK32,VK_FORMAT_A2B10G10R10_UNORM_PACK32,VK_FORMAT_R8G8B8A8_UNORM}) {
auto output=image(format,true);D4rVkStagingSet write,verify;assert(staging.set(write,output.view,format,buffers[0],W*H*8,true));assert(staging.set(verify,output.view,format,buffers[1],BYTES,false,VK_IMAGE_LAYOUT_GENERAL));
auto* packed=reinterpret_cast<uint16_t*>(mapped[0]);
const uint16_t half[]={0x3400,0x3800,0x3a00,0x3c00};
const float expected[]={0.25f,0.5f,0.75f,1.f};
for(unsigned p=0;p<W*H;++p)for(unsigned c=0;c<4;++c)packed[p*4+c]=half[c];
std::memset(mapped[1],0xa5,BYTES);begin();barrier(output,VK_IMAGE_LAYOUT_UNDEFINED,VK_IMAGE_LAYOUT_GENERAL,0,VK_ACCESS_SHADER_WRITE_BIT);
staging.record(cmd,write,W,H);barrier(output,VK_IMAGE_LAYOUT_GENERAL,VK_IMAGE_LAYOUT_GENERAL,VK_ACCESS_SHADER_WRITE_BIT,VK_ACCESS_SHADER_READ_BIT);staging.record(cmd,verify,W,H);submit();
const float epsilon=format==VK_FORMAT_R8G8B8A8_UNORM ? 1.f/255 : format==VK_FORMAT_A2B10G10R10_UNORM_PACK32 ? 1.f/1023 : 0.00001f;
for(unsigned p=0;p<W*H;++p)for(unsigned c=0;c<4;++c)assert(std::isfinite(mapped[1][p*4+c]) && std::abs(mapped[1][p*4+c]-expected[c])<=epsilon);
std::printf("storage output format %d: 35 correct pixels; no TRANSFER_DST\n",format);++tests;
if(format==VK_FORMAT_R32G32B32A32_SFLOAT) {
unsigned verified=0;
for(unsigned base=0;base<65536;base+=W*H*4) {
for(unsigned j=0;j<W*H*4;++j)packed[j]=static_cast<uint16_t>(base+j);
begin();barrier(output,VK_IMAGE_LAYOUT_GENERAL,VK_IMAGE_LAYOUT_GENERAL,VK_ACCESS_SHADER_READ_BIT,VK_ACCESS_SHADER_WRITE_BIT);
staging.record(cmd,write,W,H);barrier(output,VK_IMAGE_LAYOUT_GENERAL,VK_IMAGE_LAYOUT_GENERAL,VK_ACCESS_SHADER_WRITE_BIT,VK_ACCESS_SHADER_READ_BIT);
staging.record(cmd,verify,W,H);submit();
for(unsigned j=0;j<W*H*4 && base+j<65536;++j) {
const uint16_t h=packed[j];const unsigned e=(h>>10)&31, m=h&1023;
if(e==31)continue;
float value=e ? std::ldexp(1.f+static_cast<float>(m)/1024.f,static_cast<int>(e)-15) : std::ldexp(static_cast<float>(m),-24);
if(h&0x8000)value=-value;
uint32_t wanted,actual;std::memcpy(&wanted,&value,4);std::memcpy(&actual,&mapped[1][j],4);
assert(actual==wanted);++verified;
}
// Repack the RGBA32F sampled result using the new canonical
// input path. Every finite half must round-trip bit exactly.
begin();staging.record(cmd,verify,W,H,1);submit();
const auto* encoded=reinterpret_cast<const uint16_t*>(mapped[1]);
for(unsigned j=0;j<W*H*4 && base+j<65536;++j)
if(((packed[j]>>10)&31)!=31)assert(encoded[j]==packed[j]);
}
assert(verified==63488);std::printf("PASS: all %u finite half encodings and input round trips, including signed zeros and subnormals\n",verified);
}
check(vkResetCommandPool(device,pool,0));staging.free(write);staging.free(verify);destroyImage(output);
}
staging.destroy();vkDestroyCommandPool(device,pool,nullptr);
for(int i=0;i<2;++i){vkUnmapMemory(device,allocations[i]);vkDestroyBuffer(device,buffers[i],nullptr);vkFreeMemory(device,allocations[i],nullptr);}
vkDestroyDevice(device,nullptr);vkDestroyInstance(instance,nullptr);std::printf("PASS: %u GPU format checks\n",tests);
}
tools/wine_nvcuda_bridge.c
+64
View File
@@ -2100,6 +2100,38 @@ static void* bridge_export(const char* name)
return module != NULL ? (void*)GetProcAddress(module, name) : NULL;
}
/* Process exit. NVIDIA's NGX DLL unloads its CUDA modules from its own detach, which runs before the
NGX shim's (it was loaded later). A Vulkan game can exit with a command buffer still waiting inside
the GPU ring for the shim's output event; that teardown then blocks on the stuck job until the ring
times out and resets. The shim registers a hook releasing those waits; it runs once, before the
first teardown call made while Wine is shutting the process down. */
static void (WINAPI* shutdown_hook)(void);
void WINAPI d4rSetShutdownHook(void (WINAPI* hook)(void))
{
shutdown_hook = hook;
}
/* Returns 1 while Wine shuts the process down. ExitProcess has killed the other threads by then,
possibly inside HIP/HSA with a runtime lock held, so a teardown call into HIP can block forever
(seen: NVIDIA's NGX Shutdown1 -> cuTexObjectDestroy -> HSA mutex). Teardown and sync calls then
report success without touching HIP; the kernel reclaims the process's GPU state. */
static int before_teardown(void)
{
static BOOLEAN (WINAPI* in_progress)(void);
static LONG ran;
if (in_progress == NULL)
in_progress = (BOOLEAN (WINAPI*)(void))GetProcAddress(GetModuleHandleA("ntdll.dll"), "RtlDllShutdownInProgress");
if (in_progress == NULL || !in_progress())
return 0;
if (shutdown_hook != NULL && InterlockedExchange(&ran, 1) == 0)
{
tracef("process shutdown: running the shim's hook; CUDA teardown is skipped");
shutdown_hook();
}
return 1;
}
CUresult WINAPI cuInit(unsigned int flags)
{
CUINIT_FN function = (CUINIT_FN)find_zluda_symbol("cuInit");
@@ -2162,6 +2194,8 @@ CUresult WINAPI cuCtxCreate_v2(CUcontext* context, unsigned int flags, CUdevice
CUresult WINAPI cuCtxDestroy_v2(CUcontext context)
{
if (before_teardown())
return CUDA_SUCCESS;
CUCTXDESTROY_FN function = (CUCTXDESTROY_FN)find_zluda_symbol("cuCtxDestroy_v2");
CUresult result = function != NULL ? function(context) : CUDA_ERROR_NOT_INITIALIZED;
tracef("cuCtxDestroy_v2 context=%p result=%d", context, result);
@@ -2192,6 +2226,8 @@ static int elide_ngx_sync(void)
CUresult WINAPI d4rCtxSynchronize(void)
{
if (before_teardown())
return CUDA_SUCCESS;
CUCTX_SYNCHRONIZE_FN function = (CUCTX_SYNCHRONIZE_FN)find_zluda_symbol("cuCtxSynchronize");
CUresult result = function != NULL ? function() : missing("cuCtxSynchronize");
if (result == CUDA_SUCCESS)
@@ -2202,6 +2238,8 @@ CUresult WINAPI d4rCtxSynchronize(void)
CUresult WINAPI cuCtxSynchronize(void)
{
if (before_teardown())
return CUDA_SUCCESS;
if (elide_ngx_sync())
{
TRACE_CALL(CUDA_SUCCESS, "cuCtxSynchronize elided");
@@ -2348,6 +2386,8 @@ CUresult WINAPI cuModuleLoad(CUmodule* module, const char* filename)
CUresult WINAPI cuModuleUnload(CUmodule module)
{
if (before_teardown())
return CUDA_SUCCESS;
CUMODULEUNLOAD_FN function = (CUMODULEUNLOAD_FN)find_zluda_symbol("cuModuleUnload");
CUresult result = function != NULL ? function(module) : missing("cuModuleUnload");
if (result == CUDA_SUCCESS) microcuda_k_remove_module(module);
@@ -2474,6 +2514,8 @@ CUresult WINAPI cuMemAllocHost(void** pointer, size_t bytes)
CUresult WINAPI cuMemFree(CUdeviceptr pointer)
{
if (before_teardown())
return CUDA_SUCCESS;
CUMEMFREE_FN function = (CUMEMFREE_FN)find_zluda_symbol("cuMemFree_v2");
if (microcuda_k_forget) microcuda_k_forget(pointer);
CUresult result = function != NULL ? function(pointer) : missing("cuMemFree_v2");
@@ -2485,6 +2527,8 @@ CUresult WINAPI cuMemFree(CUdeviceptr pointer)
CUresult WINAPI cuMemFreeHost(void* pointer)
{
if (before_teardown())
return CUDA_SUCCESS;
CUMEMFREEHOST_FN function = (CUMEMFREEHOST_FN)find_zluda_symbol("cuMemFreeHost");
CUresult result = function != NULL ? function(pointer) : missing("cuMemFreeHost");
TRACE_CALL(result, "cuMemFreeHost ptr=%p result=%d", pointer, result);
@@ -2572,6 +2616,8 @@ CUresult WINAPI cuArray3DCreate(CUarray* array, const void* descriptor)
CUresult WINAPI cuArrayDestroy(CUarray array)
{
if (before_teardown())
return CUDA_SUCCESS;
CUARRAYDESTROY_FN function = (CUARRAYDESTROY_FN)find_zluda_symbol("cuArrayDestroy");
CUresult result = function != NULL ? function(array) : missing("cuArrayDestroy");
if (result == CUDA_SUCCESS)
@@ -2690,6 +2736,8 @@ CUresult WINAPI cuArrayGetDescriptor(void* descriptor, CUarray array)
CUresult WINAPI cuMipmappedArrayDestroy(CUmipmappedArray array)
{
if (before_teardown())
return CUDA_SUCCESS;
CUMIPMAPPEDARRAYDESTROY_FN function =
(CUMIPMAPPEDARRAYDESTROY_FN)find_zluda_symbol("cuMipmappedArrayDestroy");
CUresult result = function != NULL ? function(array) : missing("cuMipmappedArrayDestroy");
@@ -2699,6 +2747,8 @@ CUresult WINAPI cuMipmappedArrayDestroy(CUmipmappedArray array)
CUresult WINAPI cuDestroyExternalMemory(CUexternalMemory memory)
{
if (before_teardown())
return CUDA_SUCCESS;
CUEXTERNALMEMORYDESTROY_FN function =
(CUEXTERNALMEMORYDESTROY_FN)find_zluda_symbol("cuDestroyExternalMemory");
CUresult result = function != NULL ? function(memory) : missing("cuDestroyExternalMemory");
@@ -2731,6 +2781,8 @@ CUresult WINAPI cuSurfObjectCreate(CUsurfObject* object, const void* descriptor)
CUresult WINAPI cuSurfObjectDestroy(CUsurfObject object)
{
if (before_teardown())
return CUDA_SUCCESS;
CUSURFOBJECTDESTROY_FN function = (CUSURFOBJECTDESTROY_FN)find_zluda_symbol("cuSurfObjectDestroy");
pthread_mutex_lock(&instrumentation_lock);
for (SurfaceObjectRecord** link = &surface_objects; *link != NULL; link = &(*link)->next)
@@ -2798,6 +2850,8 @@ CUresult WINAPI cuTexObjectCreate(CUtexObject* object, const void* resource,
CUresult WINAPI cuTexObjectDestroy(CUtexObject object)
{
if (before_teardown())
return CUDA_SUCCESS;
CUTEXOBJECTDESTROY_FN function = (CUTEXOBJECTDESTROY_FN)find_zluda_symbol("cuTexObjectDestroy");
CUresult result = function != NULL ? function(object) : missing("cuTexObjectDestroy");
if (result == CUDA_SUCCESS)
@@ -2840,9 +2894,15 @@ static void microcuda_k_notify_generic_write(const char* target)
microcuda_k_write(0, SIZE_MAX); /* unsupported/general writer: invalidate all prepared weights */
}
typedef uintptr_t D4rArg;
static int d4r_forward_is_teardown(const char* target)
{
return strstr(target, "Destroy") != NULL || strstr(target, "Synchronize") != NULL || strstr(target, "Release") != NULL;
}
#define D4R_FORWARD(name, target, count, params, args) \
CUresult WINAPI name params \
{ \
if (d4r_forward_is_teardown(#target) && before_teardown()) \
return CUDA_SUCCESS; \
typedef CUresult(__attribute__((sysv_abi)) * function_type) params; \
function_type function = (function_type)find_zluda_symbol(#target); \
microcuda_k_notify_generic_write(#target); \
@@ -2857,6 +2917,8 @@ D4R_FORWARD(cuEventRecord, cuEventRecord, 2, (D4rArg a0, D4rArg a1), (a0, a1))
D4R_FORWARD(cuEventRecordWithFlags, cuEventRecordWithFlags, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
CUresult WINAPI cuEventSynchronize(D4rArg a0)
{
if (before_teardown())
return CUDA_SUCCESS;
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
if (elide_ngx_sync())
{
@@ -2872,6 +2934,8 @@ CUresult WINAPI cuEventSynchronize(D4rArg a0)
game's split command list only after this event has actually completed. */
CUresult WINAPI d4rEventSynchronize(D4rArg a0)
{
if (before_teardown())
return CUDA_SUCCESS;
typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
function_type function = (function_type)find_zluda_symbol("cuEventSynchronize");
CUresult result = function != NULL ? function(a0) : missing("cuEventSynchronize");
tools/wine_nvcuda_bridge.spec
+1
View File
@@ -92,6 +92,7 @@
@ stdcall d4rReleaseVulkanMemory(ptr) d4rReleaseVulkanMemory
@ stdcall d4rMemcpy2DAsync(ptr ptr) d4rMemcpy2DAsync
@ stdcall d4rCtxSynchronize() d4rCtxSynchronize
@ stdcall d4rSetShutdownHook(ptr) d4rSetShutdownHook
@ stdcall d4rEventSynchronize(ptr) d4rEventSynchronize
@ stdcall d4rStreamWaitValue32(int64 long) d4rStreamWaitValue32
@ stdcall d4rWriteValue32(int64 long) d4rWriteValue32