wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ac4ee6c225688f3f373e1d3fd27489194f9a9563
d4r/tools/windows/ngx_windows_runtime.h
T
ayo ac4ee6c225 Expand Windows GPU support, add native FSR RR, and improve SR interop 
Add a native D3D12 FSR Ray Regeneration route for gfx1200/gfx1201,
using the game's device and queue with embedded HLSL guide preparation
and output composition. Preserve the vkd3d compatibility route for
RDNA3 and expose explicit route overrides and diagnostics.

Improve Windows runtime compatibility:
- Retry rejected application-ID initialization with a project identity.
- Defer expensive RR backend creation until feature creation.
- Recover from transient submission failures without disabling later frames.
- Add shared-buffer export fallbacks through explicit heaps and a private
  device on the same adapter.
- Safely evict old RR descriptor sets when input views change.
- Treat requested RR presets as hints when a configured model is available.
- Expand SR/RR input formats and handle diffuse/specular hit distances,
  packed normals, depth-stencil resources, and unused FSR guides.

Reduce SR overhead by batching output downloads and format conversion
under one completion wait. Optimize strict M enc2/tube attention with
bounded FP32 accumulation and the original FP64 fallback for ambiguous
rounding cases.

Extend RDNA4 tooling with gfx1200/gfx1201 native FP8 RR model builds,
capability checks, architecture-aware deployment, and experimental K/M
kernel paths. Require matching model and runtime arithmetic families.

Add optional embedded FSR shader optimizations keyed by shader hash and
exact original SPIR-V contents. Preserve reflected metadata and record
optimizer provenance, artifact hashes, and cache identity.

Simplify Windows installation with automatic backend dependency downloads,
download progress and ETA, backed-up absolute OptiScaler library paths,
and recovery from interrupted RR model builds.

Add build-kernels.bat and a kernels-only build matrix for all eleven
registered RDNA targets, with code-object target validation and SHA256
manifests. Expand conversion probes, numerical stress tests, temporal
benchmark tooling, installer/deployment tests, and setup documentation.

Document synthetic regression and timing results, experimental build
options, and remaining RDNA4 hardware validation requirements.
2026-10-17 07:40:34 -04:00

726 lines
45 KiB
C++
Raw Blame History

#pragma once
#include "d3d12_external.h"
#include "d3d12_boundary_timing.h"
#include "pixel_conversion.h"
#include "cuda_image_api.h"
#include "ngx_parameters.h"
#include "ngx_public_abi.h"
#include "d3d12_command_hooks.h"
#include "d3d12_retirement.h"
#include <condition_variable>
#include <deque>
#include <thread>
#include <chrono>
#include <atomic>
#include <memory>
#include <mutex>
#include <array>
namespace d4r::win {
inline std::string env_path(const char* name) {
const char* value = std::getenv(name);
if (!value || !*value) throw std::runtime_error(std::string("Set the absolute path in ") + name);
if (!std::filesystem::path(diag::wide(value)).is_absolute()) throw std::runtime_error(std::string(name) + " must be absolute");
return value;
}
inline void ngx_check(unsigned result, const char* operation) {
if (result != 1 || !std::getenv("D4R_QUIET_API")) std::printf("D4R_NGX %s result=0x%08x\n", operation, result);
if (result != 1) throw std::runtime_error(std::string(operation) + " NGX=" + std::to_string(result));
}
inline void ngx_log_callback(const char* message, int level, unsigned component) {
if (message) std::printf("NGX_LOG level=%d component=%u %s\n", level, component, message);
}
inline ResourceAccess ngx_resource_access(void* parameters, const char* name, D3D12_RESOURCE_STATES fallback) {
ResourceAccess result(D3D12_RESOURCE_STATES(ngx::uint_value(parameters, name, unsigned(fallback))));
const std::string prefix(name);
result.enhanced = ngx::uint_value(parameters, (prefix + ".Enhanced").c_str()) != 0;
if (result.enhanced) {
result.layout = D3D12_BARRIER_LAYOUT(ngx::uint_value(parameters, (prefix + ".Layout").c_str()));
result.access = D3D12_BARRIER_ACCESS(ngx::uint_value(parameters, (prefix + ".Access").c_str()));
}
return result;
}
inline std::filesystem::path pixel_module_path() {
if (const char* path = std::getenv("D4R_FORMAT_MODULE")) return diag::wide(env_path("D4R_FORMAT_MODULE"));
HMODULE module = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
reinterpret_cast<LPCWSTR>(&pixel_module_path), &module)) throw std::runtime_error("Locate GPU conversion module directory");
wchar_t path[32768]{}; const DWORD length = GetModuleFileNameW(module, path, 32768);
if (!length || length == 32768) throw std::runtime_error("GPU conversion module path is invalid");
return std::filesystem::path(path).parent_path() / diag::wide(std::string("pixel_convert_") + diag::HipApi::target_arch() + ".hsaco");
}
struct Runtime {
diag::HipApi hip;
CudaApi cuda;
std::unique_ptr<diag::Library> nvapi;
diag::SearchDirectory featureSearch;
std::unique_ptr<diag::Library> dlss, core;
ExternalApi external;
cuda::ImageApi images;
ComPtr<ID3D12Device> device;
CUdevice ordinal = -1;
CUcontext context = nullptr;
bool initialized = false;
std::unique_ptr<SharedTimeline> timeline;
std::unique_ptr<PixelProgram> pixels;
std::mutex mutex;
// Features share one CUDA context. Follow submission order even when
// separate feature workers wake in a different order; otherwise a worker
// could hold the context while waiting for an earlier worker's output.
std::mutex async_mutex;
std::condition_variable async_changed;
std::deque<uint64_t> async_tickets;
std::string async_failure;
ComPtr<ID3D12CommandQueue> async_queue;
ngx::ProjectIdentity project;
ngx::FeatureCommonInfo common{};
std::vector<std::wstring> featurePaths;
std::vector<const wchar_t*> featurePathPointers;
using Init = unsigned(*)(unsigned long long, const wchar_t*, const ngx::FeatureCommonInfo*, unsigned);
using Shutdown = unsigned(*)();
using Allocate = unsigned(*)(void**);
using Destroy = unsigned(*)(void*);
using Create = unsigned(*)(unsigned, void*, void**);
using Evaluate = unsigned(*)(void*, void*, void*);
using Release = unsigned(*)(void*);
using ScratchSize = unsigned(*)(unsigned, void*, size_t*);
Allocate allocate = nullptr, capabilities = nullptr;
Destroy destroy = nullptr;
Create create = nullptr;
Evaluate evaluate = nullptr;
Release release = nullptr;
ScratchSize getScratchBufferSize = nullptr;
Shutdown shutdown = nullptr;
Runtime(ID3D12Device* d3d, unsigned long long app, const wchar_t* data, unsigned sdk,
const ngx::FeatureCommonInfo* info = nullptr, const ngx::ProjectIdentity* identity = nullptr)
: hip(env_path("D4R_HIP_ROOT")), cuda(env_path("D4R_NVCUDA_DLL")),
featureSearch(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path()),
external{hip}, images(cuda), device(d3d) {
if (!d3d) throw std::runtime_error("Null D3D12 device");
try {
hipDeviceProp_t props{};
const auto luid = adapter_luid(d3d);
hip.select_architecture(-1, props, &luid);
hip.verbose = cuda.verbose = !std::getenv("D4R_QUIET_API");
if (std::memcmp(&luid, props.luid, sizeof(luid))) throw std::runtime_error("D3D12/HIP LUID mismatch");
cuda.check(cuda.cuInit(0), "cuInit(Windows runtime)");
int count = 0; cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
for (int i = 0; i < count; ++i) {
CUdevice candidate = -1; cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
int bus = -1, pciDevice = -1, domain = -1;
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
cuda.check(cuda.cuDeviceGetAttribute(&pciDevice, 34, candidate), "PCI device");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
if (bus == props.pciBusID && pciDevice == props.pciDeviceID && domain == props.pciDomainID) ordinal = candidate;
}
if (ordinal < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain");
current();
timeline = std::make_unique<SharedTimeline>(external, d3d);
std::printf("D4R_INTEROP async=%u batch_inputs=%u cpu_image_copies=0\n",
unsigned(std::getenv("D4R_ASYNC_INTEROP") != nullptr), unsigned(std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr));
// NGX's loader can probe CUDA during DLL initialization: establish
// the correct primary context before loading either NVIDIA DLL.
nvapi = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NVAPI_DLL")));
dlss = std::make_unique<diag::Library>(diag::wide(env_path("D4R_DLSS_DLL")));
core = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NGX_CORE")));
allocate = core->symbol<Allocate>("NVSDK_NGX_CUDA_AllocateParameters");
capabilities = core->symbol<Allocate>("NVSDK_NGX_CUDA_GetCapabilityParameters");
destroy = core->symbol<Destroy>("NVSDK_NGX_CUDA_DestroyParameters");
create = core->symbol<Create>("NVSDK_NGX_CUDA_CreateFeature");
evaluate = core->symbol<Evaluate>("NVSDK_NGX_CUDA_EvaluateFeature");
release = core->symbol<Release>("NVSDK_NGX_CUDA_ReleaseFeature");
getScratchBufferSize = core->symbol<ScratchSize>("NVSDK_NGX_CUDA_GetScratchBufferSize");
shutdown = core->symbol<Shutdown>("NVSDK_NGX_CUDA_Shutdown");
const auto srDirectory = std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path();
if (std::filesystem::exists(srDirectory / L"nvngx_dlssd.dll"))
throw std::runtime_error("DLSS SR and Ray Reconstruction DLLs share a feature search directory; keep nvngx_dlssd.dll under d4r\\rr");
featurePaths.emplace_back(srDirectory.wstring());
if (info) {
if (info->PathListInfo.Length > 1024 || (info->PathListInfo.Length && !info->PathListInfo.Path))
throw std::runtime_error("Invalid NGX feature path list");
for (unsigned i = 0; i < info->PathListInfo.Length; ++i) {
if (!info->PathListInfo.Path[i]) continue;
const std::filesystem::path candidate(info->PathListInfo.Path[i]);
if (candidate.lexically_normal() != srDirectory.lexically_normal()) {
std::printf("D4R_NGX_PATH skip=%ls reason=outside-pinned-SR-directory\n", candidate.c_str());
continue;
}
std::printf("D4R_NGX_PATH duplicate=%ls reason=pinned-SR-directory\n", candidate.c_str());
}
}
for (const auto& path : featurePaths) featurePathPointers.push_back(path.c_str());
common.PathListInfo = {featurePathPointers.data(), unsigned(featurePathPointers.size())};
common.LoggingInfo = {reinterpret_cast<void*>(&ngx_log_callback), 1, false};
if (info && sdk >= 0x14) common.LoggingInfo = info->LoggingInfo;
std::filesystem::path applicationData = data && *data ? std::filesystem::path(data) : std::filesystem::current_path();
if (const char* redirected = std::getenv("D4R_NGX_DATA_DIR"); redirected && *redirected) {
applicationData = diag::wide(env_path("D4R_NGX_DATA_DIR"));
std::filesystem::create_directories(applicationData);
std::printf("D4R_NGX_DATA_PATH redirect=%ls\n", applicationData.c_str());
}
if (!applicationData.is_absolute()) throw std::runtime_error("NGX application data path must be absolute");
if (identity) {
if (identity->id.empty()) throw std::runtime_error("Empty NGX project identity");
project = *identity;
using ProjectInit = unsigned(*)(const char*, int, const char*, const wchar_t*, unsigned, const ngx::FeatureCommonInfo*);
std::printf("D4R_IDENTITY project=%s engine=%d version=%s sdk=0x%x\n", project.id.c_str(),project.engine,project.version.c_str(),sdk);
ngx_check(core->symbol<ProjectInit>("NVSDK_NGX_CUDA_Init_ProjectID")(project.id.c_str(),project.engine,
project.version.c_str(),applicationData.c_str(),sdk,&common), "CUDA Init_ProjectID");
} else {
std::printf("D4R_IDENTITY application=%llu sdk=0x%x\n", app,sdk);
ngx_check(core->symbol<Init>("NVSDK_NGX_CUDA_Init")(app, applicationData.c_str(), &common, sdk), "CUDA Init(application ABI)");
}
initialized = true;
void* caps = nullptr;
ngx_check(capabilities(&caps), "CUDA feature capabilities");
if (!caps) throw std::runtime_error("Null CUDA capability parameters");
const int available = ngx::int_value(caps, "SuperSampling.Available", -1);
const int initResult = ngx::int_value(caps, "SuperSampling.FeatureInitResult", -1);
const int needsDriver = ngx::int_value(caps, "SuperSampling.NeedsUpdatedDriver", -1);
std::printf("D4R_CAPABILITIES available=%d feature_init=0x%08x needs_driver=%d\n", available, unsigned(initResult), needsDriver);
std::fprintf(stderr, "d4r: NGX SR capabilities available=%d feature_init=0x%08x needs_driver=%d\n",
available, unsigned(initResult), needsDriver);
ngx_check(destroy(caps), "Destroy capability parameters");
if (available != 1 || initResult != 1 || needsDriver != 0)
throw std::runtime_error("CUDA DLSS is unavailable after capability initialization");
std::printf("D4R_RUNTIME platform=Windows architecture=%s cpu_frame_copies=0 frame_age=0\n", d4r::diag::HipApi::target_arch());
} catch (...) { cleanup(); throw; }
}
Runtime(const Runtime&) = delete;
~Runtime() { cleanup(); }
void current() { cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent(runtime)"); }
void drain_async_queue() {
ComPtr<ID3D12CommandQueue> queue;
{ std::lock_guard<std::mutex> lock(async_mutex); queue = async_queue; }
if (!queue) return;
// Creation/free/import may synchronize the AMD device internally.
// Drain queued D3D12 consumers before taking the CUDA mutex: workers
// must remain able to satisfy any already-published output waits.
ComPtr<ID3D12Fence> completion;
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(completion.GetAddressOf())), "Reconfigure fence");
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.value) throw std::runtime_error("Reconfigure event");
dx(queue->Signal(completion.Get(), 1), "Reconfigure queue Signal");
if (completion->GetCompletedValue() < 1) {
dx(completion->SetEventOnCompletion(1, event.value), "Reconfigure completion event");
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Reconfigure GPU timeout");
}
dx(device->GetDeviceRemovedReason(), "Reconfigure device status");
}
void cleanup() noexcept {
if (context) (void)cuda.cuCtxSetCurrent(context);
if (initialized) { (void)shutdown(); initialized = false; }
timeline.reset();
pixels.reset();
if (context) {
(void)cuda.cuCtxSynchronize(); (void)cuda.cuCtxSetCurrent(nullptr);
(void)cuda.cuDevicePrimaryCtxRelease_v2(ordinal); context = nullptr;
}
}
};
class Feature {
std::shared_ptr<Runtime> rt_;
void* parameters_ = nullptr;
void* handle_ = nullptr;
CUdeviceptr scratch_ = 0;
ComPtr<ID3D12CommandQueue> queue_;
// A boundary drains both copies before returning. Reuse their allocators
// only after that completion; no game allocator is retained here.
ComPtr<ID3D12CommandAllocator> input_allocator_, output_allocator_;
ComPtr<ID3D12GraphicsCommandList> input_list_, output_list_;
struct CopySlot {
ComPtr<ID3D12CommandAllocator> input_allocator, output_allocator;
ComPtr<ID3D12GraphicsCommandList> input, output;
std::unique_ptr<BoundaryTiming> timing;
uint64_t done = 0;
};
struct AsyncFrame {
void* parameters = d4r_ngx_parameters_create();
std::vector<ComPtr<ID3D12Resource>> resources;
SharedTimeline::Ticket ticket{};
~AsyncFrame() { if (parameters) d4r_ngx_parameters_destroy(parameters); }
};
std::vector<std::unique_ptr<CopySlot>> copy_slots_;
ComPtr<ID3D12Fence> copy_completion_;
Handle copy_event_;
uint64_t copy_value_ = 0;
std::mutex jobs_mutex_;
std::mutex submit_mutex_;
std::condition_variable jobs_changed_;
std::deque<std::shared_ptr<AsyncFrame>> jobs_;
std::thread worker_;
bool worker_stop_ = false, worker_active_ = false;
std::string worker_failure_;
struct Plane {
ComPtr<ID3D12Resource> texture;
D3D12_RESOURCE_DESC desc{};
std::unique_ptr<SharedPlane> shared;
std::unique_ptr<cuda::Image> image;
std::unique_ptr<PixelAllocation> canonical;
PixelSpec spec{};
} planes_[5];
void cleanup() noexcept {
(void)rt_->cuda.cuCtxSetCurrent(rt_->context);
(void)rt_->cuda.cuCtxSynchronize();
if (handle_) { (void)rt_->release(handle_); handle_ = nullptr; }
for (auto& plane : planes_) {
plane.image.reset(); plane.canonical.reset(); plane.shared.reset(); plane.texture.Reset();
}
if (scratch_) { (void)rt_->cuda.cuMemFree_v2(scratch_); scratch_ = 0; }
if (parameters_) { (void)rt_->destroy(parameters_); parameters_ = nullptr; }
}
public:
unsigned width = 0, height = 0;
explicit Feature(std::shared_ptr<Runtime> runtime, void* parameters) : rt_(std::move(runtime)) {
if (std::getenv("D4R_ASYNC_INTEROP")) rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current();
try {
ngx_check(rt_->allocate(&parameters_), "CUDA AllocateParameters");
ngx::copy_create(parameters, parameters_);
d4r_ngx_set_int(parameters_, "DLSS.Enable.Output.Subrects", ngx::int_value(parameters, "DLSS.Enable.Output.Subrects"));
width = ngx::uint_value(parameters, "Width"); height = ngx::uint_value(parameters, "Height");
std::printf("D4R_CREATE source=%ux%u->%ux%u cuda=%ux%u->%ux%u\n", width, height,
ngx::uint_value(parameters, "OutWidth"), ngx::uint_value(parameters, "OutHeight"),
ngx::uint_value(parameters_, "Width"), ngx::uint_value(parameters_, "Height"),
ngx::uint_value(parameters_, "OutWidth"), ngx::uint_value(parameters_, "OutHeight"));
if (!width || !height) throw std::runtime_error("Zero NGX input dimensions");
size_t scratchBytes = 0;
const unsigned scratchResult = rt_->getScratchBufferSize(1, parameters_, &scratchBytes);
if (scratchResult != 1 || !scratchBytes) {
std::printf("D4R_CREATE scratch_query=0x%08x fallback_bytes=67108864\n", scratchResult);
scratchBytes = 64ull * 1024 * 1024;
}
std::printf("D4R_CREATE scratch_bytes=%zu\n", scratchBytes);
rt_->cuda.check(rt_->cuda.cuMemAlloc_v2(&scratch_, scratchBytes), "NGX VRAM scratch");
d4r_ngx_set_void(parameters_, "Scratch", reinterpret_cast<void*>(uintptr_t(scratch_)));
d4r_ngx_set_ull(parameters_, "Scratch.SizeInBytes", scratchBytes);
ngx_check(rt_->create(1, parameters_, &handle_), "CUDA CreateFeature");
} catch (...) { cleanup(); throw; }
}
~Feature() {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_stop_ = true; }
jobs_changed_.notify_all();
if (worker_.joinable()) worker_.join();
if (copy_completion_) {
try { wait_copies(copy_value_); report_completed_copies(); }
catch (const std::exception& e) { std::fprintf(stderr, "D4R_ASYNC_CLEANUP_FAILURE %s\n", e.what()); }
}
std::lock_guard<std::mutex> lock(rt_->mutex); cleanup();
}
Feature(const Feature&) = delete;
// Query the loaded SR library, using NVIDIA's optimal-settings callback contract.
// An unavailable/invalid query keeps the conservative creation-size-only range.
std::array<unsigned, 4> render_limits() {
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current();
std::array<unsigned, 4> limits{width, height, width, height};
void* caps = nullptr;
if (rt_->capabilities(&caps) != 1 || !caps) return limits;
struct Guard { Runtime& rt; void* p; ~Guard() { (void)rt.destroy(p); } } guard{*rt_, caps};
void* callback = nullptr;
if (d4r_ngx_get_void(caps, "DLSSOptimalSettingsCallback", &callback) != 1 || !callback) return limits;
d4r_ngx_set_uint(caps, "Width", ngx::uint_value(parameters_, "OutWidth"));
d4r_ngx_set_uint(caps, "Height", ngx::uint_value(parameters_, "OutHeight"));
d4r_ngx_set_int(caps, "PerfQualityValue", ngx::int_value(parameters_, "PerfQualityValue"));
d4r_ngx_set_int(caps, "RTXValue", 0);
using Optimal = unsigned(*)(void*);
if (reinterpret_cast<Optimal>(callback)(caps) != 1) return limits;
const unsigned optimalWidth = ngx::uint_value(caps, "OutWidth");
const unsigned optimalHeight = ngx::uint_value(caps, "OutHeight");
const std::array<unsigned, 4> queried{
ngx::uint_value(caps, "DLSS.Get.Dynamic.Min.Render.Width", optimalWidth),
ngx::uint_value(caps, "DLSS.Get.Dynamic.Min.Render.Height", optimalHeight),
ngx::uint_value(caps, "DLSS.Get.Dynamic.Max.Render.Width", optimalWidth),
ngx::uint_value(caps, "DLSS.Get.Dynamic.Max.Render.Height", optimalHeight)};
if (queried[0] && queried[1] && queried[2] >= queried[0] && queried[3] >= queried[1]) limits = queried;
std::fprintf(stderr, "D4R_SR_RENDER_LIMITS min=%ux%u max=%ux%u\n", limits[0], limits[1], limits[2], limits[3]);
return limits;
}
void drain() {
if (!std::getenv("D4R_ASYNC_INTEROP")) return;
wait_worker();
if (copy_value_) wait_copies(copy_value_);
report_completed_copies();
rt_->drain_async_queue();
}
// Explicit harness boundary: all earlier producers must already be queued.
// The forthcoming command-list backend calls the same stages at queue submit.
void evaluate_boundary(ID3D12CommandQueue* queue, void* parameters) {
if (std::getenv("D4R_ASYNC_INTEROP")) { evaluate_async(queue, parameters); return; }
const auto boundaryBegin = std::chrono::steady_clock::now();
auto boundaryStage = boundaryBegin;
const bool profile = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profile) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - boundaryStage).count());
boundaryStage = std::chrono::steady_clock::now();
};
commands::InternalScope internal;
std::lock_guard<std::mutex> lock(rt_->mutex);
if (!queue) throw std::runtime_error("Null D3D12 queue");
if (queue_ && queue_.Get() != queue) throw std::runtime_error("A feature cannot switch queues without an explicit drain");
queue_ = queue;
ComPtr<ID3D12Device> queueDevice;
dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "GetDevice(queue)");
ComPtr<IUnknown> queueIdentity, runtimeIdentity;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(queueIdentity.GetAddressOf())), "Query queue device identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(runtimeIdentity.GetAddressOf())), "Query runtime device identity");
if (queueIdentity.Get() != runtimeIdentity.Get()) throw std::runtime_error("Queue belongs to a different D3D12 device");
rt_->current();
prepare(parameters);
mark("boundary_prepare");
ComPtr<ID3D12CommandAllocator> inputAllocator, outputAllocator;
ComPtr<ID3D12GraphicsCommandList> inputList, outputList;
auto list = [&](ComPtr<ID3D12CommandAllocator>& allocator, ComPtr<ID3D12GraphicsCommandList>& commandList) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator(interop)");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(commandList.GetAddressOf())), "CreateCommandList(interop)");
};
const bool cached = !std::getenv("D4R_DISABLE_INTEROP_LIST_CACHE");
if (cached) {
if (!input_list_) {
// Publish the pair only after both creations succeed.
list(inputAllocator, inputList); list(outputAllocator, outputList);
input_allocator_ = inputAllocator; output_allocator_ = outputAllocator;
input_list_ = inputList; output_list_ = outputList;
} else {
dx(input_allocator_->Reset(), "Reset(input interop allocator)");
dx(output_allocator_->Reset(), "Reset(output interop allocator)");
dx(input_list_->Reset(input_allocator_.Get(), nullptr), "Reset(input interop list)");
dx(output_list_->Reset(output_allocator_.Get(), nullptr), "Reset(output interop list)");
}
inputAllocator = input_allocator_; outputAllocator = output_allocator_;
inputList = input_list_; outputList = output_list_;
} else { list(inputAllocator, inputList); list(outputAllocator, outputList); }
mark("boundary_copy_list_setup");
record_inputs(inputList.Get(), parameters); dx(inputList->Close(), "Close(input copies)");
ID3D12CommandList* before[] = {inputList.Get()}; queue->ExecuteCommandLists(1, before);
mark("boundary_input_copy_submit");
try {
execute_cuda(queue);
mark("boundary_cuda_transaction");
record_output(outputList.Get(), parameters); dx(outputList->Close(), "Close(output copy)");
ID3D12CommandList* after[] = {outputList.Get()}; queue->ExecuteCommandLists(1, after);
mark("boundary_output_copy_submit");
rt_->timeline->drain(queue);
mark("boundary_output_copy_drain");
} catch (...) {
// Keep all recorded resources alive until submitted work is complete.
rt_->timeline->drain(queue); throw;
}
}
private:
void report_completed_copies() {
if (!copy_completion_ || !queue_) return;
const auto completed = copy_completion_->GetCompletedValue();
if (completed == UINT64_MAX) throw std::runtime_error("Device removed during boundary timing");
for (auto& slot : copy_slots_) if (slot->timing && slot->done && slot->done <= completed)
slot->timing->report(queue_.Get(), this);
}
void wait_copies(uint64_t value) {
if (copy_completion_->GetCompletedValue() >= value) return;
dx(copy_completion_->SetEventOnCompletion(value, copy_event_.value), "Async copy completion event");
if (WaitForSingleObject(copy_event_.value, 30000) != WAIT_OBJECT_0)
throw std::runtime_error("Async interop copy completion timeout");
}
void wait_worker() {
std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return jobs_.empty() && !worker_active_; });
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_);
}
CopySlot& copy_slot() {
report_completed_copies();
if (!copy_completion_) {
dx(rt_->device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(copy_completion_.GetAddressOf())), "Async copy fence");
copy_event_.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!copy_event_.value) throw std::runtime_error("Async copy event");
}
CopySlot* selected = nullptr;
const auto completed = copy_completion_->GetCompletedValue();
for (auto& slot : copy_slots_) if (slot->done <= completed) { selected = slot.get(); break; }
if (!selected && copy_slots_.size() == 3) {
selected = copy_slots_.front().get();
for (auto& slot : copy_slots_) if (slot->done < selected->done) selected = slot.get();
// Bounded ownership; this wait is outside the runtime mutex.
wait_copies(selected->done);
report_completed_copies();
}
if (selected) {
// The fence can advance between report_completed_copies() and
// slot selection. Collect that now-complete ticket before Reset
// and before submitted() overwrites its timing identity.
if (selected->timing) selected->timing->report(queue_.Get(), this);
dx(selected->input_allocator->Reset(), "Async input allocator Reset");
dx(selected->output_allocator->Reset(), "Async output allocator Reset");
dx(selected->input->Reset(selected->input_allocator.Get(), nullptr), "Async input list Reset");
dx(selected->output->Reset(selected->output_allocator.Get(), nullptr), "Async output list Reset");
return *selected;
}
auto slot = std::make_unique<CopySlot>();
auto create = [&](auto& allocator, auto& list) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Async copy allocator");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Async copy list");
};
create(slot->input_allocator, slot->input); create(slot->output_allocator, slot->output);
if (std::getenv("D4R_PROFILE_GPU_BOUNDARY")) slot->timing = std::make_unique<BoundaryTiming>(rt_->device.Get());
copy_slots_.push_back(std::move(slot)); return *copy_slots_.back();
}
bool resize_needed(void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing async resource ") + names[i]);
continue;
}
const auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Async interop requires single-sample 2D textures");
if (!planes_[i].shared || planes_[i].desc.Width != desc.Width || planes_[i].desc.Height != desc.Height || planes_[i].desc.Format != desc.Format) return true;
}
return false;
}
void evaluate_async(ID3D12CommandQueue* queue, void* parameters) {
commands::InternalScope internal;
std::lock_guard<std::mutex> submitted(submit_mutex_);
if (!queue || (queue_ && queue_.Get() != queue)) throw std::runtime_error("Async interop requires one feature queue");
ComPtr<ID3D12Device> queueDevice; dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "Async queue device");
ComPtr<IUnknown> a, b;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(a.GetAddressOf())), "Async queue identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(b.GetAddressOf())), "Async runtime identity");
if (a.Get() != b.Get()) throw std::runtime_error("Async queue device mismatch");
{ std::lock_guard<std::mutex> lock(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
{ std::lock_guard<std::mutex> lock(rt_->async_mutex);
if (!rt_->async_failure.empty()) throw std::runtime_error(rt_->async_failure);
if (rt_->async_queue && rt_->async_queue.Get() != queue) throw std::runtime_error("Async runtime features must share one D3D12 queue");
rt_->async_queue = queue; }
auto& slot = copy_slot();
if (resize_needed(parameters)) {
wait_worker(); if (copy_value_) wait_copies(copy_value_);
rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current(); prepare(parameters);
}
// Steady-state recording touches immutable allocation metadata and the
// caller's resource snapshot. It does not contend with CUDA or mutate
// the worker's NGX parameters / texture references.
queue_ = queue;
auto frame = std::make_shared<AsyncFrame>();
if (!frame->parameters) throw std::runtime_error("Async frame parameters allocation");
ngx::copy_create(parameters, frame->parameters); ngx::copy_frame(parameters, frame->parameters);
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* resource = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &resource);
frame->resources.emplace_back(resource);
d4r_ngx_set_d3d12_resource(frame->parameters, names[i], resource);
}
if (slot.timing) slot.timing->input_begin(slot.input.Get());
record_inputs(slot.input.Get(), parameters);
if (slot.timing) slot.timing->input_end(slot.input.Get());
dx(slot.input->Close(), "Async input Close");
if (slot.timing) slot.timing->output_begin(slot.output.Get());
record_output(slot.output.Get(), parameters);
if (slot.timing) slot.timing->output_end(slot.output.Get());
dx(slot.output->Close(), "Async output Close");
if (!worker_.joinable()) worker_ = std::thread([this] { work(); });
ID3D12CommandList* input[] = {slot.input.Get()}, *output[] = {slot.output.Get()};
// Allocate the job before any GPU wait is published. Worker cannot
// acquire the runtime mutex until every queue dependency is enqueued.
std::lock_guard<std::mutex> pending(jobs_mutex_);
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
frame->ticket = rt_->timeline->reserve(); rt_->async_tickets.push_back(frame->ticket.output); }
try { jobs_.push_back(frame); }
catch (...) {
std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_back(); throw;
}
try {
queue->ExecuteCommandLists(1, input);
rt_->timeline->enqueue(queue, frame->ticket);
queue->ExecuteCommandLists(1, output);
slot.done = ++copy_value_;
dx(queue->Signal(copy_completion_.Get(), slot.done), "Async output-copy retirement Signal");
if (slot.timing) slot.timing->submitted(frame->ticket.output);
} catch (const std::exception& e) {
jobs_.pop_back(); worker_failure_ = e.what();
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
rt_->async_tickets.pop_back(); rt_->async_failure = e.what(); }
(void)rt_->timeline->fence->Signal(frame->ticket.output);
jobs_changed_.notify_all(); rt_->async_changed.notify_all(); throw;
}
jobs_changed_.notify_one();
std::printf("D4R_ASYNC_SUBMIT ticket=%llu copy_slots=%zu cpu_frame_copies=0 frame_age=0\n",
(unsigned long long)frame->ticket.output, copy_slots_.size());
}
void work() noexcept {
commands::InternalScope internal;
for (;;) {
std::shared_ptr<AsyncFrame> frame;
{ std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return worker_stop_ || !jobs_.empty(); });
if (jobs_.empty()) return;
frame = std::move(jobs_.front()); jobs_.pop_front(); worker_active_ = true; }
{ std::unique_lock<std::mutex> order(rt_->async_mutex);
rt_->async_changed.wait(order, [&] { return !rt_->async_tickets.empty() && rt_->async_tickets.front() == frame->ticket.output; }); }
try {
std::lock_guard<std::mutex> lock(rt_->mutex);
{ std::lock_guard<std::mutex> pending(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
rt_->current(); prepare(frame->parameters); execute_cuda(nullptr, &frame->ticket);
} catch (const std::exception& e) {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_failure_ = e.what(); }
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_failure = e.what(); }
std::fprintf(stderr, "D4R_ASYNC_INTEROP_FAILURE %s\n", e.what());
// Poison the feature and release the queue for error handling;
// subsequent evaluates fail explicitly, never reuse an old frame.
(void)rt_->timeline->fence->Signal(frame->ticket.output);
}
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_front(); }
rt_->async_changed.notify_all();
frame.reset();
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_active_ = false; }
jobs_changed_.notify_all();
}
}
void prepare(void* parameters) {
static const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr;
(void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing resource ") + names[i]);
planes_[i].texture.Reset(); d4r_ngx_set_void(parameters_, names[i], nullptr); continue;
}
auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Interop baseline requires single-sample 2D textures");
const bool half = i == 0 || i == 2 || i == 3;
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
if (desc.Width > UINT32_MAX) throw std::runtime_error("Texture width exceeds GPU conversion ABI");
const PixelSpec spec = pixel_spec(i, desc.Format);
if (!pixel_supported(i, spec.storage)) throw std::runtime_error(std::string("Unsupported ") + names[i] + " DXGI format=" + std::to_string(desc.Format));
auto& plane = planes_[i];
if (!plane.shared || plane.desc.Width != desc.Width || plane.desc.Height != desc.Height || plane.desc.Format != desc.Format) {
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Synchronize(resize interop)");
plane.image.reset(); plane.canonical.reset(); plane.shared.reset();
plane.shared = std::make_unique<SharedPlane>(rt_->external, rt_->device.Get(), desc);
if (!spec.direct) {
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
plane.canonical = std::make_unique<PixelAllocation>(rt_->hip, unsigned(desc.Width), desc.Height, channels * (half ? 2 : 4));
}
plane.image = std::make_unique<cuda::Image>(rt_->images, unsigned(desc.Width), desc.Height, half ? 16 : 32, channels, i == 3, i == 0 ? 1 : 0);
plane.desc = desc; plane.spec = spec;
std::printf("D4R_FORMAT plane=%s dxgi=%u canonical_channels=%u canonical_bits=%u gpu_conversion=%u cpu_copy=0\n",
names[i], unsigned(desc.Format), channels, half ? 16u : 32u, !spec.direct);
}
plane.texture = texture;
d4r_ngx_set_void(parameters_, names[i], &plane.image->object);
}
ngx::copy_frame(parameters, parameters_);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Width")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Width", width);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Height")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Height", height);
for (const char* name : {"TransparencyMask", "DLSS.Input.Bias.Current.Color.Mask"}) d4r_ngx_set_void(parameters_, name, nullptr);
}
void record_inputs(ID3D12GraphicsCommandList* list, void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i : {0u, 1u, 2u, 4u}) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) continue;
static const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
planes_[i].shared->copy_input(list, texture, ngx_resource_access(parameters, states[i], D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE));
}
}
void execute_cuda(ID3D12CommandQueue* queue, const SharedTimeline::Ticket* ticket = nullptr) {
const auto begin = std::chrono::steady_clock::now();
auto stage = begin;
const bool profileStages = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profileStages) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - stage).count());
stage = std::chrono::steady_clock::now();
};
if (ticket) rt_->timeline->wait_ready(ticket->input); else rt_->timeline->wait_input(queue);
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input producer completion");
mark("input_fence_wait");
const bool batchInputs = std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr;
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
auto& plane = planes_[i];
void* data = plane.shared->mapped; uint64_t pitch = plane.shared->footprint.Footprint.RowPitch;
if (plane.canonical) {
if (batchInputs) rt_->pixels->convert_deferred(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
else rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
data = plane.canonical->data; pitch = plane.canonical->pitch;
}
plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch, batchInputs);
}
// HIP conversion and CUDA array copies are ordered on the shared
// legacy default stream. Retain an all-stream completion before NGX,
// including any internal nonblocking streams it might choose to use.
if (batchInputs) rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input batch completion");
mark("input_conversion_array_upload");
if (std::getenv("D4R_INTEROP_VERIFY")) {
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
uint32_t sample[8]{};
rt_->hip.check(rt_->hip.hipMemcpy(sample, planes_[i].shared->mapped,
std::min<size_t>(sizeof(sample), planes_[i].shared->bytes), hipMemcpyDeviceToHost), "DIAGNOSTIC external-memory sample");
std::printf("D4R_VRAM_VERIFY plane=%u words=%08x,%08x,%08x,%08x\n", i, sample[0], sample[1], sample[2], sample[3]);
const unsigned bytesPerPixel = i == 0 ? 8 : i == 2 ? 4 : 4;
std::vector<uint8_t> array(size_t(planes_[i].desc.Width) * planes_[i].desc.Height * bytesPerPixel);
planes_[i].image->download(array.data());
uint32_t first = 0; std::memcpy(&first, array.data(), 4);
std::printf("D4R_ARRAY_VERIFY plane=%u first=%08x\n", i, first);
}
}
ngx_check(rt_->evaluate(handle_, parameters_, nullptr), "CUDA EvaluateFeature");
mark("ngx_evaluate_host");
// NGX can use internal nonblocking streams. The default stream alone
// does not establish completion of those streams.
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "NGX all-stream completion");
mark("ngx_all_stream_completion");
auto& output = planes_[3];
output.image->download_device(reinterpret_cast<uintptr_t>(output.canonical ? output.canonical->data : output.shared->mapped),
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch, true);
mark("output_array_download");
if (std::getenv("D4R_VALIDATE_OUTPUT")) {
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "VRAM diagnostic output completion");
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
const auto counts = rt_->pixels->validate(output.canonical ? output.canonical->data : output.shared->mapped,
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height);
std::printf("D4R_OUTPUT_VALIDATION elements=%llu nan=%u inf=%u diagnostics_cpu_bytes=8\n", output.desc.Width * output.desc.Height * 4, counts[0], counts[1]);
if (counts[0] || counts[1]) throw std::runtime_error("DLSS output contains NaN/Inf");
}
mark("output_diagnostic");
if (output.canonical) rt_->pixels->convert_deferred(true, output.canonical->data, output.canonical->pitch, output.shared->mapped,
output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height, output.spec.storage, 3);
// One completion covers the array download and optional format encode.
// NGX's nonblocking streams were completed before this ordered batch.
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "VRAM output batch completion");
if (ticket) rt_->timeline->signal_ready(ticket->output); else rt_->timeline->signal_output(queue);
mark("output_conversion_fence_signal");
const double frameMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count();
std::printf("D4R_FRAME cpu_frame_copies=%u frame_age=0 interop_ngx_ms=%.3f\n", std::getenv("D4R_INTEROP_VERIFY") ? 1u : 0u, frameMs);
static std::atomic<bool> firstFrameLogged{false};
if (!firstFrameLogged.exchange(true))
std::fprintf(stderr, "d4r: first DLSS frame complete interop_ngx_ms=%.3f output=%ux%u\n",
frameMs, unsigned(output.desc.Width), output.desc.Height);
}
void record_output(ID3D12GraphicsCommandList* list, void* parameters) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, "Output", &texture);
if (!texture) throw std::runtime_error("Missing output resource");
planes_[3].shared->copy_output(list, texture, ngx_resource_access(parameters, "D4R.Output.State", D3D12_RESOURCE_STATE_UNORDERED_ACCESS));
}
};
}
Reference in new issue View Git Blame Copy Permalink