wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clear windows branch; preserve previous contents in win-old

Browse Source
This commit is contained in:
ayo committed 2026-10-16 05:20:08 -04:00
1 parent f0c881985b
commit 73994d4634
311 files changed
-57018
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/windows/ngx_windows_runtime.h
-652
View File
@@ -1,652 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include "d3d12_boundary_timing.h"
#include "pixel_conversion.h"
#include "cuda_image_api.h"
#include "ngx_parameters.h"
#include "ngx_public_abi.h"
#include "d3d12_command_hooks.h"
#include "d3d12_retirement.h"
#include <condition_variable>
#include <deque>
#include <thread>
#include <chrono>
#include <memory>
#include <mutex>
namespace d4r::win {
inline std::string env_path(const char* name) {
const char* value = std::getenv(name);
if (!value || !*value) throw std::runtime_error(std::string("Set the absolute path in ") + name);
if (!std::filesystem::path(diag::wide(value)).is_absolute()) throw std::runtime_error(std::string(name) + " must be absolute");
return value;
}
inline void ngx_check(unsigned result, const char* operation) {
if (result != 1 || !std::getenv("D4R_QUIET_API")) std::printf("D4R_NGX %s result=0x%08x\n", operation, result);
if (result != 1) throw std::runtime_error(std::string(operation) + " NGX=" + std::to_string(result));
}
inline ResourceAccess ngx_resource_access(void* parameters, const char* name, D3D12_RESOURCE_STATES fallback) {
ResourceAccess result(D3D12_RESOURCE_STATES(ngx::uint_value(parameters, name, unsigned(fallback))));
const std::string prefix(name);
result.enhanced = ngx::uint_value(parameters, (prefix + ".Enhanced").c_str()) != 0;
if (result.enhanced) {
result.layout = D3D12_BARRIER_LAYOUT(ngx::uint_value(parameters, (prefix + ".Layout").c_str()));
result.access = D3D12_BARRIER_ACCESS(ngx::uint_value(parameters, (prefix + ".Access").c_str()));
}
return result;
}
inline std::filesystem::path pixel_module_path() {
if (const char* path = std::getenv("D4R_FORMAT_MODULE")) return diag::wide(env_path("D4R_FORMAT_MODULE"));
HMODULE module = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
reinterpret_cast<LPCWSTR>(&pixel_module_path), &module)) throw std::runtime_error("Locate GPU conversion module directory");
wchar_t path[32768]{}; const DWORD length = GetModuleFileNameW(module, path, 32768);
if (!length || length == 32768) throw std::runtime_error("GPU conversion module path is invalid");
return std::filesystem::path(path).parent_path() / diag::wide(std::string("pixel_convert_") + diag::HipApi::target_arch() + ".hsaco");
}
struct Runtime {
diag::HipApi hip;
CudaApi cuda;
std::unique_ptr<diag::Library> nvapi;
diag::SearchDirectory featureSearch;
std::unique_ptr<diag::Library> dlss, core;
ExternalApi external;
cuda::ImageApi images;
ComPtr<ID3D12Device> device;
CUdevice ordinal = -1;
CUcontext context = nullptr;
bool initialized = false;
std::unique_ptr<SharedTimeline> timeline;
std::unique_ptr<PixelProgram> pixels;
std::mutex mutex;
// Features share one CUDA context. Follow submission order even when
// separate feature workers wake in a different order; otherwise a worker
// could hold the context while waiting for an earlier worker's output.
std::mutex async_mutex;
std::condition_variable async_changed;
std::deque<uint64_t> async_tickets;
std::string async_failure;
ComPtr<ID3D12CommandQueue> async_queue;
ngx::ProjectIdentity project;
ngx::FeatureCommonInfo common{};
std::vector<std::wstring> featurePaths;
std::vector<const wchar_t*> featurePathPointers;
using Init = unsigned(*)(unsigned long long, const wchar_t*, unsigned);
using Shutdown = unsigned(*)();
using Allocate = unsigned(*)(void**);
using Destroy = unsigned(*)(void*);
using Create = unsigned(*)(unsigned, void*, void**);
using Evaluate = unsigned(*)(void*, void*, void*);
using Release = unsigned(*)(void*);
Allocate allocate = nullptr, capabilities = nullptr;
Destroy destroy = nullptr;
Create create = nullptr;
Evaluate evaluate = nullptr;
Release release = nullptr;
Shutdown shutdown = nullptr;
Runtime(ID3D12Device* d3d, unsigned long long app, const wchar_t* data, unsigned sdk,
const ngx::FeatureCommonInfo* info = nullptr, const ngx::ProjectIdentity* identity = nullptr)
: hip(env_path("D4R_HIP_ROOT")), cuda(env_path("D4R_NVCUDA_DLL")),
featureSearch(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path()),
external{hip}, images(cuda), device(d3d) {
if (!d3d) throw std::runtime_error("Null D3D12 device");
try {
hipDeviceProp_t props{};
const auto luid = adapter_luid(d3d);
hip.select_architecture(-1, props, &luid);
hip.verbose = cuda.verbose = !std::getenv("D4R_QUIET_API");
if (std::memcmp(&luid, props.luid, sizeof(luid))) throw std::runtime_error("D3D12/HIP LUID mismatch");
cuda.check(cuda.cuInit(0), "cuInit(Windows runtime)");
int count = 0; cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
for (int i = 0; i < count; ++i) {
CUdevice candidate = -1; cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
int bus = -1, pciDevice = -1, domain = -1;
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
cuda.check(cuda.cuDeviceGetAttribute(&pciDevice, 34, candidate), "PCI device");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
if (bus == props.pciBusID && pciDevice == props.pciDeviceID && domain == props.pciDomainID) ordinal = candidate;
}
if (ordinal < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain");
current();
timeline = std::make_unique<SharedTimeline>(external, d3d);
std::printf("D4R_INTEROP async=%u batch_inputs=%u cpu_image_copies=0\n",
unsigned(std::getenv("D4R_ASYNC_INTEROP") != nullptr), unsigned(std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr));
// NGX's loader can probe CUDA during DLL initialization: establish
// the correct primary context before loading either NVIDIA DLL.
nvapi = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NVAPI_DLL")));
dlss = std::make_unique<diag::Library>(diag::wide(env_path("D4R_DLSS_DLL")));
core = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NGX_CORE")));
allocate = core->symbol<Allocate>("NVSDK_NGX_CUDA_AllocateParameters");
capabilities = core->symbol<Allocate>("NVSDK_NGX_CUDA_GetCapabilityParameters");
destroy = core->symbol<Destroy>("NVSDK_NGX_CUDA_DestroyParameters");
create = core->symbol<Create>("NVSDK_NGX_CUDA_CreateFeature");
evaluate = core->symbol<Evaluate>("NVSDK_NGX_CUDA_EvaluateFeature");
release = core->symbol<Release>("NVSDK_NGX_CUDA_ReleaseFeature");
shutdown = core->symbol<Shutdown>("NVSDK_NGX_CUDA_Shutdown");
if (identity) {
if (identity->id.empty()) throw std::runtime_error("Empty NGX project identity");
project = *identity;
featurePaths.push_back(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path().wstring());
if (info) {
if (info->PathListInfo.Length > 1024 || (info->PathListInfo.Length && !info->PathListInfo.Path)) throw std::runtime_error("Invalid NGX feature path list");
for (unsigned i = 0; i < info->PathListInfo.Length; ++i) if (info->PathListInfo.Path[i]) featurePaths.emplace_back(info->PathListInfo.Path[i]);
if (sdk >= 0x14) common.LoggingInfo = info->LoggingInfo;
}
for (const auto& path : featurePaths) featurePathPointers.push_back(path.c_str());
common.PathListInfo = {featurePathPointers.data(), unsigned(featurePathPointers.size())};
using ProjectInit = unsigned(*)(const char*, int, const char*, const wchar_t*, unsigned, const ngx::FeatureCommonInfo*);
std::printf("D4R_IDENTITY project=%s engine=%d version=%s sdk=0x%x\n", project.id.c_str(),project.engine,project.version.c_str(),sdk);
ngx_check(core->symbol<ProjectInit>("NVSDK_NGX_CUDA_Init_ProjectID")(project.id.c_str(),project.engine,project.version.c_str(),data,sdk,&common), "CUDA Init_ProjectID");
} else {
std::printf("D4R_IDENTITY application=%llu sdk=0x%x\n", app,sdk);
ngx_check(core->symbol<Init>("NVSDK_NGX_CUDA_Init")(app, data, sdk), "CUDA Init(driver ABI)");
}
initialized = true;
void* caps = nullptr;
ngx_check(capabilities(&caps), "CUDA feature capabilities");
if (!caps) throw std::runtime_error("Null CUDA capability parameters");
const int available = ngx::int_value(caps, "SuperSampling.Available", -1);
const int initResult = ngx::int_value(caps, "SuperSampling.FeatureInitResult", -1);
const int needsDriver = ngx::int_value(caps, "SuperSampling.NeedsUpdatedDriver", -1);
std::printf("D4R_CAPABILITIES available=%d feature_init=0x%08x needs_driver=%d\n", available, unsigned(initResult), needsDriver);
ngx_check(destroy(caps), "Destroy capability parameters");
if (available != 1 || initResult != 1 || needsDriver != 0)
throw std::runtime_error("CUDA DLSS is unavailable after capability initialization");
std::printf("D4R_RUNTIME platform=Windows architecture=%s cpu_frame_copies=0 frame_age=0\n", d4r::diag::HipApi::target_arch());
} catch (...) { cleanup(); throw; }
}
Runtime(const Runtime&) = delete;
~Runtime() { cleanup(); }
void current() { cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent(runtime)"); }
void drain_async_queue() {
ComPtr<ID3D12CommandQueue> queue;
{ std::lock_guard<std::mutex> lock(async_mutex); queue = async_queue; }
if (!queue) return;
// Creation/free/import may synchronize the AMD device internally.
// Drain queued D3D12 consumers before taking the CUDA mutex: workers
// must remain able to satisfy any already-published output waits.
ComPtr<ID3D12Fence> completion;
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(completion.GetAddressOf())), "Reconfigure fence");
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.value) throw std::runtime_error("Reconfigure event");
dx(queue->Signal(completion.Get(), 1), "Reconfigure queue Signal");
if (completion->GetCompletedValue() < 1) {
dx(completion->SetEventOnCompletion(1, event.value), "Reconfigure completion event");
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Reconfigure GPU timeout");
}
dx(device->GetDeviceRemovedReason(), "Reconfigure device status");
}
void cleanup() noexcept {
if (context) (void)cuda.cuCtxSetCurrent(context);
if (initialized) { (void)shutdown(); initialized = false; }
timeline.reset();
pixels.reset();
if (context) {
(void)cuda.cuCtxSynchronize(); (void)cuda.cuCtxSetCurrent(nullptr);
(void)cuda.cuDevicePrimaryCtxRelease_v2(ordinal); context = nullptr;
}
}
};
class Feature {
std::shared_ptr<Runtime> rt_;
void* parameters_ = nullptr;
void* handle_ = nullptr;
CUdeviceptr scratch_ = 0;
ComPtr<ID3D12CommandQueue> queue_;
// A boundary drains both copies before returning. Reuse their allocators
// only after that completion; no game allocator is retained here.
ComPtr<ID3D12CommandAllocator> input_allocator_, output_allocator_;
ComPtr<ID3D12GraphicsCommandList> input_list_, output_list_;
struct CopySlot {
ComPtr<ID3D12CommandAllocator> input_allocator, output_allocator;
ComPtr<ID3D12GraphicsCommandList> input, output;
std::unique_ptr<BoundaryTiming> timing;
uint64_t done = 0;
};
struct AsyncFrame {
void* parameters = d4r_ngx_parameters_create();
std::vector<ComPtr<ID3D12Resource>> resources;
SharedTimeline::Ticket ticket{};
~AsyncFrame() { if (parameters) d4r_ngx_parameters_destroy(parameters); }
};
std::vector<std::unique_ptr<CopySlot>> copy_slots_;
ComPtr<ID3D12Fence> copy_completion_;
Handle copy_event_;
uint64_t copy_value_ = 0;
std::mutex jobs_mutex_;
std::mutex submit_mutex_;
std::condition_variable jobs_changed_;
std::deque<std::shared_ptr<AsyncFrame>> jobs_;
std::thread worker_;
bool worker_stop_ = false, worker_active_ = false;
std::string worker_failure_;
struct Plane {
ComPtr<ID3D12Resource> texture;
D3D12_RESOURCE_DESC desc{};
std::unique_ptr<SharedPlane> shared;
std::unique_ptr<cuda::Image> image;
std::unique_ptr<PixelAllocation> canonical;
PixelSpec spec{};
} planes_[5];
void cleanup() noexcept {
(void)rt_->cuda.cuCtxSetCurrent(rt_->context);
(void)rt_->cuda.cuCtxSynchronize();
if (handle_) { (void)rt_->release(handle_); handle_ = nullptr; }
for (auto& plane : planes_) {
plane.image.reset(); plane.canonical.reset(); plane.shared.reset(); plane.texture.Reset();
}
if (scratch_) { (void)rt_->cuda.cuMemFree_v2(scratch_); scratch_ = 0; }
if (parameters_) { (void)rt_->destroy(parameters_); parameters_ = nullptr; }
}
public:
unsigned width = 0, height = 0;
explicit Feature(std::shared_ptr<Runtime> runtime, void* parameters) : rt_(std::move(runtime)) {
if (std::getenv("D4R_ASYNC_INTEROP")) rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current();
try {
ngx_check(rt_->allocate(&parameters_), "CUDA AllocateParameters");
ngx::copy_create(parameters, parameters_);
d4r_ngx_set_int(parameters_, "DLSS.Enable.Output.Subrects", ngx::int_value(parameters, "DLSS.Enable.Output.Subrects"));
width = ngx::uint_value(parameters, "Width"); height = ngx::uint_value(parameters, "Height");
std::printf("D4R_CREATE source=%ux%u->%ux%u cuda=%ux%u->%ux%u\n", width, height,
ngx::uint_value(parameters, "OutWidth"), ngx::uint_value(parameters, "OutHeight"),
ngx::uint_value(parameters_, "Width"), ngx::uint_value(parameters_, "Height"),
ngx::uint_value(parameters_, "OutWidth"), ngx::uint_value(parameters_, "OutHeight"));
if (!width || !height) throw std::runtime_error("Zero NGX input dimensions");
rt_->cuda.check(rt_->cuda.cuMemAlloc_v2(&scratch_, 64ull * 1024 * 1024), "NGX VRAM scratch");
d4r_ngx_set_void(parameters_, "Scratch", reinterpret_cast<void*>(uintptr_t(scratch_)));
d4r_ngx_set_ull(parameters_, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
ngx_check(rt_->create(1, parameters_, &handle_), "CUDA CreateFeature");
} catch (...) { cleanup(); throw; }
}
~Feature() {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_stop_ = true; }
jobs_changed_.notify_all();
if (worker_.joinable()) worker_.join();
if (copy_completion_) {
try { wait_copies(copy_value_); report_completed_copies(); }
catch (const std::exception& e) { std::fprintf(stderr, "D4R_ASYNC_CLEANUP_FAILURE %s\n", e.what()); }
}
std::lock_guard<std::mutex> lock(rt_->mutex); cleanup();
}
Feature(const Feature&) = delete;
void drain() {
if (!std::getenv("D4R_ASYNC_INTEROP")) return;
wait_worker();
if (copy_value_) wait_copies(copy_value_);
report_completed_copies();
rt_->drain_async_queue();
}
// Explicit harness boundary: all earlier producers must already be queued.
// The forthcoming command-list backend calls the same stages at queue submit.
void evaluate_boundary(ID3D12CommandQueue* queue, void* parameters) {
if (std::getenv("D4R_ASYNC_INTEROP")) { evaluate_async(queue, parameters); return; }
const auto boundaryBegin = std::chrono::steady_clock::now();
auto boundaryStage = boundaryBegin;
const bool profile = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profile) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - boundaryStage).count());
boundaryStage = std::chrono::steady_clock::now();
};
commands::InternalScope internal;
std::lock_guard<std::mutex> lock(rt_->mutex);
if (!queue) throw std::runtime_error("Null D3D12 queue");
if (queue_ && queue_.Get() != queue) throw std::runtime_error("A feature cannot switch queues without an explicit drain");
queue_ = queue;
ComPtr<ID3D12Device> queueDevice;
dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "GetDevice(queue)");
ComPtr<IUnknown> queueIdentity, runtimeIdentity;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(queueIdentity.GetAddressOf())), "Query queue device identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(runtimeIdentity.GetAddressOf())), "Query runtime device identity");
if (queueIdentity.Get() != runtimeIdentity.Get()) throw std::runtime_error("Queue belongs to a different D3D12 device");
rt_->current();
prepare(parameters);
mark("boundary_prepare");
ComPtr<ID3D12CommandAllocator> inputAllocator, outputAllocator;
ComPtr<ID3D12GraphicsCommandList> inputList, outputList;
auto list = [&](ComPtr<ID3D12CommandAllocator>& allocator, ComPtr<ID3D12GraphicsCommandList>& commandList) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator(interop)");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(commandList.GetAddressOf())), "CreateCommandList(interop)");
};
const bool cached = !std::getenv("D4R_DISABLE_INTEROP_LIST_CACHE");
if (cached) {
if (!input_list_) {
// Publish the pair only after both creations succeed.
list(inputAllocator, inputList); list(outputAllocator, outputList);
input_allocator_ = inputAllocator; output_allocator_ = outputAllocator;
input_list_ = inputList; output_list_ = outputList;
} else {
dx(input_allocator_->Reset(), "Reset(input interop allocator)");
dx(output_allocator_->Reset(), "Reset(output interop allocator)");
dx(input_list_->Reset(input_allocator_.Get(), nullptr), "Reset(input interop list)");
dx(output_list_->Reset(output_allocator_.Get(), nullptr), "Reset(output interop list)");
}
inputAllocator = input_allocator_; outputAllocator = output_allocator_;
inputList = input_list_; outputList = output_list_;
} else { list(inputAllocator, inputList); list(outputAllocator, outputList); }
mark("boundary_copy_list_setup");
record_inputs(inputList.Get(), parameters); dx(inputList->Close(), "Close(input copies)");
ID3D12CommandList* before[] = {inputList.Get()}; queue->ExecuteCommandLists(1, before);
mark("boundary_input_copy_submit");
try {
execute_cuda(queue);
mark("boundary_cuda_transaction");
record_output(outputList.Get(), parameters); dx(outputList->Close(), "Close(output copy)");
ID3D12CommandList* after[] = {outputList.Get()}; queue->ExecuteCommandLists(1, after);
mark("boundary_output_copy_submit");
rt_->timeline->drain(queue);
mark("boundary_output_copy_drain");
} catch (...) {
// Keep all recorded resources alive until submitted work is complete.
rt_->timeline->drain(queue); throw;
}
}
private:
void report_completed_copies() {
if (!copy_completion_ || !queue_) return;
const auto completed = copy_completion_->GetCompletedValue();
if (completed == UINT64_MAX) throw std::runtime_error("Device removed during boundary timing");
for (auto& slot : copy_slots_) if (slot->timing && slot->done && slot->done <= completed)
slot->timing->report(queue_.Get(), this);
}
void wait_copies(uint64_t value) {
if (copy_completion_->GetCompletedValue() >= value) return;
dx(copy_completion_->SetEventOnCompletion(value, copy_event_.value), "Async copy completion event");
if (WaitForSingleObject(copy_event_.value, 30000) != WAIT_OBJECT_0)
throw std::runtime_error("Async interop copy completion timeout");
}
void wait_worker() {
std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return jobs_.empty() && !worker_active_; });
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_);
}
CopySlot& copy_slot() {
report_completed_copies();
if (!copy_completion_) {
dx(rt_->device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(copy_completion_.GetAddressOf())), "Async copy fence");
copy_event_.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!copy_event_.value) throw std::runtime_error("Async copy event");
}
CopySlot* selected = nullptr;
const auto completed = copy_completion_->GetCompletedValue();
for (auto& slot : copy_slots_) if (slot->done <= completed) { selected = slot.get(); break; }
if (!selected && copy_slots_.size() == 3) {
selected = copy_slots_.front().get();
for (auto& slot : copy_slots_) if (slot->done < selected->done) selected = slot.get();
// Bounded ownership; this wait is outside the runtime mutex.
wait_copies(selected->done);
report_completed_copies();
}
if (selected) {
// The fence can advance between report_completed_copies() and
// slot selection. Collect that now-complete ticket before Reset
// and before submitted() overwrites its timing identity.
if (selected->timing) selected->timing->report(queue_.Get(), this);
dx(selected->input_allocator->Reset(), "Async input allocator Reset");
dx(selected->output_allocator->Reset(), "Async output allocator Reset");
dx(selected->input->Reset(selected->input_allocator.Get(), nullptr), "Async input list Reset");
dx(selected->output->Reset(selected->output_allocator.Get(), nullptr), "Async output list Reset");
return *selected;
}
auto slot = std::make_unique<CopySlot>();
auto create = [&](auto& allocator, auto& list) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Async copy allocator");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Async copy list");
};
create(slot->input_allocator, slot->input); create(slot->output_allocator, slot->output);
if (std::getenv("D4R_PROFILE_GPU_BOUNDARY")) slot->timing = std::make_unique<BoundaryTiming>(rt_->device.Get());
copy_slots_.push_back(std::move(slot)); return *copy_slots_.back();
}
bool resize_needed(void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing async resource ") + names[i]);
continue;
}
const auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Async interop requires single-sample 2D textures");
if (!planes_[i].shared || planes_[i].desc.Width != desc.Width || planes_[i].desc.Height != desc.Height || planes_[i].desc.Format != desc.Format) return true;
}
return false;
}
void evaluate_async(ID3D12CommandQueue* queue, void* parameters) {
commands::InternalScope internal;
std::lock_guard<std::mutex> submitted(submit_mutex_);
if (!queue || (queue_ && queue_.Get() != queue)) throw std::runtime_error("Async interop requires one feature queue");
ComPtr<ID3D12Device> queueDevice; dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "Async queue device");
ComPtr<IUnknown> a, b;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(a.GetAddressOf())), "Async queue identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(b.GetAddressOf())), "Async runtime identity");
if (a.Get() != b.Get()) throw std::runtime_error("Async queue device mismatch");
{ std::lock_guard<std::mutex> lock(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
{ std::lock_guard<std::mutex> lock(rt_->async_mutex);
if (!rt_->async_failure.empty()) throw std::runtime_error(rt_->async_failure);
if (rt_->async_queue && rt_->async_queue.Get() != queue) throw std::runtime_error("Async runtime features must share one D3D12 queue");
rt_->async_queue = queue; }
auto& slot = copy_slot();
if (resize_needed(parameters)) {
wait_worker(); if (copy_value_) wait_copies(copy_value_);
rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current(); prepare(parameters);
}
// Steady-state recording touches immutable allocation metadata and the
// caller's resource snapshot. It does not contend with CUDA or mutate
// the worker's NGX parameters / texture references.
queue_ = queue;
auto frame = std::make_shared<AsyncFrame>();
if (!frame->parameters) throw std::runtime_error("Async frame parameters allocation");
ngx::copy_create(parameters, frame->parameters); ngx::copy_frame(parameters, frame->parameters);
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* resource = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &resource);
frame->resources.emplace_back(resource);
d4r_ngx_set_d3d12_resource(frame->parameters, names[i], resource);
}
if (slot.timing) slot.timing->input_begin(slot.input.Get());
record_inputs(slot.input.Get(), parameters);
if (slot.timing) slot.timing->input_end(slot.input.Get());
dx(slot.input->Close(), "Async input Close");
if (slot.timing) slot.timing->output_begin(slot.output.Get());
record_output(slot.output.Get(), parameters);
if (slot.timing) slot.timing->output_end(slot.output.Get());
dx(slot.output->Close(), "Async output Close");
if (!worker_.joinable()) worker_ = std::thread([this] { work(); });
ID3D12CommandList* input[] = {slot.input.Get()}, *output[] = {slot.output.Get()};
// Allocate the job before any GPU wait is published. Worker cannot
// acquire the runtime mutex until every queue dependency is enqueued.
std::lock_guard<std::mutex> pending(jobs_mutex_);
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
frame->ticket = rt_->timeline->reserve(); rt_->async_tickets.push_back(frame->ticket.output); }
try { jobs_.push_back(frame); }
catch (...) {
std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_back(); throw;
}
try {
queue->ExecuteCommandLists(1, input);
rt_->timeline->enqueue(queue, frame->ticket);
queue->ExecuteCommandLists(1, output);
slot.done = ++copy_value_;
dx(queue->Signal(copy_completion_.Get(), slot.done), "Async output-copy retirement Signal");
if (slot.timing) slot.timing->submitted(frame->ticket.output);
} catch (const std::exception& e) {
jobs_.pop_back(); worker_failure_ = e.what();
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
rt_->async_tickets.pop_back(); rt_->async_failure = e.what(); }
(void)rt_->timeline->fence->Signal(frame->ticket.output);
jobs_changed_.notify_all(); rt_->async_changed.notify_all(); throw;
}
jobs_changed_.notify_one();
std::printf("D4R_ASYNC_SUBMIT ticket=%llu copy_slots=%zu cpu_frame_copies=0 frame_age=0\n",
(unsigned long long)frame->ticket.output, copy_slots_.size());
}
void work() noexcept {
commands::InternalScope internal;
for (;;) {
std::shared_ptr<AsyncFrame> frame;
{ std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return worker_stop_ || !jobs_.empty(); });
if (jobs_.empty()) return;
frame = std::move(jobs_.front()); jobs_.pop_front(); worker_active_ = true; }
{ std::unique_lock<std::mutex> order(rt_->async_mutex);
rt_->async_changed.wait(order, [&] { return !rt_->async_tickets.empty() && rt_->async_tickets.front() == frame->ticket.output; }); }
try {
std::lock_guard<std::mutex> lock(rt_->mutex);
{ std::lock_guard<std::mutex> pending(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
rt_->current(); prepare(frame->parameters); execute_cuda(nullptr, &frame->ticket);
} catch (const std::exception& e) {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_failure_ = e.what(); }
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_failure = e.what(); }
std::fprintf(stderr, "D4R_ASYNC_INTEROP_FAILURE %s\n", e.what());
// Poison the feature and release the queue for error handling;
// subsequent evaluates fail explicitly, never reuse an old frame.
(void)rt_->timeline->fence->Signal(frame->ticket.output);
}
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_front(); }
rt_->async_changed.notify_all();
frame.reset();
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_active_ = false; }
jobs_changed_.notify_all();
}
}
void prepare(void* parameters) {
static const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr;
(void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing resource ") + names[i]);
planes_[i].texture.Reset(); d4r_ngx_set_void(parameters_, names[i], nullptr); continue;
}
auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Interop baseline requires single-sample 2D textures");
const bool half = i == 0 || i == 2 || i == 3;
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
if (desc.Width > UINT32_MAX) throw std::runtime_error("Texture width exceeds GPU conversion ABI");
const PixelSpec spec = pixel_spec(i, desc.Format);
if (!pixel_supported(i, spec.storage)) throw std::runtime_error(std::string("Unsupported ") + names[i] + " DXGI format=" + std::to_string(desc.Format));
auto& plane = planes_[i];
if (!plane.shared || plane.desc.Width != desc.Width || plane.desc.Height != desc.Height || plane.desc.Format != desc.Format) {
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Synchronize(resize interop)");
plane.image.reset(); plane.canonical.reset(); plane.shared.reset();
plane.shared = std::make_unique<SharedPlane>(rt_->external, rt_->device.Get(), desc);
if (!spec.direct) {
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
plane.canonical = std::make_unique<PixelAllocation>(rt_->hip, unsigned(desc.Width), desc.Height, channels * (half ? 2 : 4));
}
plane.image = std::make_unique<cuda::Image>(rt_->images, unsigned(desc.Width), desc.Height, half ? 16 : 32, channels, i == 3, i == 0 ? 1 : 0);
plane.desc = desc; plane.spec = spec;
std::printf("D4R_FORMAT plane=%s dxgi=%u canonical_channels=%u canonical_bits=%u gpu_conversion=%u cpu_copy=0\n",
names[i], unsigned(desc.Format), channels, half ? 16u : 32u, !spec.direct);
}
plane.texture = texture;
d4r_ngx_set_void(parameters_, names[i], &plane.image->object);
}
ngx::copy_frame(parameters, parameters_);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Width")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Width", width);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Height")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Height", height);
for (const char* name : {"TransparencyMask", "DLSS.Input.Bias.Current.Color.Mask"}) d4r_ngx_set_void(parameters_, name, nullptr);
}
void record_inputs(ID3D12GraphicsCommandList* list, void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i : {0u, 1u, 2u, 4u}) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) continue;
static const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
planes_[i].shared->copy_input(list, texture, ngx_resource_access(parameters, states[i], D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE));
}
}
void execute_cuda(ID3D12CommandQueue* queue, const SharedTimeline::Ticket* ticket = nullptr) {
const auto begin = std::chrono::steady_clock::now();
auto stage = begin;
const bool profileStages = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profileStages) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - stage).count());
stage = std::chrono::steady_clock::now();
};
if (ticket) rt_->timeline->wait_ready(ticket->input); else rt_->timeline->wait_input(queue);
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input producer completion");
mark("input_fence_wait");
const bool batchInputs = std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr;
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
auto& plane = planes_[i];
void* data = plane.shared->mapped; uint64_t pitch = plane.shared->footprint.Footprint.RowPitch;
if (plane.canonical) {
if (batchInputs) rt_->pixels->convert_deferred(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
else rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
data = plane.canonical->data; pitch = plane.canonical->pitch;
}
plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch, batchInputs);
}
// HIP conversion and CUDA array copies are ordered on the shared
// legacy default stream. Retain an all-stream completion before NGX,
// including any internal nonblocking streams it might choose to use.
if (batchInputs) rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input batch completion");
mark("input_conversion_array_upload");
if (std::getenv("D4R_INTEROP_VERIFY")) {
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
uint32_t sample[8]{};
rt_->hip.check(rt_->hip.hipMemcpy(sample, planes_[i].shared->mapped,
std::min<size_t>(sizeof(sample), planes_[i].shared->bytes), hipMemcpyDeviceToHost), "DIAGNOSTIC external-memory sample");
std::printf("D4R_VRAM_VERIFY plane=%u words=%08x,%08x,%08x,%08x\n", i, sample[0], sample[1], sample[2], sample[3]);
const unsigned bytesPerPixel = i == 0 ? 8 : i == 2 ? 4 : 4;
std::vector<uint8_t> array(size_t(planes_[i].desc.Width) * planes_[i].desc.Height * bytesPerPixel);
planes_[i].image->download(array.data());
uint32_t first = 0; std::memcpy(&first, array.data(), 4);
std::printf("D4R_ARRAY_VERIFY plane=%u first=%08x\n", i, first);
}
}
ngx_check(rt_->evaluate(handle_, parameters_, nullptr), "CUDA EvaluateFeature");
mark("ngx_evaluate_host");
// NGX can use internal nonblocking streams. The default stream alone
// does not establish completion of those streams.
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "NGX all-stream completion");
mark("ngx_all_stream_completion");
auto& output = planes_[3];
output.image->download_device(reinterpret_cast<uintptr_t>(output.canonical ? output.canonical->data : output.shared->mapped),
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch);
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "VRAM output completion");
mark("output_array_download");
if (std::getenv("D4R_VALIDATE_OUTPUT")) {
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
const auto counts = rt_->pixels->validate(output.canonical ? output.canonical->data : output.shared->mapped,
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height);
std::printf("D4R_OUTPUT_VALIDATION elements=%llu nan=%u inf=%u diagnostics_cpu_bytes=8\n", output.desc.Width * output.desc.Height * 4, counts[0], counts[1]);
if (counts[0] || counts[1]) throw std::runtime_error("DLSS output contains NaN/Inf");
}
mark("output_diagnostic");
if (output.canonical) rt_->pixels->convert(true, output.canonical->data, output.canonical->pitch, output.shared->mapped,
output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height, output.spec.storage, 3);
if (ticket) rt_->timeline->signal_ready(ticket->output); else rt_->timeline->signal_output(queue);
mark("output_conversion_fence_signal");
std::printf("D4R_FRAME cpu_frame_copies=%u frame_age=0 interop_ngx_ms=%.3f\n", std::getenv("D4R_INTEROP_VERIFY") ? 1u : 0u,
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count());
}
void record_output(ID3D12GraphicsCommandList* list, void* parameters) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, "Output", &texture);
if (!texture) throw std::runtime_error("Missing output resource");
planes_[3].shared->copy_output(list, texture, ngx_resource_access(parameters, "D4R.Output.State", D3D12_RESOURCE_STATE_UNORDERED_ACCESS));
}
};
}