feat: add d4r inference engines and Windows parity tooling
Add native K/M/RR inference engines, HIP kernels, replay and validation harnesses, and Windows DLSS setup and runtime support. Include GPU profiling tools, build and packaging scripts, platform integration patches, and implementation documentation.
This commit is contained in:
ayo committed
1 parent
25493b3198
commit
7c5d3c9157
430 files changed
+76191
No files matched your search
@@ -0,0 +1,693 @@
|
||||
#pragma once
|
||||
#include "d3d12_external.h"
|
||||
#include "d3d12_boundary_timing.h"
|
||||
#include "pixel_conversion.h"
|
||||
#include "cuda_image_api.h"
|
||||
#include "ngx_parameters.h"
|
||||
#include "ngx_public_abi.h"
|
||||
#include "d3d12_command_hooks.h"
|
||||
#include "d3d12_retirement.h"
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <thread>
|
||||
#include <chrono>
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
|
||||
namespace d4r::win {
|
||||
inline std::string env_path(const char* name) {
|
||||
const char* value = std::getenv(name);
|
||||
if (!value || !*value) throw std::runtime_error(std::string("Set the absolute path in ") + name);
|
||||
if (!std::filesystem::path(diag::wide(value)).is_absolute()) throw std::runtime_error(std::string(name) + " must be absolute");
|
||||
return value;
|
||||
}
|
||||
inline void ngx_check(unsigned result, const char* operation) {
|
||||
if (result != 1 || !std::getenv("D4R_QUIET_API")) std::printf("D4R_NGX %s result=0x%08x\n", operation, result);
|
||||
if (result != 1) throw std::runtime_error(std::string(operation) + " NGX=" + std::to_string(result));
|
||||
}
|
||||
inline void ngx_log_callback(const char* message, int level, unsigned component) {
|
||||
if (message) std::printf("NGX_LOG level=%d component=%u %s\n", level, component, message);
|
||||
}
|
||||
inline ResourceAccess ngx_resource_access(void* parameters, const char* name, D3D12_RESOURCE_STATES fallback) {
|
||||
ResourceAccess result(D3D12_RESOURCE_STATES(ngx::uint_value(parameters, name, unsigned(fallback))));
|
||||
const std::string prefix(name);
|
||||
result.enhanced = ngx::uint_value(parameters, (prefix + ".Enhanced").c_str()) != 0;
|
||||
if (result.enhanced) {
|
||||
result.layout = D3D12_BARRIER_LAYOUT(ngx::uint_value(parameters, (prefix + ".Layout").c_str()));
|
||||
result.access = D3D12_BARRIER_ACCESS(ngx::uint_value(parameters, (prefix + ".Access").c_str()));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
inline std::filesystem::path pixel_module_path() {
|
||||
if (const char* path = std::getenv("D4R_FORMAT_MODULE")) return diag::wide(env_path("D4R_FORMAT_MODULE"));
|
||||
HMODULE module = nullptr;
|
||||
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
||||
reinterpret_cast<LPCWSTR>(&pixel_module_path), &module)) throw std::runtime_error("Locate GPU conversion module directory");
|
||||
wchar_t path[32768]{}; const DWORD length = GetModuleFileNameW(module, path, 32768);
|
||||
if (!length || length == 32768) throw std::runtime_error("GPU conversion module path is invalid");
|
||||
return std::filesystem::path(path).parent_path() / diag::wide(std::string("pixel_convert_") + diag::HipApi::target_arch() + ".hsaco");
|
||||
}
|
||||
|
||||
struct Runtime {
|
||||
diag::HipApi hip;
|
||||
CudaApi cuda;
|
||||
std::unique_ptr<diag::Library> nvapi;
|
||||
diag::SearchDirectory featureSearch;
|
||||
std::unique_ptr<diag::Library> dlss, core;
|
||||
ExternalApi external;
|
||||
cuda::ImageApi images;
|
||||
ComPtr<ID3D12Device> device;
|
||||
CUdevice ordinal = -1;
|
||||
CUcontext context = nullptr;
|
||||
bool initialized = false;
|
||||
std::unique_ptr<SharedTimeline> timeline;
|
||||
std::unique_ptr<PixelProgram> pixels;
|
||||
std::mutex mutex;
|
||||
// Features share one CUDA context. Follow submission order even when
|
||||
// separate feature workers wake in a different order; otherwise a worker
|
||||
// could hold the context while waiting for an earlier worker's output.
|
||||
std::mutex async_mutex;
|
||||
std::condition_variable async_changed;
|
||||
std::deque<uint64_t> async_tickets;
|
||||
std::string async_failure;
|
||||
ComPtr<ID3D12CommandQueue> async_queue;
|
||||
ngx::ProjectIdentity project;
|
||||
ngx::FeatureCommonInfo common{};
|
||||
std::vector<std::wstring> featurePaths;
|
||||
std::vector<const wchar_t*> featurePathPointers;
|
||||
using Init = unsigned(*)(unsigned long long, const wchar_t*, const ngx::FeatureCommonInfo*, unsigned);
|
||||
using Shutdown = unsigned(*)();
|
||||
using Allocate = unsigned(*)(void**);
|
||||
using Destroy = unsigned(*)(void*);
|
||||
using Create = unsigned(*)(unsigned, void*, void**);
|
||||
using Evaluate = unsigned(*)(void*, void*, void*);
|
||||
using Release = unsigned(*)(void*);
|
||||
using ScratchSize = unsigned(*)(unsigned, void*, size_t*);
|
||||
Allocate allocate = nullptr, capabilities = nullptr;
|
||||
Destroy destroy = nullptr;
|
||||
Create create = nullptr;
|
||||
Evaluate evaluate = nullptr;
|
||||
Release release = nullptr;
|
||||
ScratchSize getScratchBufferSize = nullptr;
|
||||
Shutdown shutdown = nullptr;
|
||||
|
||||
Runtime(ID3D12Device* d3d, unsigned long long app, const wchar_t* data, unsigned sdk,
|
||||
const ngx::FeatureCommonInfo* info = nullptr, const ngx::ProjectIdentity* identity = nullptr)
|
||||
: hip(env_path("D4R_HIP_ROOT")), cuda(env_path("D4R_NVCUDA_DLL")),
|
||||
featureSearch(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path()),
|
||||
external{hip}, images(cuda), device(d3d) {
|
||||
if (!d3d) throw std::runtime_error("Null D3D12 device");
|
||||
try {
|
||||
hipDeviceProp_t props{};
|
||||
const auto luid = adapter_luid(d3d);
|
||||
hip.select_architecture(-1, props, &luid);
|
||||
hip.verbose = cuda.verbose = !std::getenv("D4R_QUIET_API");
|
||||
if (std::memcmp(&luid, props.luid, sizeof(luid))) throw std::runtime_error("D3D12/HIP LUID mismatch");
|
||||
cuda.check(cuda.cuInit(0), "cuInit(Windows runtime)");
|
||||
int count = 0; cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
|
||||
for (int i = 0; i < count; ++i) {
|
||||
CUdevice candidate = -1; cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
|
||||
int bus = -1, pciDevice = -1, domain = -1;
|
||||
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
|
||||
cuda.check(cuda.cuDeviceGetAttribute(&pciDevice, 34, candidate), "PCI device");
|
||||
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
|
||||
if (bus == props.pciBusID && pciDevice == props.pciDeviceID && domain == props.pciDomainID) ordinal = candidate;
|
||||
}
|
||||
if (ordinal < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
|
||||
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain");
|
||||
current();
|
||||
timeline = std::make_unique<SharedTimeline>(external, d3d);
|
||||
std::printf("D4R_INTEROP async=%u batch_inputs=%u cpu_image_copies=0\n",
|
||||
unsigned(std::getenv("D4R_ASYNC_INTEROP") != nullptr), unsigned(std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr));
|
||||
// NGX's loader can probe CUDA during DLL initialization: establish
|
||||
// the correct primary context before loading either NVIDIA DLL.
|
||||
nvapi = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NVAPI_DLL")));
|
||||
dlss = std::make_unique<diag::Library>(diag::wide(env_path("D4R_DLSS_DLL")));
|
||||
core = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NGX_CORE")));
|
||||
allocate = core->symbol<Allocate>("NVSDK_NGX_CUDA_AllocateParameters");
|
||||
capabilities = core->symbol<Allocate>("NVSDK_NGX_CUDA_GetCapabilityParameters");
|
||||
destroy = core->symbol<Destroy>("NVSDK_NGX_CUDA_DestroyParameters");
|
||||
create = core->symbol<Create>("NVSDK_NGX_CUDA_CreateFeature");
|
||||
evaluate = core->symbol<Evaluate>("NVSDK_NGX_CUDA_EvaluateFeature");
|
||||
release = core->symbol<Release>("NVSDK_NGX_CUDA_ReleaseFeature");
|
||||
getScratchBufferSize = core->symbol<ScratchSize>("NVSDK_NGX_CUDA_GetScratchBufferSize");
|
||||
shutdown = core->symbol<Shutdown>("NVSDK_NGX_CUDA_Shutdown");
|
||||
const auto srDirectory = std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path();
|
||||
if (std::filesystem::exists(srDirectory / L"nvngx_dlssd.dll"))
|
||||
throw std::runtime_error("DLSS SR and Ray Reconstruction DLLs share a feature search directory; keep nvngx_dlssd.dll under d4r\\rr");
|
||||
featurePaths.emplace_back(srDirectory.wstring());
|
||||
if (info) {
|
||||
if (info->PathListInfo.Length > 1024 || (info->PathListInfo.Length && !info->PathListInfo.Path))
|
||||
throw std::runtime_error("Invalid NGX feature path list");
|
||||
for (unsigned i = 0; i < info->PathListInfo.Length; ++i) {
|
||||
if (!info->PathListInfo.Path[i]) continue;
|
||||
const std::filesystem::path candidate(info->PathListInfo.Path[i]);
|
||||
if (candidate.lexically_normal() != srDirectory.lexically_normal()) {
|
||||
std::printf("D4R_NGX_PATH skip=%ls reason=outside-pinned-SR-directory\n", candidate.c_str());
|
||||
continue;
|
||||
}
|
||||
std::printf("D4R_NGX_PATH duplicate=%ls reason=pinned-SR-directory\n", candidate.c_str());
|
||||
}
|
||||
}
|
||||
for (const auto& path : featurePaths) featurePathPointers.push_back(path.c_str());
|
||||
common.PathListInfo = {featurePathPointers.data(), unsigned(featurePathPointers.size())};
|
||||
common.LoggingInfo = {reinterpret_cast<void*>(&ngx_log_callback), 1, false};
|
||||
if (info && sdk >= 0x14) common.LoggingInfo = info->LoggingInfo;
|
||||
std::filesystem::path applicationData = data && *data ? std::filesystem::path(data) : std::filesystem::current_path();
|
||||
if (const char* redirected = std::getenv("D4R_NGX_DATA_DIR"); redirected && *redirected) {
|
||||
applicationData = diag::wide(env_path("D4R_NGX_DATA_DIR"));
|
||||
std::filesystem::create_directories(applicationData);
|
||||
std::printf("D4R_NGX_DATA_PATH redirect=%ls\n", applicationData.c_str());
|
||||
}
|
||||
if (!applicationData.is_absolute()) throw std::runtime_error("NGX application data path must be absolute");
|
||||
if (identity) {
|
||||
if (identity->id.empty()) throw std::runtime_error("Empty NGX project identity");
|
||||
project = *identity;
|
||||
using ProjectInit = unsigned(*)(const char*, int, const char*, const wchar_t*, unsigned, const ngx::FeatureCommonInfo*);
|
||||
std::printf("D4R_IDENTITY project=%s engine=%d version=%s sdk=0x%x\n", project.id.c_str(),project.engine,project.version.c_str(),sdk);
|
||||
ngx_check(core->symbol<ProjectInit>("NVSDK_NGX_CUDA_Init_ProjectID")(project.id.c_str(),project.engine,
|
||||
project.version.c_str(),applicationData.c_str(),sdk,&common), "CUDA Init_ProjectID");
|
||||
} else {
|
||||
std::printf("D4R_IDENTITY application=%llu sdk=0x%x\n", app,sdk);
|
||||
ngx_check(core->symbol<Init>("NVSDK_NGX_CUDA_Init")(app, applicationData.c_str(), &common, sdk), "CUDA Init(application ABI)");
|
||||
}
|
||||
initialized = true;
|
||||
void* caps = nullptr;
|
||||
ngx_check(capabilities(&caps), "CUDA feature capabilities");
|
||||
if (!caps) throw std::runtime_error("Null CUDA capability parameters");
|
||||
const int available = ngx::int_value(caps, "SuperSampling.Available", -1);
|
||||
const int initResult = ngx::int_value(caps, "SuperSampling.FeatureInitResult", -1);
|
||||
const int needsDriver = ngx::int_value(caps, "SuperSampling.NeedsUpdatedDriver", -1);
|
||||
std::printf("D4R_CAPABILITIES available=%d feature_init=0x%08x needs_driver=%d\n", available, unsigned(initResult), needsDriver);
|
||||
std::fprintf(stderr, "d4r: NGX SR capabilities available=%d feature_init=0x%08x needs_driver=%d\n",
|
||||
available, unsigned(initResult), needsDriver);
|
||||
ngx_check(destroy(caps), "Destroy capability parameters");
|
||||
if (available != 1 || initResult != 1 || needsDriver != 0)
|
||||
throw std::runtime_error("CUDA DLSS is unavailable after capability initialization");
|
||||
std::printf("D4R_RUNTIME platform=Windows architecture=%s cpu_frame_copies=0 frame_age=0\n", d4r::diag::HipApi::target_arch());
|
||||
} catch (...) { cleanup(); throw; }
|
||||
}
|
||||
Runtime(const Runtime&) = delete;
|
||||
~Runtime() { cleanup(); }
|
||||
void current() { cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent(runtime)"); }
|
||||
void drain_async_queue() {
|
||||
ComPtr<ID3D12CommandQueue> queue;
|
||||
{ std::lock_guard<std::mutex> lock(async_mutex); queue = async_queue; }
|
||||
if (!queue) return;
|
||||
// Creation/free/import may synchronize the AMD device internally.
|
||||
// Drain queued D3D12 consumers before taking the CUDA mutex: workers
|
||||
// must remain able to satisfy any already-published output waits.
|
||||
ComPtr<ID3D12Fence> completion;
|
||||
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(completion.GetAddressOf())), "Reconfigure fence");
|
||||
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
|
||||
if (!event.value) throw std::runtime_error("Reconfigure event");
|
||||
dx(queue->Signal(completion.Get(), 1), "Reconfigure queue Signal");
|
||||
if (completion->GetCompletedValue() < 1) {
|
||||
dx(completion->SetEventOnCompletion(1, event.value), "Reconfigure completion event");
|
||||
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Reconfigure GPU timeout");
|
||||
}
|
||||
dx(device->GetDeviceRemovedReason(), "Reconfigure device status");
|
||||
}
|
||||
void cleanup() noexcept {
|
||||
if (context) (void)cuda.cuCtxSetCurrent(context);
|
||||
if (initialized) { (void)shutdown(); initialized = false; }
|
||||
timeline.reset();
|
||||
pixels.reset();
|
||||
if (context) {
|
||||
(void)cuda.cuCtxSynchronize(); (void)cuda.cuCtxSetCurrent(nullptr);
|
||||
(void)cuda.cuDevicePrimaryCtxRelease_v2(ordinal); context = nullptr;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class Feature {
|
||||
std::shared_ptr<Runtime> rt_;
|
||||
void* parameters_ = nullptr;
|
||||
void* handle_ = nullptr;
|
||||
CUdeviceptr scratch_ = 0;
|
||||
ComPtr<ID3D12CommandQueue> queue_;
|
||||
// A boundary drains both copies before returning. Reuse their allocators
|
||||
// only after that completion; no game allocator is retained here.
|
||||
ComPtr<ID3D12CommandAllocator> input_allocator_, output_allocator_;
|
||||
ComPtr<ID3D12GraphicsCommandList> input_list_, output_list_;
|
||||
struct CopySlot {
|
||||
ComPtr<ID3D12CommandAllocator> input_allocator, output_allocator;
|
||||
ComPtr<ID3D12GraphicsCommandList> input, output;
|
||||
std::unique_ptr<BoundaryTiming> timing;
|
||||
uint64_t done = 0;
|
||||
};
|
||||
struct AsyncFrame {
|
||||
void* parameters = d4r_ngx_parameters_create();
|
||||
std::vector<ComPtr<ID3D12Resource>> resources;
|
||||
SharedTimeline::Ticket ticket{};
|
||||
~AsyncFrame() { if (parameters) d4r_ngx_parameters_destroy(parameters); }
|
||||
};
|
||||
std::vector<std::unique_ptr<CopySlot>> copy_slots_;
|
||||
ComPtr<ID3D12Fence> copy_completion_;
|
||||
Handle copy_event_;
|
||||
uint64_t copy_value_ = 0;
|
||||
std::mutex jobs_mutex_;
|
||||
std::mutex submit_mutex_;
|
||||
std::condition_variable jobs_changed_;
|
||||
std::deque<std::shared_ptr<AsyncFrame>> jobs_;
|
||||
std::thread worker_;
|
||||
bool worker_stop_ = false, worker_active_ = false;
|
||||
std::string worker_failure_;
|
||||
struct Plane {
|
||||
ComPtr<ID3D12Resource> texture;
|
||||
D3D12_RESOURCE_DESC desc{};
|
||||
std::unique_ptr<SharedPlane> shared;
|
||||
std::unique_ptr<cuda::Image> image;
|
||||
std::unique_ptr<PixelAllocation> canonical;
|
||||
PixelSpec spec{};
|
||||
} planes_[5];
|
||||
void cleanup() noexcept {
|
||||
(void)rt_->cuda.cuCtxSetCurrent(rt_->context);
|
||||
(void)rt_->cuda.cuCtxSynchronize();
|
||||
if (handle_) { (void)rt_->release(handle_); handle_ = nullptr; }
|
||||
for (auto& plane : planes_) {
|
||||
plane.image.reset(); plane.canonical.reset(); plane.shared.reset(); plane.texture.Reset();
|
||||
}
|
||||
if (scratch_) { (void)rt_->cuda.cuMemFree_v2(scratch_); scratch_ = 0; }
|
||||
if (parameters_) { (void)rt_->destroy(parameters_); parameters_ = nullptr; }
|
||||
}
|
||||
public:
|
||||
unsigned width = 0, height = 0;
|
||||
explicit Feature(std::shared_ptr<Runtime> runtime, void* parameters) : rt_(std::move(runtime)) {
|
||||
if (std::getenv("D4R_ASYNC_INTEROP")) rt_->drain_async_queue();
|
||||
std::lock_guard<std::mutex> lock(rt_->mutex);
|
||||
rt_->current();
|
||||
try {
|
||||
ngx_check(rt_->allocate(¶meters_), "CUDA AllocateParameters");
|
||||
ngx::copy_create(parameters, parameters_);
|
||||
d4r_ngx_set_int(parameters_, "DLSS.Enable.Output.Subrects", ngx::int_value(parameters, "DLSS.Enable.Output.Subrects"));
|
||||
width = ngx::uint_value(parameters, "Width"); height = ngx::uint_value(parameters, "Height");
|
||||
std::printf("D4R_CREATE source=%ux%u->%ux%u cuda=%ux%u->%ux%u\n", width, height,
|
||||
ngx::uint_value(parameters, "OutWidth"), ngx::uint_value(parameters, "OutHeight"),
|
||||
ngx::uint_value(parameters_, "Width"), ngx::uint_value(parameters_, "Height"),
|
||||
ngx::uint_value(parameters_, "OutWidth"), ngx::uint_value(parameters_, "OutHeight"));
|
||||
if (!width || !height) throw std::runtime_error("Zero NGX input dimensions");
|
||||
size_t scratchBytes = 0;
|
||||
const unsigned scratchResult = rt_->getScratchBufferSize(1, parameters_, &scratchBytes);
|
||||
if (scratchResult != 1 || !scratchBytes) {
|
||||
std::printf("D4R_CREATE scratch_query=0x%08x fallback_bytes=67108864\n", scratchResult);
|
||||
scratchBytes = 64ull * 1024 * 1024;
|
||||
}
|
||||
std::printf("D4R_CREATE scratch_bytes=%zu\n", scratchBytes);
|
||||
rt_->cuda.check(rt_->cuda.cuMemAlloc_v2(&scratch_, scratchBytes), "NGX VRAM scratch");
|
||||
d4r_ngx_set_void(parameters_, "Scratch", reinterpret_cast<void*>(uintptr_t(scratch_)));
|
||||
d4r_ngx_set_ull(parameters_, "Scratch.SizeInBytes", scratchBytes);
|
||||
ngx_check(rt_->create(1, parameters_, &handle_), "CUDA CreateFeature");
|
||||
} catch (...) { cleanup(); throw; }
|
||||
}
|
||||
~Feature() {
|
||||
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_stop_ = true; }
|
||||
jobs_changed_.notify_all();
|
||||
if (worker_.joinable()) worker_.join();
|
||||
if (copy_completion_) {
|
||||
try { wait_copies(copy_value_); report_completed_copies(); }
|
||||
catch (const std::exception& e) { std::fprintf(stderr, "D4R_ASYNC_CLEANUP_FAILURE %s\n", e.what()); }
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(rt_->mutex); cleanup();
|
||||
}
|
||||
Feature(const Feature&) = delete;
|
||||
void drain() {
|
||||
if (!std::getenv("D4R_ASYNC_INTEROP")) return;
|
||||
wait_worker();
|
||||
if (copy_value_) wait_copies(copy_value_);
|
||||
report_completed_copies();
|
||||
rt_->drain_async_queue();
|
||||
}
|
||||
|
||||
// Explicit harness boundary: all earlier producers must already be queued.
|
||||
// The forthcoming command-list backend calls the same stages at queue submit.
|
||||
void evaluate_boundary(ID3D12CommandQueue* queue, void* parameters) {
|
||||
if (std::getenv("D4R_ASYNC_INTEROP")) { evaluate_async(queue, parameters); return; }
|
||||
const auto boundaryBegin = std::chrono::steady_clock::now();
|
||||
auto boundaryStage = boundaryBegin;
|
||||
const bool profile = std::getenv("D4R_PROFILE_STAGES") != nullptr;
|
||||
auto mark = [&](const char* name) {
|
||||
if (!profile) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
|
||||
std::chrono::duration<double, std::milli>(now - boundaryStage).count());
|
||||
boundaryStage = std::chrono::steady_clock::now();
|
||||
};
|
||||
commands::InternalScope internal;
|
||||
std::lock_guard<std::mutex> lock(rt_->mutex);
|
||||
if (!queue) throw std::runtime_error("Null D3D12 queue");
|
||||
if (queue_ && queue_.Get() != queue) throw std::runtime_error("A feature cannot switch queues without an explicit drain");
|
||||
queue_ = queue;
|
||||
ComPtr<ID3D12Device> queueDevice;
|
||||
dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "GetDevice(queue)");
|
||||
ComPtr<IUnknown> queueIdentity, runtimeIdentity;
|
||||
dx(queueDevice->QueryInterface(IID_PPV_ARGS(queueIdentity.GetAddressOf())), "Query queue device identity");
|
||||
dx(rt_->device->QueryInterface(IID_PPV_ARGS(runtimeIdentity.GetAddressOf())), "Query runtime device identity");
|
||||
if (queueIdentity.Get() != runtimeIdentity.Get()) throw std::runtime_error("Queue belongs to a different D3D12 device");
|
||||
rt_->current();
|
||||
prepare(parameters);
|
||||
mark("boundary_prepare");
|
||||
ComPtr<ID3D12CommandAllocator> inputAllocator, outputAllocator;
|
||||
ComPtr<ID3D12GraphicsCommandList> inputList, outputList;
|
||||
auto list = [&](ComPtr<ID3D12CommandAllocator>& allocator, ComPtr<ID3D12GraphicsCommandList>& commandList) {
|
||||
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator(interop)");
|
||||
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
|
||||
IID_PPV_ARGS(commandList.GetAddressOf())), "CreateCommandList(interop)");
|
||||
};
|
||||
const bool cached = !std::getenv("D4R_DISABLE_INTEROP_LIST_CACHE");
|
||||
if (cached) {
|
||||
if (!input_list_) {
|
||||
// Publish the pair only after both creations succeed.
|
||||
list(inputAllocator, inputList); list(outputAllocator, outputList);
|
||||
input_allocator_ = inputAllocator; output_allocator_ = outputAllocator;
|
||||
input_list_ = inputList; output_list_ = outputList;
|
||||
} else {
|
||||
dx(input_allocator_->Reset(), "Reset(input interop allocator)");
|
||||
dx(output_allocator_->Reset(), "Reset(output interop allocator)");
|
||||
dx(input_list_->Reset(input_allocator_.Get(), nullptr), "Reset(input interop list)");
|
||||
dx(output_list_->Reset(output_allocator_.Get(), nullptr), "Reset(output interop list)");
|
||||
}
|
||||
inputAllocator = input_allocator_; outputAllocator = output_allocator_;
|
||||
inputList = input_list_; outputList = output_list_;
|
||||
} else { list(inputAllocator, inputList); list(outputAllocator, outputList); }
|
||||
mark("boundary_copy_list_setup");
|
||||
record_inputs(inputList.Get(), parameters); dx(inputList->Close(), "Close(input copies)");
|
||||
ID3D12CommandList* before[] = {inputList.Get()}; queue->ExecuteCommandLists(1, before);
|
||||
mark("boundary_input_copy_submit");
|
||||
try {
|
||||
execute_cuda(queue);
|
||||
mark("boundary_cuda_transaction");
|
||||
record_output(outputList.Get(), parameters); dx(outputList->Close(), "Close(output copy)");
|
||||
ID3D12CommandList* after[] = {outputList.Get()}; queue->ExecuteCommandLists(1, after);
|
||||
mark("boundary_output_copy_submit");
|
||||
rt_->timeline->drain(queue);
|
||||
mark("boundary_output_copy_drain");
|
||||
} catch (...) {
|
||||
// Keep all recorded resources alive until submitted work is complete.
|
||||
rt_->timeline->drain(queue); throw;
|
||||
}
|
||||
}
|
||||
private:
|
||||
void report_completed_copies() {
|
||||
if (!copy_completion_ || !queue_) return;
|
||||
const auto completed = copy_completion_->GetCompletedValue();
|
||||
if (completed == UINT64_MAX) throw std::runtime_error("Device removed during boundary timing");
|
||||
for (auto& slot : copy_slots_) if (slot->timing && slot->done && slot->done <= completed)
|
||||
slot->timing->report(queue_.Get(), this);
|
||||
}
|
||||
void wait_copies(uint64_t value) {
|
||||
if (copy_completion_->GetCompletedValue() >= value) return;
|
||||
dx(copy_completion_->SetEventOnCompletion(value, copy_event_.value), "Async copy completion event");
|
||||
if (WaitForSingleObject(copy_event_.value, 30000) != WAIT_OBJECT_0)
|
||||
throw std::runtime_error("Async interop copy completion timeout");
|
||||
}
|
||||
void wait_worker() {
|
||||
std::unique_lock<std::mutex> lock(jobs_mutex_);
|
||||
jobs_changed_.wait(lock, [&] { return jobs_.empty() && !worker_active_; });
|
||||
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_);
|
||||
}
|
||||
CopySlot& copy_slot() {
|
||||
report_completed_copies();
|
||||
if (!copy_completion_) {
|
||||
dx(rt_->device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(copy_completion_.GetAddressOf())), "Async copy fence");
|
||||
copy_event_.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
|
||||
if (!copy_event_.value) throw std::runtime_error("Async copy event");
|
||||
}
|
||||
CopySlot* selected = nullptr;
|
||||
const auto completed = copy_completion_->GetCompletedValue();
|
||||
for (auto& slot : copy_slots_) if (slot->done <= completed) { selected = slot.get(); break; }
|
||||
if (!selected && copy_slots_.size() == 3) {
|
||||
selected = copy_slots_.front().get();
|
||||
for (auto& slot : copy_slots_) if (slot->done < selected->done) selected = slot.get();
|
||||
// Bounded ownership; this wait is outside the runtime mutex.
|
||||
wait_copies(selected->done);
|
||||
report_completed_copies();
|
||||
}
|
||||
if (selected) {
|
||||
// The fence can advance between report_completed_copies() and
|
||||
// slot selection. Collect that now-complete ticket before Reset
|
||||
// and before submitted() overwrites its timing identity.
|
||||
if (selected->timing) selected->timing->report(queue_.Get(), this);
|
||||
dx(selected->input_allocator->Reset(), "Async input allocator Reset");
|
||||
dx(selected->output_allocator->Reset(), "Async output allocator Reset");
|
||||
dx(selected->input->Reset(selected->input_allocator.Get(), nullptr), "Async input list Reset");
|
||||
dx(selected->output->Reset(selected->output_allocator.Get(), nullptr), "Async output list Reset");
|
||||
return *selected;
|
||||
}
|
||||
auto slot = std::make_unique<CopySlot>();
|
||||
auto create = [&](auto& allocator, auto& list) {
|
||||
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Async copy allocator");
|
||||
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Async copy list");
|
||||
};
|
||||
create(slot->input_allocator, slot->input); create(slot->output_allocator, slot->output);
|
||||
if (std::getenv("D4R_PROFILE_GPU_BOUNDARY")) slot->timing = std::make_unique<BoundaryTiming>(rt_->device.Get());
|
||||
copy_slots_.push_back(std::move(slot)); return *copy_slots_.back();
|
||||
}
|
||||
bool resize_needed(void* parameters) {
|
||||
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
|
||||
for (unsigned i = 0; i < 5; ++i) {
|
||||
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
|
||||
if (!texture) {
|
||||
if (i < 4) throw std::runtime_error(std::string("Missing async resource ") + names[i]);
|
||||
continue;
|
||||
}
|
||||
const auto desc = resource_desc(texture);
|
||||
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
|
||||
throw std::runtime_error("Async interop requires single-sample 2D textures");
|
||||
if (!planes_[i].shared || planes_[i].desc.Width != desc.Width || planes_[i].desc.Height != desc.Height || planes_[i].desc.Format != desc.Format) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
void evaluate_async(ID3D12CommandQueue* queue, void* parameters) {
|
||||
commands::InternalScope internal;
|
||||
std::lock_guard<std::mutex> submitted(submit_mutex_);
|
||||
if (!queue || (queue_ && queue_.Get() != queue)) throw std::runtime_error("Async interop requires one feature queue");
|
||||
ComPtr<ID3D12Device> queueDevice; dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "Async queue device");
|
||||
ComPtr<IUnknown> a, b;
|
||||
dx(queueDevice->QueryInterface(IID_PPV_ARGS(a.GetAddressOf())), "Async queue identity");
|
||||
dx(rt_->device->QueryInterface(IID_PPV_ARGS(b.GetAddressOf())), "Async runtime identity");
|
||||
if (a.Get() != b.Get()) throw std::runtime_error("Async queue device mismatch");
|
||||
{ std::lock_guard<std::mutex> lock(jobs_mutex_);
|
||||
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
|
||||
{ std::lock_guard<std::mutex> lock(rt_->async_mutex);
|
||||
if (!rt_->async_failure.empty()) throw std::runtime_error(rt_->async_failure);
|
||||
if (rt_->async_queue && rt_->async_queue.Get() != queue) throw std::runtime_error("Async runtime features must share one D3D12 queue");
|
||||
rt_->async_queue = queue; }
|
||||
auto& slot = copy_slot();
|
||||
if (resize_needed(parameters)) {
|
||||
wait_worker(); if (copy_value_) wait_copies(copy_value_);
|
||||
rt_->drain_async_queue();
|
||||
std::lock_guard<std::mutex> lock(rt_->mutex);
|
||||
rt_->current(); prepare(parameters);
|
||||
}
|
||||
// Steady-state recording touches immutable allocation metadata and the
|
||||
// caller's resource snapshot. It does not contend with CUDA or mutate
|
||||
// the worker's NGX parameters / texture references.
|
||||
queue_ = queue;
|
||||
auto frame = std::make_shared<AsyncFrame>();
|
||||
if (!frame->parameters) throw std::runtime_error("Async frame parameters allocation");
|
||||
ngx::copy_create(parameters, frame->parameters); ngx::copy_frame(parameters, frame->parameters);
|
||||
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
|
||||
for (unsigned i = 0; i < 5; ++i) {
|
||||
ID3D12Resource* resource = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &resource);
|
||||
frame->resources.emplace_back(resource);
|
||||
d4r_ngx_set_d3d12_resource(frame->parameters, names[i], resource);
|
||||
}
|
||||
if (slot.timing) slot.timing->input_begin(slot.input.Get());
|
||||
record_inputs(slot.input.Get(), parameters);
|
||||
if (slot.timing) slot.timing->input_end(slot.input.Get());
|
||||
dx(slot.input->Close(), "Async input Close");
|
||||
if (slot.timing) slot.timing->output_begin(slot.output.Get());
|
||||
record_output(slot.output.Get(), parameters);
|
||||
if (slot.timing) slot.timing->output_end(slot.output.Get());
|
||||
dx(slot.output->Close(), "Async output Close");
|
||||
if (!worker_.joinable()) worker_ = std::thread([this] { work(); });
|
||||
ID3D12CommandList* input[] = {slot.input.Get()}, *output[] = {slot.output.Get()};
|
||||
// Allocate the job before any GPU wait is published. Worker cannot
|
||||
// acquire the runtime mutex until every queue dependency is enqueued.
|
||||
std::lock_guard<std::mutex> pending(jobs_mutex_);
|
||||
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
|
||||
frame->ticket = rt_->timeline->reserve(); rt_->async_tickets.push_back(frame->ticket.output); }
|
||||
try { jobs_.push_back(frame); }
|
||||
catch (...) {
|
||||
std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_back(); throw;
|
||||
}
|
||||
try {
|
||||
queue->ExecuteCommandLists(1, input);
|
||||
rt_->timeline->enqueue(queue, frame->ticket);
|
||||
queue->ExecuteCommandLists(1, output);
|
||||
slot.done = ++copy_value_;
|
||||
dx(queue->Signal(copy_completion_.Get(), slot.done), "Async output-copy retirement Signal");
|
||||
if (slot.timing) slot.timing->submitted(frame->ticket.output);
|
||||
} catch (const std::exception& e) {
|
||||
jobs_.pop_back(); worker_failure_ = e.what();
|
||||
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
|
||||
rt_->async_tickets.pop_back(); rt_->async_failure = e.what(); }
|
||||
(void)rt_->timeline->fence->Signal(frame->ticket.output);
|
||||
jobs_changed_.notify_all(); rt_->async_changed.notify_all(); throw;
|
||||
}
|
||||
jobs_changed_.notify_one();
|
||||
std::printf("D4R_ASYNC_SUBMIT ticket=%llu copy_slots=%zu cpu_frame_copies=0 frame_age=0\n",
|
||||
(unsigned long long)frame->ticket.output, copy_slots_.size());
|
||||
}
|
||||
void work() noexcept {
|
||||
commands::InternalScope internal;
|
||||
for (;;) {
|
||||
std::shared_ptr<AsyncFrame> frame;
|
||||
{ std::unique_lock<std::mutex> lock(jobs_mutex_);
|
||||
jobs_changed_.wait(lock, [&] { return worker_stop_ || !jobs_.empty(); });
|
||||
if (jobs_.empty()) return;
|
||||
frame = std::move(jobs_.front()); jobs_.pop_front(); worker_active_ = true; }
|
||||
{ std::unique_lock<std::mutex> order(rt_->async_mutex);
|
||||
rt_->async_changed.wait(order, [&] { return !rt_->async_tickets.empty() && rt_->async_tickets.front() == frame->ticket.output; }); }
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(rt_->mutex);
|
||||
{ std::lock_guard<std::mutex> pending(jobs_mutex_);
|
||||
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
|
||||
rt_->current(); prepare(frame->parameters); execute_cuda(nullptr, &frame->ticket);
|
||||
} catch (const std::exception& e) {
|
||||
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_failure_ = e.what(); }
|
||||
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_failure = e.what(); }
|
||||
std::fprintf(stderr, "D4R_ASYNC_INTEROP_FAILURE %s\n", e.what());
|
||||
// Poison the feature and release the queue for error handling;
|
||||
// subsequent evaluates fail explicitly, never reuse an old frame.
|
||||
(void)rt_->timeline->fence->Signal(frame->ticket.output);
|
||||
}
|
||||
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_front(); }
|
||||
rt_->async_changed.notify_all();
|
||||
frame.reset();
|
||||
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_active_ = false; }
|
||||
jobs_changed_.notify_all();
|
||||
}
|
||||
}
|
||||
void prepare(void* parameters) {
|
||||
static const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
|
||||
for (unsigned i = 0; i < 5; ++i) {
|
||||
ID3D12Resource* texture = nullptr;
|
||||
(void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
|
||||
if (!texture) {
|
||||
if (i < 4) throw std::runtime_error(std::string("Missing resource ") + names[i]);
|
||||
planes_[i].texture.Reset(); d4r_ngx_set_void(parameters_, names[i], nullptr); continue;
|
||||
}
|
||||
auto desc = resource_desc(texture);
|
||||
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
|
||||
throw std::runtime_error("Interop baseline requires single-sample 2D textures");
|
||||
const bool half = i == 0 || i == 2 || i == 3;
|
||||
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
|
||||
if (desc.Width > UINT32_MAX) throw std::runtime_error("Texture width exceeds GPU conversion ABI");
|
||||
const PixelSpec spec = pixel_spec(i, desc.Format);
|
||||
if (!pixel_supported(i, spec.storage)) throw std::runtime_error(std::string("Unsupported ") + names[i] + " DXGI format=" + std::to_string(desc.Format));
|
||||
auto& plane = planes_[i];
|
||||
if (!plane.shared || plane.desc.Width != desc.Width || plane.desc.Height != desc.Height || plane.desc.Format != desc.Format) {
|
||||
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Synchronize(resize interop)");
|
||||
plane.image.reset(); plane.canonical.reset(); plane.shared.reset();
|
||||
plane.shared = std::make_unique<SharedPlane>(rt_->external, rt_->device.Get(), desc);
|
||||
if (!spec.direct) {
|
||||
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
|
||||
plane.canonical = std::make_unique<PixelAllocation>(rt_->hip, unsigned(desc.Width), desc.Height, channels * (half ? 2 : 4));
|
||||
}
|
||||
plane.image = std::make_unique<cuda::Image>(rt_->images, unsigned(desc.Width), desc.Height, half ? 16 : 32, channels, i == 3, i == 0 ? 1 : 0);
|
||||
plane.desc = desc; plane.spec = spec;
|
||||
std::printf("D4R_FORMAT plane=%s dxgi=%u canonical_channels=%u canonical_bits=%u gpu_conversion=%u cpu_copy=0\n",
|
||||
names[i], unsigned(desc.Format), channels, half ? 16u : 32u, !spec.direct);
|
||||
}
|
||||
plane.texture = texture;
|
||||
d4r_ngx_set_void(parameters_, names[i], &plane.image->object);
|
||||
}
|
||||
ngx::copy_frame(parameters, parameters_);
|
||||
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Width")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Width", width);
|
||||
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Height")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Height", height);
|
||||
for (const char* name : {"TransparencyMask", "DLSS.Input.Bias.Current.Color.Mask"}) d4r_ngx_set_void(parameters_, name, nullptr);
|
||||
}
|
||||
void record_inputs(ID3D12GraphicsCommandList* list, void* parameters) {
|
||||
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
|
||||
for (unsigned i : {0u, 1u, 2u, 4u}) {
|
||||
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
|
||||
if (!texture) continue;
|
||||
static const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
|
||||
planes_[i].shared->copy_input(list, texture, ngx_resource_access(parameters, states[i], D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE));
|
||||
}
|
||||
}
|
||||
void execute_cuda(ID3D12CommandQueue* queue, const SharedTimeline::Ticket* ticket = nullptr) {
|
||||
const auto begin = std::chrono::steady_clock::now();
|
||||
auto stage = begin;
|
||||
const bool profileStages = std::getenv("D4R_PROFILE_STAGES") != nullptr;
|
||||
auto mark = [&](const char* name) {
|
||||
if (!profileStages) return;
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
|
||||
std::chrono::duration<double, std::milli>(now - stage).count());
|
||||
stage = std::chrono::steady_clock::now();
|
||||
};
|
||||
if (ticket) rt_->timeline->wait_ready(ticket->input); else rt_->timeline->wait_input(queue);
|
||||
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input producer completion");
|
||||
mark("input_fence_wait");
|
||||
const bool batchInputs = std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr;
|
||||
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
|
||||
auto& plane = planes_[i];
|
||||
void* data = plane.shared->mapped; uint64_t pitch = plane.shared->footprint.Footprint.RowPitch;
|
||||
if (plane.canonical) {
|
||||
if (batchInputs) rt_->pixels->convert_deferred(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
|
||||
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
|
||||
else rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
|
||||
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
|
||||
data = plane.canonical->data; pitch = plane.canonical->pitch;
|
||||
}
|
||||
plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch, batchInputs);
|
||||
}
|
||||
// HIP conversion and CUDA array copies are ordered on the shared
|
||||
// legacy default stream. Retain an all-stream completion before NGX,
|
||||
// including any internal nonblocking streams it might choose to use.
|
||||
if (batchInputs) rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input batch completion");
|
||||
mark("input_conversion_array_upload");
|
||||
if (std::getenv("D4R_INTEROP_VERIFY")) {
|
||||
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
|
||||
uint32_t sample[8]{};
|
||||
rt_->hip.check(rt_->hip.hipMemcpy(sample, planes_[i].shared->mapped,
|
||||
std::min<size_t>(sizeof(sample), planes_[i].shared->bytes), hipMemcpyDeviceToHost), "DIAGNOSTIC external-memory sample");
|
||||
std::printf("D4R_VRAM_VERIFY plane=%u words=%08x,%08x,%08x,%08x\n", i, sample[0], sample[1], sample[2], sample[3]);
|
||||
const unsigned bytesPerPixel = i == 0 ? 8 : i == 2 ? 4 : 4;
|
||||
std::vector<uint8_t> array(size_t(planes_[i].desc.Width) * planes_[i].desc.Height * bytesPerPixel);
|
||||
planes_[i].image->download(array.data());
|
||||
uint32_t first = 0; std::memcpy(&first, array.data(), 4);
|
||||
std::printf("D4R_ARRAY_VERIFY plane=%u first=%08x\n", i, first);
|
||||
}
|
||||
}
|
||||
ngx_check(rt_->evaluate(handle_, parameters_, nullptr), "CUDA EvaluateFeature");
|
||||
mark("ngx_evaluate_host");
|
||||
// NGX can use internal nonblocking streams. The default stream alone
|
||||
// does not establish completion of those streams.
|
||||
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "NGX all-stream completion");
|
||||
mark("ngx_all_stream_completion");
|
||||
auto& output = planes_[3];
|
||||
output.image->download_device(reinterpret_cast<uintptr_t>(output.canonical ? output.canonical->data : output.shared->mapped),
|
||||
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch);
|
||||
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "VRAM output completion");
|
||||
mark("output_array_download");
|
||||
if (std::getenv("D4R_VALIDATE_OUTPUT")) {
|
||||
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
|
||||
const auto counts = rt_->pixels->validate(output.canonical ? output.canonical->data : output.shared->mapped,
|
||||
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height);
|
||||
std::printf("D4R_OUTPUT_VALIDATION elements=%llu nan=%u inf=%u diagnostics_cpu_bytes=8\n", output.desc.Width * output.desc.Height * 4, counts[0], counts[1]);
|
||||
if (counts[0] || counts[1]) throw std::runtime_error("DLSS output contains NaN/Inf");
|
||||
}
|
||||
mark("output_diagnostic");
|
||||
if (output.canonical) rt_->pixels->convert(true, output.canonical->data, output.canonical->pitch, output.shared->mapped,
|
||||
output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height, output.spec.storage, 3);
|
||||
if (ticket) rt_->timeline->signal_ready(ticket->output); else rt_->timeline->signal_output(queue);
|
||||
mark("output_conversion_fence_signal");
|
||||
const double frameMs = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count();
|
||||
std::printf("D4R_FRAME cpu_frame_copies=%u frame_age=0 interop_ngx_ms=%.3f\n", std::getenv("D4R_INTEROP_VERIFY") ? 1u : 0u, frameMs);
|
||||
static std::atomic<bool> firstFrameLogged{false};
|
||||
if (!firstFrameLogged.exchange(true))
|
||||
std::fprintf(stderr, "d4r: first DLSS frame complete interop_ngx_ms=%.3f output=%ux%u\n",
|
||||
frameMs, unsigned(output.desc.Width), output.desc.Height);
|
||||
}
|
||||
void record_output(ID3D12GraphicsCommandList* list, void* parameters) {
|
||||
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, "Output", &texture);
|
||||
if (!texture) throw std::runtime_error("Missing output resource");
|
||||
planes_[3].shared->copy_output(list, texture, ngx_resource_access(parameters, "D4R.Output.State", D3D12_RESOURCE_STATE_UNORDERED_ACCESS));
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user