ayo
ac4ee6c225
Add a native D3D12 FSR Ray Regeneration route for gfx1200/gfx1201, using the game's device and queue with embedded HLSL guide preparation and output composition. Preserve the vkd3d compatibility route for RDNA3 and expose explicit route overrides and diagnostics. Improve Windows runtime compatibility: - Retry rejected application-ID initialization with a project identity. - Defer expensive RR backend creation until feature creation. - Recover from transient submission failures without disabling later frames. - Add shared-buffer export fallbacks through explicit heaps and a private device on the same adapter. - Safely evict old RR descriptor sets when input views change. - Treat requested RR presets as hints when a configured model is available. - Expand SR/RR input formats and handle diffuse/specular hit distances, packed normals, depth-stencil resources, and unused FSR guides. Reduce SR overhead by batching output downloads and format conversion under one completion wait. Optimize strict M enc2/tube attention with bounded FP32 accumulation and the original FP64 fallback for ambiguous rounding cases. Extend RDNA4 tooling with gfx1200/gfx1201 native FP8 RR model builds, capability checks, architecture-aware deployment, and experimental K/M kernel paths. Require matching model and runtime arithmetic families. Add optional embedded FSR shader optimizations keyed by shader hash and exact original SPIR-V contents. Preserve reflected metadata and record optimizer provenance, artifact hashes, and cache identity. Simplify Windows installation with automatic backend dependency downloads, download progress and ETA, backed-up absolute OptiScaler library paths, and recovery from interrupted RR model builds. Add build-kernels.bat and a kernels-only build matrix for all eleven registered RDNA targets, with code-object target validation and SHA256 manifests. Expand conversion probes, numerical stress tests, temporal benchmark tooling, installer/deployment tests, and setup documentation. Document synthetic regression and timing results, experimental build options, and remaining RDNA4 hardware validation requirements.
657 lines
39 KiB
C++
657 lines
39 KiB
C++
#include "d3d12_command_hooks.h"
|
|
#include "d3d12_newer_commands.h"
|
|
#include "d3d12_retirement.h"
|
|
#include <MinHook.h>
|
|
#include <atomic>
|
|
#include <memory>
|
|
#include <mutex>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
#include <chrono>
|
|
|
|
namespace d4r::win::commands {
|
|
namespace {
|
|
thread_local bool internal = false;
|
|
std::mutex map_mutex, installation_mutex, error_mutex;
|
|
std::string last_error;
|
|
bool installed = false;
|
|
using Clock = std::chrono::steady_clock;
|
|
bool profile_commands() { static const bool enabled = std::getenv("D4R_PROFILE_STAGES") != nullptr; return enabled; }
|
|
double milliseconds(Clock::time_point start) { return std::chrono::duration<double, std::milli>(Clock::now() - start).count(); }
|
|
struct Event { UINT metadata; std::vector<uint8_t> data; };
|
|
using State = std::function<void(ID3D12GraphicsCommandList*)>;
|
|
struct TrackedResource {
|
|
// D3D12 command lists do not retain resource references. The caller keeps
|
|
// them alive through submission/completion. An extra tracking reference
|
|
// would keep swap-chain buffers alive and make ResizeBuffers fail.
|
|
ResourceAccess state;
|
|
};
|
|
using ResourceStates = std::unordered_map<ID3D12Resource*, TrackedResource>;
|
|
static constexpr GUID submitted_access_tag = {0x741b92df,0x9b13,0x4fc9,{0xb0,0x02,0x2c,0xe3,0x5a,0x68,0x92,0x41}};
|
|
void publish_states(const ResourceStates& states) {
|
|
for (const auto& entry : states) {
|
|
// Buffers/simultaneous-access textures have automatic legacy state
|
|
// decay at ExecuteCommandLists completion. Only ordinary textures
|
|
// retain these explicit transitions across submission boundaries.
|
|
const auto desc = resource_desc(entry.first);
|
|
if (desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER || (desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS)) continue;
|
|
dx(entry.first->SetPrivateData(submitted_access_tag, sizeof(ResourceAccess), &entry.second.state), "Publish submitted texture access");
|
|
}
|
|
}
|
|
struct Segment {
|
|
// The application's original list is a weak pointer; holding it in its
|
|
// own routing map would prevent final Release and leak the entire feature.
|
|
ID3D12GraphicsCommandList* list = nullptr;
|
|
ComPtr<ID3D12GraphicsCommandList> owner;
|
|
ComPtr<ID3D12CommandAllocator> allocator;
|
|
Boundary boundary;
|
|
ResourceStates resource_states;
|
|
};
|
|
struct Recording {
|
|
std::recursive_mutex mutex;
|
|
ID3D12GraphicsCommandList* original;
|
|
ComPtr<ID3D12GraphicsCommandList> current;
|
|
ComPtr<ID3D12CommandAllocator> current_allocator;
|
|
std::vector<Segment> segments;
|
|
std::vector<State> state;
|
|
std::vector<Event> events;
|
|
ResourceStates resource_states;
|
|
unsigned active_queries = 0;
|
|
bool render_pass = false, predication = false, indirect_state_unknown = false, closed = false;
|
|
explicit Recording(ID3D12GraphicsCommandList* list) : original(list) {}
|
|
};
|
|
std::unordered_map<ID3D12GraphicsCommandList*, std::shared_ptr<Recording>> recordings;
|
|
std::atomic<uint64_t> recording_generation{0};
|
|
struct RecordingCache {
|
|
ID3D12GraphicsCommandList* list = nullptr;
|
|
uint64_t generation = 0;
|
|
// A thread's last lookup must not retain suffix lists, allocators or game
|
|
// resources after Reset/final Release. Address reuse also needs a new
|
|
// generation, even if the old Recording is alive in a submit callback.
|
|
std::weak_ptr<Recording> value;
|
|
};
|
|
thread_local RecordingCache recording_cache;
|
|
std::shared_ptr<Recording> get_recording(ID3D12GraphicsCommandList* self, bool create = true) {
|
|
if (internal) return {};
|
|
const auto generation = recording_generation.load(std::memory_order_acquire);
|
|
if (recording_cache.list == self && recording_cache.generation == generation) {
|
|
if (auto value = recording_cache.value.lock()) return value;
|
|
}
|
|
std::lock_guard<std::mutex> lock(map_mutex);
|
|
auto found = recordings.find(self);
|
|
std::shared_ptr<Recording> value;
|
|
if (found != recordings.end()) value = found->second;
|
|
else {
|
|
if (!create) return {};
|
|
value = std::make_shared<Recording>(self);
|
|
recordings.emplace(self, value);
|
|
recording_generation.fetch_add(1, std::memory_order_release);
|
|
}
|
|
recording_cache.list = self;
|
|
recording_cache.value = value;
|
|
recording_cache.generation = recording_generation.load(std::memory_order_relaxed);
|
|
return value;
|
|
}
|
|
void forget(ID3D12GraphicsCommandList* self) {
|
|
std::shared_ptr<Recording> released;
|
|
{ std::lock_guard<std::mutex> lock(map_mutex);
|
|
auto it = recordings.find(self);
|
|
if (it != recordings.end()) {
|
|
released = std::move(it->second); recordings.erase(it);
|
|
recording_generation.fetch_add(1, std::memory_order_release);
|
|
} }
|
|
// Release retained COM objects without holding the map mutex.
|
|
}
|
|
struct HookSample {
|
|
uint32_t random = GetCurrentThreadId() | 1u;
|
|
uint64_t calls = 0, samples = 0, marked = 0;
|
|
double access_ms = 0, capture_ms = 0, driver_ms = 0;
|
|
Clock::time_point since = Clock::now();
|
|
bool choose() {
|
|
++calls; random ^= random << 13; random ^= random >> 17; random ^= random << 5;
|
|
return !(random & 63u);
|
|
}
|
|
void flush() {
|
|
const double elapsed = milliseconds(since);
|
|
if (elapsed < 2000) return;
|
|
std::printf("D4R_COMMAND_HOOK_SAMPLE thread=%lu period_ms=%.3f calls=%llu samples=%llu marked=%llu access_sum_ms=%.6f capture_sum_ms=%.6f driver_sum_ms=%.6f sampling=64\n",
|
|
GetCurrentThreadId(), elapsed, (unsigned long long)calls, (unsigned long long)samples,
|
|
(unsigned long long)marked, access_ms, capture_ms, driver_ms);
|
|
calls = samples = marked = 0; access_ms = capture_ms = driver_ms = 0; since = Clock::now();
|
|
}
|
|
};
|
|
thread_local HookSample hook_sample;
|
|
bool sample_hooks() { static const bool enabled = std::getenv("D4R_PROFILE_COMMAND_HOOKS") != nullptr; return enabled; }
|
|
struct Access {
|
|
std::shared_ptr<Recording> recording;
|
|
std::unique_lock<std::recursive_mutex> lock;
|
|
ID3D12GraphicsCommandList* self;
|
|
bool sampled = false;
|
|
Clock::time_point begin{}, acquired{}, driver{};
|
|
explicit Access(ID3D12GraphicsCommandList* list) : self(list) {
|
|
sampled = !internal && sample_hooks() && hook_sample.choose();
|
|
if (sampled) begin = Clock::now();
|
|
recording = get_recording(list);
|
|
if (recording) lock = std::unique_lock<std::recursive_mutex>(recording->mutex);
|
|
if (sampled) acquired = Clock::now();
|
|
}
|
|
void driver_call() {
|
|
if (sampled) driver = Clock::now();
|
|
}
|
|
~Access() {
|
|
if (!sampled) return;
|
|
++hook_sample.samples;
|
|
hook_sample.access_ms += std::chrono::duration<double, std::milli>(acquired - begin).count();
|
|
if (driver != Clock::time_point{}) {
|
|
++hook_sample.marked;
|
|
hook_sample.capture_ms += std::chrono::duration<double, std::milli>(driver - acquired).count();
|
|
hook_sample.driver_ms += milliseconds(driver);
|
|
}
|
|
hook_sample.flush();
|
|
}
|
|
ID3D12GraphicsCommandList* target() const {
|
|
return recording && recording->current ? recording->current.Get() : self;
|
|
}
|
|
};
|
|
template<class T> ComPtr<T> retain(T* pointer) { return ComPtr<T>(pointer); }
|
|
template<class T> std::vector<T> copy_values(const T* pointer, size_t count) {
|
|
return pointer ? std::vector<T>(pointer, pointer + count) : std::vector<T>();
|
|
}
|
|
template<class T> T* values_pointer(std::vector<T>& values) { return values.empty() ? nullptr : values.data(); }
|
|
template<class T> struct RetainedArray { std::vector<ComPtr<T>> owners; std::vector<T*> raw; };
|
|
template<class T> RetainedArray<T> retain_values(T* const* pointer, size_t count) {
|
|
RetainedArray<T> result;
|
|
for (size_t i = 0; pointer && i < count; ++i) { result.owners.emplace_back(pointer[i]); result.raw.push_back(pointer[i]); }
|
|
return result;
|
|
}
|
|
struct Hook { void* target; void* replacement; };
|
|
std::vector<Hook> attached;
|
|
void attach_method(void* target, void* replacement, void** original, const char* name) {
|
|
for (const auto& hook : attached) if (hook.target == target)
|
|
throw std::runtime_error(std::string("Shared D3D12 implementation needs an explicit alias adapter: ") + name);
|
|
const auto result = MH_CreateHook(target, replacement, original);
|
|
if (result != MH_OK) throw std::runtime_error(std::string("Cannot intercept D3D12 ") + name + ": " + MH_StatusToString(result));
|
|
attached.push_back({target, replacement});
|
|
std::printf("D4R_D3D12_HOOK method=%s\n", name);
|
|
}
|
|
|
|
#include "d3d12_command_hooks.generated.h"
|
|
|
|
static constexpr GUID signature_description_tag = {0xbdda6148,0x46f0,0x4d82,{0xa1,0x90,0x08,0x47,0x7e,0xda,0x32,0x12}};
|
|
using SignatureFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12Device*, const D3D12_COMMAND_SIGNATURE_DESC*, ID3D12RootSignature*, REFIID, void**);
|
|
SignatureFn original_signature = nullptr;
|
|
using CreateListFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12Device*, UINT, D3D12_COMMAND_LIST_TYPE,
|
|
ID3D12CommandAllocator*, ID3D12PipelineState*, REFIID, void**);
|
|
CreateListFn original_create_list = nullptr;
|
|
HRESULT STDMETHODCALLTYPE hook_create_list(ID3D12Device* device, UINT node, D3D12_COMMAND_LIST_TYPE type,
|
|
ID3D12CommandAllocator* allocator, ID3D12PipelineState* initial, REFIID iid, void** out) {
|
|
const auto result = original_create_list(device, node, type, allocator, initial, iid, out);
|
|
if (!internal && SUCCEEDED(result) && out && *out) try {
|
|
ComPtr<ID3D12GraphicsCommandList> list;
|
|
if (SUCCEEDED(static_cast<IUnknown*>(*out)->QueryInterface(IID_PPV_ARGS(list.GetAddressOf())))) {
|
|
auto recording = get_recording(list.Get());
|
|
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
|
|
auto pipeline = retain(initial);
|
|
if (pipeline) recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) {
|
|
original_SetPipelineState(target, pipeline.Get());
|
|
});
|
|
}
|
|
} catch (const std::exception& failure) {
|
|
// A public Create must retain its result even if tracking runs out of
|
|
// memory. A later split must fail explicitly instead of replaying an
|
|
// incomplete initial state.
|
|
{ std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
|
|
std::fprintf(stderr, "D4R_CREATE_LIST_TRACK_FAILURE %s\n", failure.what());
|
|
}
|
|
return result;
|
|
}
|
|
HRESULT STDMETHODCALLTYPE hook_signature(ID3D12Device* device, const D3D12_COMMAND_SIGNATURE_DESC* desc, ID3D12RootSignature* root, REFIID iid, void** out) {
|
|
const auto result = original_signature(device, desc, root, iid, out);
|
|
if (SUCCEEDED(result) && out && *out && desc && desc->NumArgumentDescs <= 4096) try {
|
|
ComPtr<ID3D12CommandSignature> signature;
|
|
if (SUCCEEDED(static_cast<IUnknown*>(*out)->QueryInterface(IID_PPV_ARGS(signature.GetAddressOf())))) {
|
|
std::vector<uint8_t> value(sizeof(UINT) + desc->NumArgumentDescs * sizeof(D3D12_INDIRECT_ARGUMENT_DESC));
|
|
std::memcpy(value.data(), &desc->NumArgumentDescs, sizeof(UINT));
|
|
if (desc->NumArgumentDescs)
|
|
std::memcpy(value.data() + sizeof(UINT), desc->pArgumentDescs, value.size() - sizeof(UINT));
|
|
const auto tagged = signature->SetPrivateData(signature_description_tag, UINT(value.size()), value.data());
|
|
if (FAILED(tagged)) std::fprintf(stderr, "D4R_SIGNATURE_TRACK_FAILURE result=0x%08x\n", unsigned(tagged));
|
|
else std::fprintf(stderr, "D4R_COMMAND_SIGNATURE arguments=%u tracked=1\n", desc->NumArgumentDescs);
|
|
}
|
|
} catch (const std::exception& failure) {
|
|
// Preserve the successful public API call. An untracked signature is
|
|
// rejected at a later split instead of unwinding into the application.
|
|
std::fprintf(stderr, "D4R_SIGNATURE_TRACK_FAILURE %s\n", failure.what());
|
|
}
|
|
return result;
|
|
}
|
|
void STDMETHODCALLTYPE hook_SetProgram(ID3D12GraphicsCommandList* self, const NewProgramDescription* desc) {
|
|
Access access(self);
|
|
if (access.recording && desc) {
|
|
auto saved = *desc;
|
|
// INITIALIZE changes backing memory. Rebinding an already initialized
|
|
// work graph in the suffix must preserve its prefix's GPU state.
|
|
if (saved.type == 5) saved.work_graph.flags &= ~1u;
|
|
access.recording->state.emplace_back([saved](ID3D12GraphicsCommandList* target) { original_SetProgram(target, &saved); });
|
|
}
|
|
original_SetProgram(access.target(), desc);
|
|
}
|
|
|
|
using ReleaseFn = ULONG(STDMETHODCALLTYPE*)(IUnknown*);
|
|
ReleaseFn original_release = nullptr;
|
|
ULONG STDMETHODCALLTYPE hook_release(IUnknown* self) {
|
|
const ULONG count = original_release(self);
|
|
if (!count) forget(reinterpret_cast<ID3D12GraphicsCommandList*>(self));
|
|
return count;
|
|
}
|
|
HRESULT STDMETHODCALLTYPE hook_Close(ID3D12GraphicsCommandList* self) {
|
|
Access access(self);
|
|
const auto result = original_Close(access.target());
|
|
if (SUCCEEDED(result) && access.recording) access.recording->closed = true;
|
|
return result;
|
|
}
|
|
HRESULT STDMETHODCALLTYPE hook_Reset(ID3D12GraphicsCommandList* self, ID3D12CommandAllocator* allocator, ID3D12PipelineState* initial_state) {
|
|
if (internal) return original_Reset(self, allocator, initial_state);
|
|
forget(self);
|
|
const auto result = original_Reset(self, allocator, initial_state);
|
|
if (SUCCEEDED(result)) {
|
|
auto recording = get_recording(self);
|
|
auto pipeline = retain(initial_state);
|
|
recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) {
|
|
if (pipeline) original_SetPipelineState(target, pipeline.Get());
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
void STDMETHODCALLTYPE hook_ClearState(ID3D12GraphicsCommandList* self, ID3D12PipelineState* pipeline_state) {
|
|
Access access(self);
|
|
if (access.recording) {
|
|
access.recording->state.clear(); access.recording->indirect_state_unknown = false;
|
|
auto pipeline = retain(pipeline_state);
|
|
access.recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) { original_ClearState(target, pipeline.Get()); });
|
|
}
|
|
original_ClearState(access.target(), pipeline_state);
|
|
}
|
|
void STDMETHODCALLTYPE hook_ResourceBarrier(ID3D12GraphicsCommandList* self, UINT barrier_count, const D3D12_RESOURCE_BARRIER* barriers) {
|
|
Access access(self);
|
|
if (access.recording) for (UINT i = 0; i < barrier_count; ++i)
|
|
if (barriers[i].Type == D3D12_RESOURCE_BARRIER_TYPE_TRANSITION &&
|
|
(barriers[i].Transition.Subresource == 0 || barriers[i].Transition.Subresource == D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES)) {
|
|
auto& tracked = access.recording->resource_states[barriers[i].Transition.pResource];
|
|
auto& state = tracked.state;
|
|
state = ResourceAccess(barriers[i].Transition.StateAfter);
|
|
state.pending_split = barriers[i].Flags == D3D12_RESOURCE_BARRIER_FLAG_BEGIN_ONLY;
|
|
}
|
|
original_ResourceBarrier(access.target(), barrier_count, barriers);
|
|
}
|
|
void STDMETHODCALLTYPE hook_Barrier(ID3D12GraphicsCommandList* self, UINT32 barrier_groups_count, const D3D12_BARRIER_GROUP* barrier_groups) {
|
|
Access access(self);
|
|
if (access.recording) for (UINT32 g = 0; g < barrier_groups_count; ++g) if (barrier_groups[g].Type == D3D12_BARRIER_TYPE_TEXTURE)
|
|
for (UINT32 b = 0; b < barrier_groups[g].NumBarriers; ++b) {
|
|
const auto& barrier = barrier_groups[g].pTextureBarriers[b]; const auto& range = barrier.Subresources;
|
|
const bool containsZero = range.NumMipLevels == 0 ? (range.IndexOrFirstMipLevel == 0 || range.IndexOrFirstMipLevel == UINT32_MAX) :
|
|
range.IndexOrFirstMipLevel == 0 && range.FirstArraySlice == 0 && range.FirstPlane == 0;
|
|
if (containsZero) {
|
|
auto& tracked = access.recording->resource_states[barrier.pResource];
|
|
auto& state = tracked.state; state.enhanced = true; state.inherited = false;
|
|
state.layout = barrier.LayoutAfter; state.access = barrier.AccessAfter; state.sync = barrier.SyncAfter;
|
|
state.pending_split = barrier.SyncAfter == D3D12_BARRIER_SYNC_SPLIT;
|
|
}
|
|
}
|
|
original_Barrier(access.target(), barrier_groups_count, barrier_groups);
|
|
}
|
|
void STDMETHODCALLTYPE hook_BeginEvent(ID3D12GraphicsCommandList* self, UINT metadata, const void* data, UINT size) {
|
|
Access access(self);
|
|
if (access.recording) access.recording->events.push_back({metadata, copy_values(static_cast<const uint8_t*>(data), size)});
|
|
original_BeginEvent(access.target(), metadata, data, size);
|
|
}
|
|
void STDMETHODCALLTYPE hook_EndEvent(ID3D12GraphicsCommandList* self) {
|
|
Access access(self);
|
|
if (access.recording && !access.recording->events.empty()) access.recording->events.pop_back();
|
|
original_EndEvent(access.target());
|
|
}
|
|
void STDMETHODCALLTYPE hook_BeginRenderPass(ID3D12GraphicsCommandList* self, UINT render_target_count,
|
|
const D3D12_RENDER_PASS_RENDER_TARGET_DESC* render_targets, const D3D12_RENDER_PASS_DEPTH_STENCIL_DESC* depth_stencil, D3D12_RENDER_PASS_FLAGS flags) {
|
|
Access access(self); if (access.recording) access.recording->render_pass = true;
|
|
original_BeginRenderPass(access.target(), render_target_count, render_targets, depth_stencil, flags);
|
|
}
|
|
void STDMETHODCALLTYPE hook_EndRenderPass(ID3D12GraphicsCommandList* self) {
|
|
Access access(self); if (access.recording) access.recording->render_pass = false;
|
|
original_EndRenderPass(access.target());
|
|
}
|
|
void STDMETHODCALLTYPE hook_BeginQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) {
|
|
Access access(self); if (access.recording) ++access.recording->active_queries;
|
|
original_BeginQuery(access.target(), heap, type, index);
|
|
}
|
|
void STDMETHODCALLTYPE hook_EndQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) {
|
|
Access access(self); if (access.recording && access.recording->active_queries) --access.recording->active_queries;
|
|
original_EndQuery(access.target(), heap, type, index);
|
|
}
|
|
void STDMETHODCALLTYPE hook_SetPredication(ID3D12GraphicsCommandList* self, ID3D12Resource* buffer, UINT64 aligned_buffer_offset, D3D12_PREDICATION_OP operation) {
|
|
Access access(self);
|
|
if (access.recording) {
|
|
access.recording->predication = buffer != nullptr;
|
|
auto retained = retain(buffer);
|
|
access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) { original_SetPredication(target, retained.Get(), aligned_buffer_offset, operation); });
|
|
}
|
|
original_SetPredication(access.target(), buffer, aligned_buffer_offset, operation);
|
|
}
|
|
void STDMETHODCALLTYPE hook_ExecuteBundle(ID3D12GraphicsCommandList* self, ID3D12GraphicsCommandList* command_list) {
|
|
Access access(self);
|
|
auto bundle = get_recording(command_list, false);
|
|
if (access.recording && bundle) {
|
|
std::lock_guard<std::recursive_mutex> lock(bundle->mutex);
|
|
access.recording->state.insert(access.recording->state.end(), bundle->state.begin(), bundle->state.end());
|
|
access.recording->indirect_state_unknown |= bundle->indirect_state_unknown;
|
|
} else if (access.recording) access.recording->indirect_state_unknown = true;
|
|
original_ExecuteBundle(access.target(), command_list);
|
|
}
|
|
void STDMETHODCALLTYPE hook_ExecuteIndirect(ID3D12GraphicsCommandList* self, ID3D12CommandSignature* command_signature,
|
|
UINT max_command_count, ID3D12Resource* arg_buffer, UINT64 arg_buffer_offset, ID3D12Resource* count_buffer, UINT64 count_buffer_offset) {
|
|
Access access(self);
|
|
if (access.recording) {
|
|
UINT bytes = 0;
|
|
(void)command_signature->GetPrivateData(signature_description_tag, &bytes, nullptr);
|
|
if (bytes < sizeof(UINT) || bytes > sizeof(UINT) + 4096 * sizeof(D3D12_INDIRECT_ARGUMENT_DESC)) {
|
|
access.recording->indirect_state_unknown = true;
|
|
} else {
|
|
std::vector<uint8_t> description(bytes);
|
|
UINT count = 0;
|
|
const auto result = command_signature->GetPrivateData(signature_description_tag, &bytes, description.data());
|
|
std::memcpy(&count, description.data(), sizeof(count));
|
|
if (FAILED(result) || count > 4096 || bytes != sizeof(UINT) + count * sizeof(D3D12_INDIRECT_ARGUMENT_DESC)) access.recording->indirect_state_unknown = true;
|
|
else {
|
|
std::vector<D3D12_INDIRECT_ARGUMENT_DESC> arguments(count);
|
|
std::memcpy(arguments.data(), description.data() + sizeof(UINT), count * sizeof(D3D12_INDIRECT_ARGUMENT_DESC));
|
|
bool compute = false;
|
|
for (const auto& arg : arguments) compute |= arg.Type == D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH || arg.Type == D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH_RAYS;
|
|
// Microsoft documents that only bindings modified by the
|
|
// signature are reset to zero/NULL after ExecuteIndirect.
|
|
access.recording->state.emplace_back([arguments, compute](ID3D12GraphicsCommandList* target) {
|
|
for (const auto& arg : arguments) switch (arg.Type) {
|
|
// Public SDK addition after the pinned MinGW header:
|
|
// INCREMENTING_CONSTANT has the same first two UINTs.
|
|
case static_cast<D3D12_INDIRECT_ARGUMENT_TYPE>(11):
|
|
(compute ? original_SetComputeRoot32BitConstant : original_SetGraphicsRoot32BitConstant)
|
|
(target, arg.Constant.RootParameterIndex, 0, arg.Constant.DestOffsetIn32BitValues); break;
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT: {
|
|
std::vector<UINT> zeros(arg.Constant.Num32BitValuesToSet);
|
|
auto setter = compute ? original_SetComputeRoot32BitConstants : original_SetGraphicsRoot32BitConstants;
|
|
setter(target, arg.Constant.RootParameterIndex, UINT(zeros.size()), zeros.data(), arg.Constant.DestOffsetIn32BitValues); break;
|
|
}
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT_BUFFER_VIEW:
|
|
(compute ? original_SetComputeRootConstantBufferView : original_SetGraphicsRootConstantBufferView)(target, arg.ConstantBufferView.RootParameterIndex, 0); break;
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_SHADER_RESOURCE_VIEW:
|
|
(compute ? original_SetComputeRootShaderResourceView : original_SetGraphicsRootShaderResourceView)(target, arg.ShaderResourceView.RootParameterIndex, 0); break;
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_UNORDERED_ACCESS_VIEW:
|
|
(compute ? original_SetComputeRootUnorderedAccessView : original_SetGraphicsRootUnorderedAccessView)(target, arg.UnorderedAccessView.RootParameterIndex, 0); break;
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_INDEX_BUFFER_VIEW: original_IASetIndexBuffer(target, nullptr); break;
|
|
case D3D12_INDIRECT_ARGUMENT_TYPE_VERTEX_BUFFER_VIEW: {
|
|
D3D12_VERTEX_BUFFER_VIEW empty{}; original_IASetVertexBuffers(target, arg.VertexBuffer.Slot, 1, &empty); break;
|
|
}
|
|
default: break;
|
|
}
|
|
});
|
|
}
|
|
}
|
|
}
|
|
original_ExecuteIndirect(access.target(), command_signature, max_command_count, arg_buffer, arg_buffer_offset, count_buffer, count_buffer_offset);
|
|
}
|
|
|
|
using ExecuteFn = void(STDMETHODCALLTYPE*)(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*);
|
|
ExecuteFn original_execute = nullptr;
|
|
// Queue-owned synchronization metadata has no reference back to the queue.
|
|
// It serializes a split submission against other threads using the same queue.
|
|
static constexpr GUID submission_lock_tag = {0x6c6158ba,0x209f,0x4a09,{0x97,0x10,0xa7,0x03,0x52,0x32,0xa0,0x69}};
|
|
std::mutex queue_metadata_mutex;
|
|
struct SubmissionLock final : IUnknown {
|
|
std::atomic<ULONG> references{1};
|
|
std::recursive_mutex mutex;
|
|
ComPtr<ID3D12Fence> completion_fence;
|
|
Handle completion_event;
|
|
uint64_t completion_value = 0;
|
|
HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, void** out) override {
|
|
if (!out) return E_POINTER; *out = nullptr;
|
|
if (iid != IID_IUnknown) return E_NOINTERFACE;
|
|
*out = static_cast<IUnknown*>(this); AddRef(); return S_OK;
|
|
}
|
|
ULONG STDMETHODCALLTYPE AddRef() override { return ++references; }
|
|
ULONG STDMETHODCALLTYPE Release() override { const auto n = --references; if (!n) delete this; return n; }
|
|
};
|
|
ComPtr<SubmissionLock> submission_lock(ID3D12CommandQueue* queue) {
|
|
std::lock_guard<std::mutex> lock(queue_metadata_mutex);
|
|
ComPtr<SubmissionLock> result; UINT bytes = sizeof(IUnknown*);
|
|
if (SUCCEEDED(queue->GetPrivateData(submission_lock_tag, &bytes, result.GetAddressOf()))) return result;
|
|
result.Attach(new SubmissionLock);
|
|
dx(queue->SetPrivateDataInterface(submission_lock_tag, result.Get()), "Queue submission metadata");
|
|
return result;
|
|
}
|
|
using QueueFenceFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12CommandQueue*, ID3D12Fence*, UINT64);
|
|
QueueFenceFn original_queue_signal = nullptr, original_queue_wait = nullptr;
|
|
HRESULT queue_fence(QueueFenceFn function, ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) noexcept {
|
|
if (internal) return function(queue, fence, value);
|
|
try {
|
|
auto queueLock = submission_lock(queue);
|
|
std::lock_guard<std::recursive_mutex> ordered(queueLock->mutex);
|
|
return function(queue, fence, value);
|
|
} catch (const std::exception& failure) {
|
|
{ std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
|
|
std::fprintf(stderr, "D4R_QUEUE_FENCE_FAILURE %s\n", failure.what()); return E_FAIL;
|
|
}
|
|
}
|
|
HRESULT STDMETHODCALLTYPE hook_queue_signal(ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) {
|
|
return queue_fence(original_queue_signal, queue, fence, value);
|
|
}
|
|
HRESULT STDMETHODCALLTYPE hook_queue_wait(ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) {
|
|
return queue_fence(original_queue_wait, queue, fence, value);
|
|
}
|
|
void complete(ID3D12CommandQueue* queue) {
|
|
auto owner = submission_lock(queue);
|
|
std::lock_guard<std::recursive_mutex> ordered(owner->mutex);
|
|
if (!owner->completion_fence) {
|
|
ComPtr<ID3D12Device> device; dx(queue->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Queue device(submit drain)");
|
|
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(owner->completion_fence.GetAddressOf())), "Queue fence(submit drain)");
|
|
}
|
|
if (!owner->completion_event.value) owner->completion_event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
|
|
if (!owner->completion_event.value) throw std::runtime_error("CreateEvent(submit drain)");
|
|
const auto value = ++owner->completion_value;
|
|
dx(queue->Signal(owner->completion_fence.Get(), value), "Queue Signal(submit drain)");
|
|
if (owner->completion_fence->GetCompletedValue() < value) {
|
|
dx(owner->completion_fence->SetEventOnCompletion(value, owner->completion_event.value), "Queue event(submit drain)");
|
|
if (WaitForSingleObject(owner->completion_event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("GPU submit boundary timed out");
|
|
}
|
|
}
|
|
void retire(ID3D12CommandQueue* queue, std::vector<std::shared_ptr<Recording>> owners) {
|
|
auto owner = submission_lock(queue);
|
|
if (!owner->completion_fence) {
|
|
ComPtr<ID3D12Device> device; dx(queue->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Retirement device");
|
|
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(owner->completion_fence.GetAddressOf())), "Retirement fence");
|
|
}
|
|
const auto value = ++owner->completion_value;
|
|
dx(queue->Signal(owner->completion_fence.Get(), value), "Queue Signal(retirement)");
|
|
GpuRetirement::instance().submit(owner->completion_fence.Get(), value,
|
|
std::make_shared<std::vector<std::shared_ptr<Recording>>>(std::move(owners)));
|
|
}
|
|
void STDMETHODCALLTYPE hook_execute(ID3D12CommandQueue* queue, UINT count, ID3D12CommandList* const* lists) {
|
|
if (internal) { original_execute(queue, count, lists); return; }
|
|
const auto submitStart = Clock::now();
|
|
double publishMs = 0, boundaryMs = 0, drainMs = 0;
|
|
std::vector<std::shared_ptr<Recording>> owners;
|
|
std::vector<ID3D12CommandList*> pending;
|
|
std::vector<ResourceStates> pending_states;
|
|
InternalScope scope;
|
|
auto flush = [&] {
|
|
if (!pending.empty()) {
|
|
original_execute(queue, UINT(pending.size()), pending.data());
|
|
// Publish in the actual submitted order, never while recording.
|
|
// The callback's external fence waits for these producers on GPU.
|
|
const auto publishStart = Clock::now();
|
|
for (const auto& states : pending_states) publish_states(states);
|
|
publishMs += milliseconds(publishStart);
|
|
pending.clear(); pending_states.clear();
|
|
}
|
|
};
|
|
bool boundaries = false;
|
|
try {
|
|
auto queueLock = submission_lock(queue);
|
|
std::lock_guard<std::recursive_mutex> ordered(queueLock->mutex);
|
|
for (UINT i = 0; i < count; ++i) {
|
|
std::shared_ptr<Recording> recording;
|
|
{ std::lock_guard<std::mutex> lock(map_mutex);
|
|
auto found = recordings.find(static_cast<ID3D12GraphicsCommandList*>(lists[i]));
|
|
if (found != recordings.end()) recording = found->second; }
|
|
if (!recording) { pending.push_back(lists[i]); pending_states.emplace_back(); continue; }
|
|
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
|
|
if (recording->segments.empty()) {
|
|
pending.push_back(lists[i]); pending_states.push_back(recording->resource_states); continue;
|
|
}
|
|
if (!recording->closed) throw std::runtime_error("Logical command list submitted before Close");
|
|
owners.push_back(recording); boundaries = true;
|
|
for (const auto& segment : recording->segments) {
|
|
pending.push_back(segment.list); pending_states.push_back(segment.resource_states); flush();
|
|
const auto boundaryStart = Clock::now();
|
|
segment.boundary(queue);
|
|
boundaryMs += milliseconds(boundaryStart);
|
|
}
|
|
pending.push_back(recording->current.Get());
|
|
pending_states.push_back(recording->resource_states);
|
|
}
|
|
flush();
|
|
if (boundaries) {
|
|
const auto drainStart = Clock::now();
|
|
if (std::getenv("D4R_ASYNC_INTEROP")) retire(queue, std::move(owners)); else complete(queue);
|
|
drainMs = milliseconds(drainStart);
|
|
if (profile_commands()) std::printf("D4R_COMMAND_SUBMIT total_ms=%.3f boundary_ms=%.3f publish_ms=%.3f suffix_drain_ms=%.3f\n",
|
|
milliseconds(submitStart), boundaryMs, publishMs, drainMs);
|
|
}
|
|
} catch (const std::exception& failure) {
|
|
// Only this submission fails. Command tracking is intact, so later frames
|
|
// record and run normally; a transient failure (one refused shared-buffer
|
|
// export after a feature recreate) must not disable DLSS for the session.
|
|
std::fprintf(stderr, "D4R_D3D12_SUBMIT_FAILURE %s\n", failure.what());
|
|
try { complete(queue); } catch (...) {}
|
|
// Do not submit a consumer suffix after a failed DLSS boundary.
|
|
}
|
|
}
|
|
}
|
|
|
|
InternalScope::InternalScope() : previous(internal) { internal = true; }
|
|
InternalScope::~InternalScope() { internal = previous; }
|
|
std::string error() { std::lock_guard<std::mutex> lock(error_mutex); return last_error; }
|
|
size_t live_recordings() { std::lock_guard<std::mutex> lock(map_mutex); return recordings.size(); }
|
|
D3D12_RESOURCE_STATES resource_state(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES fallback) {
|
|
const auto state = resource_access(list, resource, ResourceAccess(fallback));
|
|
if (state.enhanced) throw std::runtime_error("Enhanced texture layout cannot be treated as a legacy state");
|
|
return state.legacy;
|
|
}
|
|
ResourceAccess resource_access(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, ResourceAccess fallback) {
|
|
auto recording = get_recording(list, false);
|
|
fallback.inherited = true;
|
|
if (!recording) return fallback;
|
|
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
|
|
auto found = recording->resource_states.find(resource);
|
|
return found == recording->resource_states.end() ? fallback : found->second.state;
|
|
}
|
|
ResourceAccess submitted_resource_access(ID3D12Resource* resource, ResourceAccess fallback) {
|
|
if (!resource) return fallback;
|
|
ResourceAccess submitted; UINT size = sizeof(submitted);
|
|
if (SUCCEEDED(resource->GetPrivateData(submitted_access_tag, &size, &submitted)) && size == sizeof(submitted)) return submitted;
|
|
return fallback;
|
|
}
|
|
void install(ID3D12Device* device) {
|
|
std::lock_guard<std::mutex> lock(installation_mutex);
|
|
if (installed) return;
|
|
InternalScope scope;
|
|
// An application may unload its NGX handle while native D3D12 objects are
|
|
// still alive. Public method callbacks must remain callable until exit.
|
|
HMODULE owner = nullptr;
|
|
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_PIN,
|
|
reinterpret_cast<LPCWSTR>(&install), &owner)) throw std::runtime_error("Cannot retain the D3D12 callback module");
|
|
const auto init = MH_Initialize();
|
|
if (init != MH_OK && init != MH_ERROR_ALREADY_INITIALIZED) throw std::runtime_error(MH_StatusToString(init));
|
|
ComPtr<ID3D12CommandAllocator> allocator; ComPtr<ID3D12GraphicsCommandList> list;
|
|
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Create hook allocator");
|
|
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Create hook command list");
|
|
try {
|
|
// The public COM interface inherits one ordered method table.
|
|
unsigned version = 0;
|
|
ComPtr<IUnknown> newest;
|
|
const GUID* ids[] = {&IID_ID3D12GraphicsCommandList, &IID_ID3D12GraphicsCommandList1, &IID_ID3D12GraphicsCommandList2,
|
|
&IID_ID3D12GraphicsCommandList3, &IID_ID3D12GraphicsCommandList4, &IID_ID3D12GraphicsCommandList5,
|
|
&IID_ID3D12GraphicsCommandList6, &IID_ID3D12GraphicsCommandList7,
|
|
&command_list8_iid, &command_list9_iid, &command_list10_iid};
|
|
for (unsigned n = 0; n < 11; ++n) {
|
|
ComPtr<IUnknown> candidate;
|
|
if (SUCCEEDED(list->QueryInterface(*ids[n], reinterpret_cast<void**>(candidate.GetAddressOf())))) { newest = std::move(candidate); version = n; }
|
|
}
|
|
if (newest.Get() != static_cast<IUnknown*>(list.Get())) throw std::runtime_error("D3D12 command-list versions need a separate interface identity adapter");
|
|
auto table = *reinterpret_cast<void***>(list.Get());
|
|
attach_recording_methods(table, version);
|
|
attach_method(table[2], reinterpret_cast<void*>(hook_release), reinterpret_cast<void**>(&original_release), "Release");
|
|
D3D12_COMMAND_QUEUE_DESC desc{}; desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
|
|
ComPtr<ID3D12CommandQueue> queue; dx(device->CreateCommandQueue(&desc, IID_PPV_ARGS(queue.GetAddressOf())), "Create hook queue");
|
|
auto queueTable = *reinterpret_cast<void***>(queue.Get());
|
|
// Queue inherits IUnknown, Object, DeviceChild; UpdateTileMappings,
|
|
// CopyTileMappings precede ExecuteCommandLists in the public SDK.
|
|
attach_method(queueTable[10], reinterpret_cast<void*>(hook_execute), reinterpret_cast<void**>(&original_execute), "ExecuteCommandLists");
|
|
attach_method(queueTable[queue_signal_slot], reinterpret_cast<void*>(hook_queue_signal), reinterpret_cast<void**>(&original_queue_signal), "QueueSignal");
|
|
attach_method(queueTable[queue_wait_slot], reinterpret_cast<void*>(hook_queue_wait), reinterpret_cast<void**>(&original_queue_wait), "QueueWait");
|
|
auto deviceTable = *reinterpret_cast<void***>(device);
|
|
attach_method(deviceTable[device_command_list_slot], reinterpret_cast<void*>(hook_create_list), reinterpret_cast<void**>(&original_create_list), "CreateCommandList");
|
|
attach_method(deviceTable[device_signature_slot], reinterpret_cast<void*>(hook_signature), reinterpret_cast<void**>(&original_signature), "CreateCommandSignature");
|
|
for (const auto& hook : attached) {
|
|
const auto result = MH_QueueEnableHook(hook.target);
|
|
if (result != MH_OK) throw std::runtime_error(MH_StatusToString(result));
|
|
}
|
|
const auto result = MH_ApplyQueued();
|
|
if (result != MH_OK) throw std::runtime_error(MH_StatusToString(result));
|
|
installed = true;
|
|
std::printf("D4R_D3D12_COMMAND_BACKEND installed=1 interface=%u hooks=%zu frame_age=0\n", version, attached.size());
|
|
} catch (...) {
|
|
for (const auto& hook : attached) { (void)MH_DisableHook(hook.target); (void)MH_RemoveHook(hook.target); }
|
|
attached.clear(); throw;
|
|
}
|
|
}
|
|
void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback) {
|
|
const auto recordStart = Clock::now();
|
|
if (!installed || !list || !callback) throw std::runtime_error("D3D12 command backend is not initialized");
|
|
if (!error().empty()) throw std::runtime_error(error());
|
|
if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) throw std::runtime_error("DLSS requires a direct command list");
|
|
Access access(list);
|
|
auto& recording = *access.recording;
|
|
if (recording.closed || recording.render_pass || recording.active_queries || recording.predication || recording.indirect_state_unknown)
|
|
throw std::runtime_error("Cannot split a closed list or an active render/query/predication/unknown indirect state scope");
|
|
// Construct/replay the suffix before modifying the original list so an
|
|
// allocation failure leaves the caller's recording intact.
|
|
InternalScope scope;
|
|
ComPtr<ID3D12Device> device; dx(list->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Boundary device");
|
|
Segment segment; segment.list = access.target(); segment.owner = recording.current;
|
|
segment.allocator = recording.current_allocator; segment.boundary = std::move(callback);
|
|
segment.resource_states = recording.resource_states;
|
|
ComPtr<ID3D12CommandAllocator> nextAllocator; ComPtr<ID3D12GraphicsCommandList> next;
|
|
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(nextAllocator.GetAddressOf())), "Suffix allocator");
|
|
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, nextAllocator.Get(), nullptr, IID_PPV_ARGS(next.GetAddressOf())), "Suffix command list");
|
|
const auto replayStart = Clock::now();
|
|
for (auto& state : recording.state) state(next.Get());
|
|
const auto replayMs = milliseconds(replayStart);
|
|
recording.segments.reserve(recording.segments.size() + 1);
|
|
for (const auto& event : recording.events) original_BeginEvent(next.Get(), event.metadata, event.data.data(), UINT(event.data.size()));
|
|
for (size_t n = 0; n < recording.events.size(); ++n) original_EndEvent(segment.list);
|
|
dx(original_Close(segment.list), "Close boundary prefix");
|
|
// Each suffix owns an independent allocator until GPU completion/reset.
|
|
recording.segments.push_back(std::move(segment));
|
|
recording.current_allocator = std::move(nextAllocator);
|
|
recording.current = std::move(next);
|
|
std::printf("D4R_D3D12_RECORD boundary=%zu state_calls=%zu frame_age=0\n", recording.segments.size(), recording.state.size());
|
|
if (profile_commands()) {
|
|
// This is the interval between NGX recording calls on this thread,
|
|
// not a Present/FPS measurement or an extra queued frame.
|
|
thread_local Clock::time_point previous{};
|
|
const double interval = previous == Clock::time_point{} ? 0 : std::chrono::duration<double, std::milli>(recordStart - previous).count();
|
|
previous = recordStart;
|
|
std::printf("D4R_COMMAND_RECORD total_ms=%.3f replay_ms=%.3f state_calls=%zu interval_ms=%.3f\n",
|
|
milliseconds(recordStart), replayMs, recording.state.size(), interval);
|
|
}
|
|
}
|
|
}
|