shim: adaptive marker polling, opt-in GPU-side input wait/completion and RGBA8 output

- [Interop] MarkerPollUs = auto is the new default: the input-marker wait learns when
  each feature's inputs become ready, sleeps in <=200 us chunks, yields near the
  prediction and backs off when late (d4r_input_wait.h); a number keeps fixed pacing
  (0 yields). d4r_config.py no longer exports 200 when the key is unset

- D4R_SHIM_GPU_INPUT_WAIT=1 (experimental, split VRAM, age 0): NGX's null stream
  waits on a coherent Vulkan word imported into HIP (d4rStreamWaitValue32Eq) instead
  of CPU polling. The original readback marker stays the authority; a CPU watchdog
  releases discarded lists and timeouts. Denoiser, verify, capture and dilation
  frames keep the CPU wait

- D4R_SHIM_GPU_COMPLETION=1 (needs GPU_INPUT_WAIT): the bridge writes a completion
  word right after the native output kernel (d4rNgxWriteValue32), and the game's queue
  polls it with a bounded compute wait (kernels/support/vulkan_wait.comp), replacing
  the split list's CPU timeline signal. frame_retired writes the same word on every
  failure path

- D4R_SHIM_NATIVE_COMPLETION=1: a null-stream callback in the bridge signals the
  split timeline (d4rNgxTimeline*); D4rFrameCompletion::can_signal only releases the
  oldest outstanding frame

- RGBA8 output: tex_common.h adds an 'R4' redirect tag, so K's native output kernel
  stores packed RGBA8 directly (D4R_SHIM_OUTPUT_RGBA8=1, checked against the kernel
  manifest by d4rOutputKernelRgba8). Host packing handles RGBA8 too.
  patches/vkd3d-proton/0004 (VKD3D_D4R_LINEAR_RGBA8=WxH) makes such outputs linear and
  exportable so the result can be written in place

- D4R_SHIM_FUSE_RGBA8_INPUT=1: one Vulkan pass converts and copies RGBA8 color input
  (vulkan_read.comp mode 5 matches the blit's toward-zero halves, plus a row stride)

- profiling: thread CPU time from the Unix bridge (Wine's GetThreadTimes ticks at
  10 ms); game-queue D3D12 timestamps (vk_input/vk_dlss_wait/vk_output/vk_total);
  gpu_eval split into input_gpu_wait and dlss_gpu; D4R_PROFILE_GPU_EVENTS,
  _TIMESTAMPS, _LOG and _CPU_TIME=0 switch off individual parts for bisection

- bridge: D4R_NGX_STREAM_PRIORITY=high|normal runs NGX on a priority stream
  (off by default; no gain measured)

- release_feature: a feature already marked retiring by a failed drain can still
  be released normally

- harness: RGBA8 with motion and jitter scenes (re-quantized each frame),
  D4R_HARNESS_RGBA8_RAMP, D4R_HARNESS_THRASH_MB to evict L2/Infinity Cache before
  each evaluation

- kernels/native: in_hoist.py and split_output_paths.py (opt-in scheduling
  experiments); scripts/build_k_ldr_output.sh builds K's LDR output with an
  optional D4R_LDR_OUTPUT_MAX_VGPR cap

- tests: test_input_wait.py and test_latency_optimizations.py; MarkerPollUs and
  frame-completion tests updated
This commit is contained in:
ayo committed 2026-10-13 22:30:41 -04:00
1 parent f0b55a850a
commit 162bbd77c8
22 files changed
+1989 -109

No files matched your search

+2
View File
@@ -30,8 +30,10 @@ struct Feature {
int splitSemaphore = 1;
bool engineNetTimeline = false; // the Vulkan timeline of this test, not the native network's
int engineNet = 0;
void* ngxTimeline = nullptr;
};
static struct { int (*signalTimeline)(int, uint64_t) = nullptr; } g_engineNet;
static struct { struct { int (*timelineSignal)(void*, uint64_t) = nullptr; } cu; } g;
static uint64_t gpuTimeline = 0;
static bool failSignal = false;
static int signal(int, const VkSemaphoreSignalInfo* info) {
+188
View File
@@ -0,0 +1,188 @@
"""CPU-only input-wait cancellation, polling and profiling contracts. No HIP or Wine runs."""
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
def function(source, signature):
start = source.index(signature)
opening = source.index('{', start)
depth = 1
end = opening + 1
while depth:
depth += (source[end] == '{') - (source[end] == '}')
end += 1
return source[start:end]
def run(source):
with tempfile.TemporaryDirectory() as tmp:
runner = Path(tmp) / 'input-wait'
subprocess.run(['c++', '-x', 'c++', '-std=c++20', '-pthread', '-Wall', '-Wextra', '-Werror',
'-I', str(ROOT / 'tools'), '-o', str(runner), '-'],
input=source, text=True, check=True, capture_output=True)
subprocess.run([str(runner)], check=True, timeout=5)
class InputWaitTests(unittest.TestCase):
def test_bridge_queues_exact_null_stream_wait_or_reports_unsupported(self):
bridge = (ROOT / 'tools/wine_nvcuda_bridge.c').read_text()
wait = function(bridge, 'CUresult WINAPI d4rStreamWaitValue32Eq(')
run(r'''
#include <cstdint>
#include <cstring>
#include <cassert>
#define WINAPI
using CUresult = int;
using CUdeviceptr = uint64_t;
using HIP_STREAM_WAIT_VALUE32_FN = int(*)(void*, void*, uint32_t, unsigned, uint32_t);
constexpr int CUDA_SUCCESS = 0, CUDA_ERROR_NOT_SUPPORTED = 801;
int context_setup_result = 0, supported = 1, waitResult = 0, queued = 0;
bool hasWait = true;
void ensure_context() {}
void* d4r_ngx_stream(void* stream) { return stream; } // D4R_NGX_STREAM_PRIORITY off
void tracef(const char*, ...) {}
bool trace_verbose() { return false; }
int device(int* out) { *out = 3; return 0; }
int attribute(int* out, int attr, int dev) { assert(attr == 10013 && dev == 3); *out = supported; return 0; }
int queue(void* stream, void* ptr, uint32_t value, unsigned flags, uint32_t mask) {
assert(stream == nullptr && ptr == reinterpret_cast<void*>(0x10008));
assert(value == 0xffffffffu && flags == 1 && mask == 0xffffffffu);
if (waitResult == 0) ++queued;
return waitResult;
}
void* hip_symbol(const char* name) {
if (strcmp(name, "hipStreamWaitValue32") == 0) return hasWait ? reinterpret_cast<void*>(queue) : nullptr;
if (strcmp(name, "hipDeviceGetAttribute") == 0) return reinterpret_cast<void*>(attribute);
if (strcmp(name, "hipGetDevice") == 0) return reinterpret_cast<void*>(device);
return nullptr;
}
''' + wait + r'''
int main() {
assert(d4rStreamWaitValue32Eq(0x10008, 0xffffffffu) == 0 && queued == 1);
supported = 0;
assert(d4rStreamWaitValue32Eq(0x10008, 0xffffffffu) == 801 && queued == 1);
supported = 1; hasWait = false;
assert(d4rStreamWaitValue32Eq(0x10008, 0xffffffffu) == 801 && queued == 1);
hasWait = true; waitResult = 1;
assert(d4rStreamWaitValue32Eq(0x10008, 0xffffffffu) == 801 && queued == 1);
context_setup_result = 700;
assert(d4rStreamWaitValue32Eq(0x10008, 0xffffffffu) == 700 && queued == 1);
}
''')
def test_readiness_discard_retirement_timeout_wrap_and_adaptive_pacing(self):
run(r'''
#include "d4r_input_wait.h"
#include <cassert>
int main() {
using A = D4rInputWaitAbort;
assert(d4r_input_wait_abort(7, 6, 6, false, 100) == A::None);
assert(d4r_input_wait_abort(7, 6, 8, false, 100) == A::Discarded);
assert(d4r_input_wait_abort(7, 7, 8, false, 100) == A::None); // slot recheck wins
assert(d4r_input_wait_abort(7, 7, 7, true, 100) == A::Retired);
assert(d4r_input_wait_abort(7, 6, 6, false, 5000001) == A::Timeout);
assert(d4r_input_wait_abort(7, 7, 7, false, 5000001) == A::Timeout); // broken wait after input
assert(d4r_input_wait_abort(0xffffffffu, 3, 0, false, 100) == A::Discarded);
assert(d4r_input_wait_abort(0, 0xffffffffu, 0xffffffffu, false, 100) == A::None);
assert(d4r_input_wait_abort(0, 0xffffffffu, 1, false, 100) == A::Discarded);
assert(d4r_input_wait_abort(0, 0, 1, false, 100) == A::None);
assert(d4r_marker_poll_delay_us(2000, 0) == 200);
assert(d4r_marker_poll_delay_us(2000, 1800) == 50);
assert(d4r_marker_poll_delay_us(2000, 1850) == 0);
assert(d4r_marker_poll_delay_us(2000, 2149) == 0);
assert(d4r_marker_poll_delay_us(2000, 2150) == 20);
assert(d4r_marker_poll_delay_us(2000, 3750) == 200);
assert(d4r_marker_poll_delay_us(0, 5000) == 200); // no warm prediction
}
''')
def test_actual_release_and_drain_helpers_preserve_real_markers_and_ownership(self):
shim = (ROOT / 'tools/d4r_nvngx_shim.cpp').read_text()
release = function(shim, 'static void release_ngx_input_wait_locked(')
complete = function(shim, 'static bool complete_ngx_input_wait(Feature& feature, uint32_t frame, bool abort)\n{')
run(r'''
#include <atomic>
#include <cassert>
#include <mutex>
#include <thread>
#include <future>
void logf(const char*, ...) {}
void MemoryBarrier() { std::atomic_thread_fence(std::memory_order_seq_cst); }
struct Feature {
std::atomic<bool> ngxGpuWait{true}, retiring{false};
void* ngxTimeline = nullptr;
std::mutex ngxWaitMutex;
bool ngxWaitActive = true, ngxWaitCancelled = false;
uint32_t ngxWaitFrame = 7;
int ngxWaitSlot = 1;
struct { void* mapped; } ngxInputSync;
unsigned evaluatedFrames = 100;
};
uint32_t words[4] = {0, 3, 4, 5}, markers[4] = {6, 3, 4, 5};
Feature* current;
int drains = 0, drainResult = 0;
int sync() {
++drains;
assert(words[2] == 7); // drain may only run after the CPU released the blocked queue
assert(current->ngxWaitActive); // ticket and input ownership survive the drain
return drainResult;
}
struct { struct { int(*ctxSynchronize)() = sync; void(*timelineCancel)(void*) = nullptr; } cu; } g;
''' + release + '\n' + complete + r'''
int main() {
Feature f; f.ngxInputSync.mapped = words; current = &f;
assert(complete_ngx_input_wait(f, 8, true)); // another frame cannot release this ticket
assert(drains == 0 && words[2] == 4);
// An independent thread can release a blocked host submission without invoking HIP.
auto blocked = std::async(std::launch::async, [&] {
for (;;) {
{ std::lock_guard<std::mutex> lock(f.ngxWaitMutex); if (words[2] == 7) return; }
std::this_thread::yield();
}
});
{ std::lock_guard<std::mutex> lock(f.ngxWaitMutex);
release_ngx_input_wait_locked(f, "discard", false); }
blocked.get();
assert(drains == 0 && f.ngxWaitCancelled && f.ngxGpuWait);
assert(words[0] == 0 && words[1] == 3 && words[3] == 5);
assert(markers[0] == 6 && markers[2] == 4); // never counterfeit game readiness
assert(!complete_ngx_input_wait(f, 7, true));
assert(drains == 1 && !f.ngxWaitActive && f.evaluatedFrames == 0);
assert(complete_ngx_input_wait(f, 7, true) && drains == 1); // idempotent retirement
f.ngxWaitActive = true; f.ngxWaitCancelled = false; f.evaluatedFrames = 2;
assert(complete_ngx_input_wait(f, 7, false)); // output already drained: no extra sync
assert(drains == 1 && !f.ngxWaitActive && f.evaluatedFrames == 2);
f.ngxWaitActive = true; f.ngxWaitCancelled = false;
{ std::lock_guard<std::mutex> lock(f.ngxWaitMutex);
release_ngx_input_wait_locked(f, "timeout", true); }
assert(!f.ngxGpuWait);
drainResult = 700;
assert(!complete_ngx_input_wait(f, 7, true));
assert(f.retiring && !f.ngxWaitActive && drains == 2);
}
''')
def test_thread_clock_coarse_or_unavailable_reports_na(self):
bridge = (ROOT / 'tools/wine_nvcuda_bridge.c').read_text()
clock = function(bridge, 'double WINAPI d4rThreadCpuTimeMs(void)')
run(r'''
#include <time.h>
#include <cassert>
#define WINAPI
long resolutionNs = 1;
bool available = true;
int fake_resolution(clockid_t, timespec* out) { out->tv_sec = 0; out->tv_nsec = resolutionNs; return 0; }
int timestamp(clockid_t, timespec* out) { out->tv_sec = 2; out->tv_nsec = 123456789; return available ? 0 : -1; }
#define clock_getres fake_resolution
#define clock_gettime timestamp
''' + clock + r'''
int main() {
assert(d4rThreadCpuTimeMs() == 2123.456789);
resolutionNs = 10000000; assert(d4rThreadCpuTimeMs() == -1);
resolutionNs = 1; available = false; assert(d4rThreadCpuTimeMs() == -1);
}
''')
+204
View File
@@ -0,0 +1,204 @@
"""CPU contracts for native completion and packed-output capability gating."""
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
def function(source, signature):
start = source.index(signature)
opening = source.index('{', start)
depth, end = 1, opening + 1
while depth:
depth += (source[end] == '{') - (source[end] == '}')
end += 1
return source[start:end]
def run(source, cpp=False):
with tempfile.TemporaryDirectory() as tmp:
exe = Path(tmp) / 'check'
result = subprocess.run(['c++' if cpp else 'cc', '-x', 'c++' if cpp else 'c',
'-std=c++20' if cpp else '-std=c11', '-pthread', '-Wall', '-Wextra', '-Werror',
'-I', str(ROOT / 'tools'), '-o', str(exe), '-'],
input=source, text=True, capture_output=True)
if result.returncode: raise AssertionError(result.stderr)
subprocess.run([str(exe)], check=True, timeout=5, cwd=tmp)
class LatencyContracts(unittest.TestCase):
def test_output_membership_filter_keeps_registry_results_exact(self):
bridge = (ROOT / 'tools/wine_nvcuda_bridge.c').read_text()
start = bridge.index('enum { OUTPUT_KERNEL_CAPACITY')
end = bridge.index('/* 1 when the last output kernel', start)
code = bridge[start:end]
run(r'''
#include <stdint.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <pthread.h>
#include <assert.h>
#define CUfunction void*
#define R_OK 4
static const char* fake_getenv(const char* name) { (void)name; return "/objects"; }
static int fake_access(const char* path, int mode) { (void)path; (void)mode; return 0; }
#define getenv fake_getenv
#define access fake_access
typedef struct { char name[128]; int rgba8; } NativeKernel;
NativeKernel objects[1] = {{"hiluma_engine_output_candidate", 1}};
NativeKernel* native_kernels = objects;
size_t native_kernel_count = 1;
''' + code + r'''
int main(void) {
void* output = (void*)(uintptr_t)0x5000;
note_function_lookup(output, "hiluma_engine_output_candidate"); note_launch(output);
assert(last_output_native == 1 && last_output_rgba8 == 1);
for (uintptr_t i = 1; i < 1000; ++i) {
void* network = (void*)(i * 16);
if (network == output) continue;
note_launch(network);
assert(last_output_native == 1 && last_output_rgba8 == 1); // negatives/collisions change nothing
}
note_function_lookup(output, "rrlite_downsample_kernel_static_ldr"); note_launch(output);
assert(last_output_native == 1 && last_output_rgba8 == 0); // newest lookup wins on handle reuse
}
''')
def test_completed_prefix_callback_gate(self):
run(r'''
#include "d4r_frame_completion.h"
#include <cassert>
int main() {
D4rFrameCompletion completion;
assert(!completion.can_signal(1));
completion.admit(1); completion.admit(2); completion.admit(3);
assert(completion.can_signal(1) && !completion.can_signal(2));
assert(completion.retire(3) == 0); // later discard does not cross unfinished work
assert(!completion.can_signal(3));
assert(completion.retire(1) == 1 && completion.can_signal(2));
assert(completion.retire(2) == 3 && !completion.can_signal(3));
}
''', cpp=True)
def test_actual_native_callback_cancellation_monotonicity_and_detached_cleanup(self):
bridge = (ROOT / 'tools/wine_nvcuda_bridge.c').read_text()
start = bridge.index('typedef struct', bridge.index("/* NGX's null-stream completion"))
end = bridge.index('} D4rNgxTimeline;', start) + len('} D4rNgxTimeline;')
definitions = bridge[start:end]
bodies = '\n'.join(function(bridge, sig) for sig in (
'int WINAPI d4rNgxTimelineSignal(', 'static void d4r_ngx_completed(',
'CUresult WINAPI d4rNgxTimelinePrepare(', 'void WINAPI d4rNgxTimelineCancel(',
'CUresult WINAPI d4rNgxTimelineQueue(', 'void WINAPI d4rNgxTimelineDestroy(',
'CUresult WINAPI d4rNgxTimelineWait('))
run(r'''
#define _POSIX_C_SOURCE 200809L
#include <stdint.h>
#include <stdlib.h>
#include <pthread.h>
#include <time.h>
#include <errno.h>
#include <assert.h>
#define WINAPI
#define CUDA_SUCCESS 0
#define CUDA_ERROR_NOT_READY 600
#define CUDA_ERROR_NOT_SUPPORTED 801
#define CUDA_ERROR_UNKNOWN 999
#define CUresult int
typedef struct { int sType; const void* pNext; uint64_t semaphore, value; } D4rVkSemaphoreSignalInfo;
typedef int (*VK_SIGNAL_SEMAPHORE_FN)(void*, const D4rVkSemaphoreSignalInfo*);
typedef int (*HIP_LAUNCH_HOST_FN)(void*, void(*)(void*), void*);
static void* d4r_ngx_stream(void* stream) { return stream; } /* D4R_NGX_STREAM_PRIORITY off */
void (*callback)(void*); void* argument;
uint64_t signalled = 0; int signals = 0, launchResult = 0;
int signal_fake(void* device, const D4rVkSemaphoreSignalInfo* info) {
(void)device; assert(info->value > signalled); signalled = info->value; ++signals; return 0;
}
static void* complete_later(void* unused) {
(void)unused; struct timespec pause = {0, 20000000}; nanosleep(&pause, NULL);
callback(argument); return NULL;
}
int launch_fake(void* stream, void(*fn)(void*), void* arg) {
assert(stream == NULL);
if (!launchResult) { callback = fn; argument = arg; }
return launchResult;
}
''' + definitions + '\n' + bodies + r'''
int main(void) {
D4rNgxTimeline* t = calloc(1, sizeof(*t));
assert(pthread_mutex_init(&t->lock, NULL) == 0);
pthread_condattr_t attr; assert(pthread_condattr_init(&attr) == 0);
assert(pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) == 0);
assert(pthread_cond_init(&t->completed, &attr) == 0); pthread_condattr_destroy(&attr);
t->signal = signal_fake; t->launch = launch_fake;
assert(d4rNgxTimelinePrepare(t) == 0);
assert(d4rNgxTimelineQueue(t, 1) == 0);
assert(d4rNgxTimelinePrepare(t) == 600 && d4rNgxTimelineQueue(t, 2) == 600);
d4rNgxTimelineCancel(t); callback(argument); assert(signals == 0 && t->busy == 0);
assert(d4rNgxTimelineWait(t) == 0);
assert(d4rNgxTimelineSignal(t, 1) == 0 && signals == 1);
assert(d4rNgxTimelinePrepare(t) == 0 && d4rNgxTimelineQueue(t, 2) == 0);
callback(argument); assert(signalled == 2 && signals == 2);
assert(d4rNgxTimelinePrepare(t) == 0 && d4rNgxTimelineQueue(t, 3) == 0);
assert(d4rNgxTimelineSignal(t, 4) == 0); callback(argument);
assert(signalled == 4 && signals == 3); // late callback never signals backwards
launchResult = 1;
assert(d4rNgxTimelinePrepare(t) == 0 && d4rNgxTimelineQueue(t, 5) == 801 && t->busy == 0);
launchResult = 0;
assert(d4rNgxTimelineQueue(t, 5) == 0);
pthread_t worker; assert(pthread_create(&worker, NULL, complete_later, NULL) == 0);
assert(d4rNgxTimelineWait(t) == 0); pthread_join(worker, NULL);
assert(signalled == 5 && signals == 4);
assert(d4rNgxTimelinePrepare(t) == 0 && d4rNgxTimelineQueue(t, 6) == 0);
d4rNgxTimelineDestroy(t); callback(argument); // ticket outlives owner, cancellation forbids a signal
assert(signals == 4);
}
''')
def test_all_half_values_pack_like_the_reference_unorm_conversion(self):
header = (ROOT / 'kernels/tex/tex_common.h').read_text()
body = function(header, '__attribute__((device)) static inline uint32_t redirect_unorm(')
body = body.replace('__attribute__((device)) ', '')
run(r'''
#include <cstdint>
#include <cmath>
#include <cassert>
''' + body + r'''
int main() {
for (uint32_t bits = 0; bits < 65536; ++bits) {
const int exponent = (bits >> 10) & 31, mantissa = bits & 1023;
double value = exponent == 0 ? std::ldexp(double(mantissa), -24)
: exponent == 31 ? (mantissa ? NAN : INFINITY)
: std::ldexp(1.0 + double(mantissa) / 1024.0, exponent - 15);
if (bits & 0x8000) value = -value;
value = value > 0.0 ? value : 0.0;
value = value < 1.0 ? value : 1.0;
const uint32_t expected = uint32_t(std::floor(value * 255.0 + 0.5));
assert(redirect_unorm(uint16_t(bits), 255.0f) == expected);
}
}
''', cpp=True)
def test_packed_capability_requires_matching_code_object_bytes(self):
bridge = (ROOT / 'tools/wine_nvcuda_bridge.c').read_text()
body = function(bridge, 'static int native_rgba8_verified(')
run(r'''
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
''' + body + r'''
int main(void) {
assert(!native_rgba8_verified("/does/not/exist", "output"));
FILE* f = fopen("output.hsaco", "wb"); assert(f != NULL); fputs("abc", f); fclose(f);
assert(!native_rgba8_verified(".", "output"));
f = fopen("d4r-rgba8.txt", "w"); fputs("output e71fa2190541574b\n", f); fclose(f);
assert(native_rgba8_verified(".", "output"));
f = fopen("output.hsaco", "ab"); fputc('d', f); fclose(f);
assert(!native_rgba8_verified(".", "output"));
assert(!native_rgba8_verified(".", "other"));
}
''')
+5 -2
View File
@@ -35,10 +35,13 @@ class ShimConfigTests(unittest.TestCase):
ini = Path(temporary) / 'd4r.ini'
env = {key: value for key, value in os.environ.items() if not key.startswith('D4R_')}
cmd = ['python3', str(ROOT / 'scripts/d4r_config.py'), '--config', str(ini)]
for value, expected in ((None, '200'), ('auto', '200'), ('200', '200'), ('0', '0'), ('75', '75')):
for value, expected in ((None, None), ('auto', None), ('200', '200'), ('0', '0'), ('75', '75')):
ini.write_text('[Interop]\n' + (f'MarkerPollUs = {value}\n' if value is not None else ''))
result = subprocess.run(cmd, env=env, text=True, capture_output=True, check=True)
self.assertIn(f'export D4R_SHIM_MARKER_POLL_US={expected}\n', result.stdout)
if expected is None:
self.assertNotIn('export D4R_SHIM_MARKER_POLL_US=', result.stdout)
else:
self.assertIn(f'export D4R_SHIM_MARKER_POLL_US={expected}\n', result.stdout)
# An explicit environment variable keeps its existing precedence.
result = subprocess.run(cmd, env=dict(env, D4R_SHIM_MARKER_POLL_US='35'),
text=True, capture_output=True, check=True)