shim: adaptive marker polling, opt-in GPU-side input wait/completion and RGBA8 output

- [Interop] MarkerPollUs = auto is the new default: the input-marker wait learns when
  each feature's inputs become ready, sleeps in <=200 us chunks, yields near the
  prediction and backs off when late (d4r_input_wait.h); a number keeps fixed pacing
  (0 yields). d4r_config.py no longer exports 200 when the key is unset

- D4R_SHIM_GPU_INPUT_WAIT=1 (experimental, split VRAM, age 0): NGX's null stream
  waits on a coherent Vulkan word imported into HIP (d4rStreamWaitValue32Eq) instead
  of CPU polling. The original readback marker stays the authority; a CPU watchdog
  releases discarded lists and timeouts. Denoiser, verify, capture and dilation
  frames keep the CPU wait

- D4R_SHIM_GPU_COMPLETION=1 (needs GPU_INPUT_WAIT): the bridge writes a completion
  word right after the native output kernel (d4rNgxWriteValue32), and the game's queue
  polls it with a bounded compute wait (kernels/support/vulkan_wait.comp), replacing
  the split list's CPU timeline signal. frame_retired writes the same word on every
  failure path

- D4R_SHIM_NATIVE_COMPLETION=1: a null-stream callback in the bridge signals the
  split timeline (d4rNgxTimeline*); D4rFrameCompletion::can_signal only releases the
  oldest outstanding frame

- RGBA8 output: tex_common.h adds an 'R4' redirect tag, so K's native output kernel
  stores packed RGBA8 directly (D4R_SHIM_OUTPUT_RGBA8=1, checked against the kernel
  manifest by d4rOutputKernelRgba8). Host packing handles RGBA8 too.
  patches/vkd3d-proton/0004 (VKD3D_D4R_LINEAR_RGBA8=WxH) makes such outputs linear and
  exportable so the result can be written in place

- D4R_SHIM_FUSE_RGBA8_INPUT=1: one Vulkan pass converts and copies RGBA8 color input
  (vulkan_read.comp mode 5 matches the blit's toward-zero halves, plus a row stride)

- profiling: thread CPU time from the Unix bridge (Wine's GetThreadTimes ticks at
  10 ms); game-queue D3D12 timestamps (vk_input/vk_dlss_wait/vk_output/vk_total);
  gpu_eval split into input_gpu_wait and dlss_gpu; D4R_PROFILE_GPU_EVENTS,
  _TIMESTAMPS, _LOG and _CPU_TIME=0 switch off individual parts for bisection

- bridge: D4R_NGX_STREAM_PRIORITY=high|normal runs NGX on a priority stream
  (off by default; no gain measured)

- release_feature: a feature already marked retiring by a failed drain can still
  be released normally

- harness: RGBA8 with motion and jitter scenes (re-quantized each frame),
  D4R_HARNESS_RGBA8_RAMP, D4R_HARNESS_THRASH_MB to evict L2/Infinity Cache before
  each evaluation

- kernels/native: in_hoist.py and split_output_paths.py (opt-in scheduling
  experiments); scripts/build_k_ldr_output.sh builds K's LDR output with an
  optional D4R_LDR_OUTPUT_MAX_VGPR cap

- tests: test_input_wait.py and test_latency_optimizations.py; MarkerPollUs and
  frame-completion tests updated

tools/d3d12_dlss_harness.cpp

@@ -1125,9 +1125,9 @@ int main(int argc, char** argv)
         std::fprintf(stderr, "RGB10A2 output cannot be combined with RGBA8 or the quality scene\n");
         return 2;
     }
-    if (rgba8 && (qualityScene || motionScene || jitterScene || replayDir != nullptr))
+    if (rgba8 && (qualityScene || replayDir != nullptr))
     {
-        std::fprintf(stderr, "RGBA8 format probe requires the static synthetic scene\n");
+        std::fprintf(stderr, "RGBA8 format probe supports static, motion and jitter scenes\n");
         return 2;
     }
     const bool temporal = std::getenv("D4R_HARNESS_TEMPORAL") != nullptr || motionScene || jitterScene || qualityScene ||
@@ -1226,6 +1226,8 @@ int main(int argc, char** argv)
         for (size_t index = 0; index < color.size(); ++index)
             color8[index] = static_cast<uint8_t>(std::lround(
                 std::clamp(static_cast<float>(std::bit_cast<_Float16>(color[index])), 0.0f, 1.0f) * 255.0f));
+        if (std::getenv("D4R_HARNESS_RGBA8_RAMP") != nullptr)
+            for (size_t i = 0; i < color8.size(); ++i) color8[i] = static_cast<uint8_t>(i);
         outputInit8.resize(static_cast<size_t>(outWidth) * outHeight * 4);
     }
 
@@ -1421,6 +1423,17 @@ int main(int argc, char** argv)
                                                    : rgb10 ? DXGI_FORMAT_R10G10B10A2_UNORM
                                                            : DXGI_FORMAT_R16G16B16A16_FLOAT,
                                                    D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS);
+    // Moving/jittered RGBA8 fixtures quantize the same canonical half input as the initial upload.
+    // Never upload the half byte layout into an RGBA8 resource: that would make a format test silently
+    // exercise different content instead of checking the conversion/packing route used by the game.
+    const auto update_color = [&] {
+        if (rgba8)
+            for (size_t i = 0; i < color.size(); ++i)
+                color8[i] = static_cast<uint8_t>(std::lround(std::clamp(
+                    static_cast<float>(std::bit_cast<_Float16>(color[i])), 0.0f, 1.0f) * 255.0f));
+        update_texture(colorTexture, rgba8 ? static_cast<const void*>(color8.data())
+                                          : static_cast<const void*>(color.data()), resourceWidth * (rgba8 ? 4 : 8), srv);
+    };
     upload(colorTexture, rgba8 ? static_cast<const void*>(color8.data()) : static_cast<const void*>(color.data()),
            resourceWidth * (rgba8 ? 4 : 8), srv);
     upload(depthTexture, depth.data(), resourceWidth * 4, srv);
@@ -1757,7 +1770,7 @@ int main(int argc, char** argv)
                     color[pixel * 4 + 0] = color[pixel * 4 + 1] = color[pixel * 4 + 2] = half;
                     color[pixel * 4 + 3] = 0x3c00u;
                 }
-            update_texture(colorTexture, color.data(), resourceWidth * 8, srv);
+            update_color();
             if (frame == 1 && (panX != 0 || panY != 0))
             {
                 // Current position -> previous position. Without MVLowRes DLSS
@@ -1800,7 +1813,7 @@ int main(int argc, char** argv)
                 {0.0f, 0.25f, -0.25f, -0.125f, 0.375f, 0.125f, -0.375f, 0.0f};
             const size_t phase = static_cast<size_t>(frame - 1) % jitterX.size();
             render_jittered_background(color, inWidth, inHeight, resourceWidth, jitterX[phase], jitterY[phase]);
-            update_texture(colorTexture, color.data(), resourceWidth * 8, srv);
+            update_color();
             d4r_ngx_set_float(parameters, "Jitter.Offset.X", jitterX[phase]);
             d4r_ngx_set_float(parameters, "Jitter.Offset.Y", jitterY[phase]);
         }
@@ -1823,7 +1836,7 @@ int main(int argc, char** argv)
                     motion[motionPixel * 2 + 0] = 0xc400u; // -4 render pixels: current position -> previous
                     motion[motionPixel * 2 + 1] = 0;
                 }
-            update_texture(colorTexture, color.data(), resourceWidth * 8, srv);
+            update_color();
             update_texture(depthTexture, depth.data(), resourceWidth * 4, srv);
             update_texture(motionTexture, motion_rows(motion.data()), motionRowBytes, srv);
         }
@@ -2054,6 +2067,20 @@ int main(int argc, char** argv)
         LARGE_INTEGER counterStart, counterRecorded, counterEnd, counterFrequency;
         QueryPerformanceFrequency(&counterFrequency);
         QueryPerformanceCounter(&counterStart);
+        // D4R_HARNESS_THRASH_MB=N: copy N MB between two private buffers ahead of the evaluation, as a game's
+        // own rendering does, so DLSS starts with its weights and history evicted from L2 and Infinity Cache
+        // instead of finding them warm from the previous back-to-back frame.
+        static const UINT64 thrashBytes = std::getenv("D4R_HARNESS_THRASH_MB") != nullptr
+            ? std::clamp<UINT64>(std::strtoull(std::getenv("D4R_HARNESS_THRASH_MB"), nullptr, 10), 0, 1024) << 20 : 0;
+        static ID3D12Resource* thrashSource = nullptr;
+        static ID3D12Resource* thrashTarget = nullptr;
+        if (thrashBytes != 0 && thrashSource == nullptr)
+        {
+            thrashSource = create_buffer(D3D12_HEAP_TYPE_DEFAULT, thrashBytes, D3D12_RESOURCE_STATE_COMMON);
+            thrashTarget = create_buffer(D3D12_HEAP_TYPE_DEFAULT, thrashBytes, D3D12_RESOURCE_STATE_COMMON);
+        }
+        if (thrashSource != nullptr && thrashTarget != nullptr)
+            g_list->CopyBufferRegion(thrashTarget, 0, thrashSource, 0, thrashBytes);
         result = evaluate(g_list, feature, parameters, nullptr);
         QueryPerformanceCounter(&counterRecorded);
         if (const char* delay = std::getenv("D4R_HARNESS_DELAY_SUBMIT_MS"))

tools/d4r_frame_completion.h

@@ -11,6 +11,11 @@ class D4rFrameCompletion
 public:
     void admit(uint32_t frame) { pending_.push_back({frame, false}); }
 
+    // A native completion callback can release this value without crossing an unfinished producer
+    // only when it is the first admitted frame still outstanding. Retirement itself stays on the
+    // worker, which also folds any later discarded frames into the completed prefix.
+    bool can_signal(uint32_t frame) const { return !pending_.empty() && pending_.front().frame == frame; }
+
     uint64_t retire(uint32_t frame)
     {
         for (auto& pending : pending_)

tools/d4r_input_wait.h

@@ -0,0 +1,30 @@
+#pragma once
+#include <cstdint>
+
+// The original marker remains the authority for whether a command list ran. A CPU release of the
+// separate, coherent GPU wait word must never make discarded input look ready to the worker.
+enum class D4rInputWaitAbort { None, Retired, Discarded, Timeout };
+
+inline D4rInputWaitAbort d4r_input_wait_abort(uint32_t frame, uint32_t slotMarker, uint32_t aggregate,
+                                            bool retiring, double elapsedUs)
+{
+    if (retiring)
+        return D4rInputWaitAbort::Retired;
+    if (slotMarker != frame && static_cast<int32_t>(aggregate - frame) > 0)
+        return D4rInputWaitAbort::Discarded;
+    // Also covers a stream wait that failed to observe an input word already written by Vulkan.
+    if (elapsedUs > 5000000.0)
+        return D4rInputWaitAbort::Timeout;
+    return D4rInputWaitAbort::None;
+}
+
+inline unsigned d4r_marker_poll_delay_us(double predictedUs, double elapsedUs)
+{
+    const double remaining = predictedUs - 150.0 - elapsedUs;
+    if (remaining > 20.0)
+        return static_cast<unsigned>(remaining < 200.0 ? remaining : 200.0);
+    if (elapsedUs < predictedUs + 150.0)
+        return 0; // yield during the final 150 us and a short late window
+    const double late = 20.0 * (1.0 + (elapsedUs - predictedUs - 150.0) / 160.0);
+    return static_cast<unsigned>(late < 200.0 ? late : 200.0);
+}

tools/d4r_vulkan_staging.h

@@ -16,6 +16,10 @@ struct D4rVkStaging {
     VkDescriptorSetLayout readSet = VK_NULL_HANDLE, writeSet = VK_NULL_HANDLE;
     VkPipelineLayout readLayout = VK_NULL_HANDLE, writeLayout = VK_NULL_HANDLE;
     VkPipeline readPipeline = VK_NULL_HANDLE;
+    // D4R_SHIM_GPU_COMPLETION: one storage buffer of words, push constants {index, value, limit}
+    VkDescriptorSetLayout waitSet = VK_NULL_HANDLE;
+    VkPipelineLayout waitLayout = VK_NULL_HANDLE;
+    VkPipeline waitPipeline = VK_NULL_HANDLE;
     struct OutputPipeline { VkFormat format; VkPipeline pipeline; } outputs[5] = {};
     PFN_vkCreateSampler createSampler = nullptr;
     PFN_vkDestroySampler destroySampler = nullptr;
@@ -81,7 +85,7 @@ struct D4rVkStaging {
             set.bindingCount = 2; set.pBindings = bindings;
             auto& setLayout = output ? writeSet : readSet;
             if (createSetLayout(d, &set, nullptr, &setLayout) != VK_SUCCESS) return false;
-            VkPushConstantRange range = {VK_SHADER_STAGE_COMPUTE_BIT, 0, output ? 8u : 12u};
+            VkPushConstantRange range = {VK_SHADER_STAGE_COMPUTE_BIT, 0, output ? 8u : 16u};
             VkPipelineLayoutCreateInfo layout = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
             layout.setLayoutCount = 1; layout.pSetLayouts = &setLayout;
             layout.pushConstantRangeCount = 1; layout.pPushConstantRanges = ⦥
@@ -89,6 +93,44 @@ struct D4rVkStaging {
         }
         return pipeline(d4r_vk_read, sizeof(d4r_vk_read), readLayout, readPipeline);
     }
+    // Created on first use; false leaves GPU completion off.
+    bool wait_ready() {
+        if (waitPipeline) return true;
+        VkDescriptorSetLayoutBinding binding = {0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
+        VkDescriptorSetLayoutCreateInfo set = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
+        set.bindingCount = 1; set.pBindings = &binding;
+        if (!waitSet && createSetLayout(device, &set, nullptr, &waitSet) != VK_SUCCESS) return false;
+        VkPushConstantRange range = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 12};
+        VkPipelineLayoutCreateInfo layout = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
+        layout.setLayoutCount = 1; layout.pSetLayouts = &waitSet;
+        layout.pushConstantRangeCount = 1; layout.pPushConstantRanges = ⦥
+        if (!waitLayout && createLayout(device, &layout, nullptr, &waitLayout) != VK_SUCCESS) return false;
+        return pipeline(d4r_vk_wait, sizeof(d4r_vk_wait), waitLayout, waitPipeline);
+    }
+    bool wait_set(D4rVkStagingSet& result, VkBuffer buffer, VkDeviceSize bytes) {
+        if (!wait_ready()) return false;
+        VkDescriptorPoolSize size = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1};
+        VkDescriptorPoolCreateInfo pool = {VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
+        pool.maxSets = 1; pool.poolSizeCount = 1; pool.pPoolSizes = &size;
+        if (!result.pool && createPool(device, &pool, nullptr, &result.pool) != VK_SUCCESS) return false;
+        VkDescriptorSetAllocateInfo alloc = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
+        alloc.descriptorPool = result.pool; alloc.descriptorSetCount = 1; alloc.pSetLayouts = &waitSet;
+        if (!result.set && allocateSets(device, &alloc, &result.set) != VK_SUCCESS) return false;
+        VkDescriptorBufferInfo storage = {buffer, 0, bytes};
+        VkWriteDescriptorSet write = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
+        write.dstSet = result.set; write.dstBinding = 0; write.descriptorCount = 1;
+        write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; write.pBufferInfo = &storage;
+        updateSets(device, 1, &write, 0, nullptr);
+        result.pipeline = waitPipeline;
+        return true;
+    }
+    void record_wait(VkCommandBuffer cmd, const D4rVkStagingSet& set, uint32_t index, uint32_t value, uint32_t limit) {
+        bindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, waitPipeline);
+        bindSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, waitLayout, 0, 1, &set.set, 0, nullptr);
+        const uint32_t constants[] = {index, value, limit};
+        push(cmd, waitLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, 12, constants);
+        dispatch(cmd, 1, 1, 1);
+    }
     VkPipeline output_pipeline(VkFormat format) {
         for (auto& item : outputs) if (item.pipeline && item.format == format) return item.pipeline;
         const uint32_t* code = nullptr; size_t bytes = 0;
@@ -135,12 +177,12 @@ struct D4rVkStaging {
         updateSets(device, 2, writes, 0, nullptr);
         return true;
     }
-    void record(VkCommandBuffer cmd, const D4rVkStagingSet& set, uint32_t width, uint32_t height, uint32_t inputMode = 0) {
+    void record(VkCommandBuffer cmd, const D4rVkStagingSet& set, uint32_t width, uint32_t height, uint32_t inputMode = 0, uint32_t stride = 0) {
         const auto layout = set.output ? writeLayout : readLayout;
         bindPipeline(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, set.pipeline);
         bindSets(cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &set.set, 0, nullptr);
-        const uint32_t dimensions[] = {width, height, inputMode};
-        push(cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, set.output ? 8 : 12, dimensions);
+        const uint32_t dimensions[] = {width, height, inputMode, stride ? stride : width};
+        push(cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, set.output ? 8 : 16, dimensions);
         dispatch(cmd, (width+7)/8, (height+7)/8, 1);
     }
     void free(D4rVkStagingSet& set) {
@@ -149,6 +191,9 @@ struct D4rVkStaging {
     }
     void destroy() {
         if (readPipeline) destroyPipeline(device, readPipeline, nullptr);
+        if (waitPipeline) destroyPipeline(device, waitPipeline, nullptr);
+        if (waitLayout) destroyLayout(device, waitLayout, nullptr);
+        if (waitSet) destroySetLayout(device, waitSet, nullptr);
         for (auto& item : outputs) if (item.pipeline) destroyPipeline(device, item.pipeline, nullptr);
         if (readLayout) destroyLayout(device, readLayout, nullptr);
         if (writeLayout) destroyLayout(device, writeLayout, nullptr);

tools/wine_nvcuda_bridge.c

@@ -11,6 +11,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <time.h>
 #include "d4r_native_selection.h"
 #include "d4r_motion_dilation.h"
 #ifdef D4R_MOTION_KERNELS
@@ -366,11 +367,41 @@ typedef struct
     char name[128];
     uint64_t hash;
     int linked;
+    int rgba8;
 } NativeKernel;
 
 static NativeKernel* native_kernels;
 static size_t native_kernel_count;
 static char native_source[1024];
+
+/* Packed stores are a code-object capability, not a property of the kernel's name. A legacy native
+   kernel ignores the R4 redirect tag. The sidecar must therefore name the exact bytes installed
+   alongside the PTX compatibility manifest; the per-process cache retains those same bytes. */
+static int native_rgba8_verified(const char* directory, const char* name)
+{
+    char path[1400], row[256], entry[128];
+    unsigned long long expected = 0;
+    snprintf(path, sizeof(path), "%s/d4r-rgba8.txt", directory);
+    FILE* manifest = fopen(path, "r");
+    if (manifest == NULL) return 0;
+    int found = 0;
+    while (fgets(row, sizeof(row), manifest) != NULL)
+        if (sscanf(row, "%127s %llx", entry, &expected) == 2 && strcmp(entry, name) == 0)
+        { found = 1; break; }
+    fclose(manifest);
+    if (!found) return 0;
+    snprintf(path, sizeof(path), "%s/%s.hsaco", directory, name);
+    FILE* object = fopen(path, "rb");
+    if (object == NULL) return 0;
+    uint64_t hash = UINT64_C(14695981039346656037);
+    unsigned char bytes[4096];
+    size_t count;
+    while ((count = fread(bytes, 1, sizeof(bytes), object)) != 0)
+        for (size_t i = 0; i < count; ++i) hash = (hash ^ bytes[i]) * UINT64_C(1099511628211);
+    const int valid = !ferror(object) && hash == (uint64_t)expected;
+    fclose(object);
+    return valid;
+}
 static char native_served[1024];
 static pthread_mutex_t native_lock = PTHREAD_MUTEX_INITIALIZER;
 
@@ -506,6 +537,7 @@ static void prepare_native_kernels(const char* cache_home)
         snprintf(kernel->name, sizeof(kernel->name), "%s", name);
         kernel->hash = hash;
         kernel->linked = 0;
+        kernel->rgba8 = native_rgba8_verified(source, name);
     }
     fclose(manifest);
     if (!d4r_native_swin_encoders_enabled())
@@ -733,9 +765,22 @@ static struct
 {
     CUfunction function;
     int native;
+    int rgba8;
 } output_kernels[OUTPUT_KERNEL_CAPACITY]; /* the latest lookups, newest last */
 static pthread_mutex_t output_kernel_lock = PTHREAD_MUTEX_INITIALIZER;
 static int last_output_native = -1;
+static int last_output_rgba8;
+/* Most NGX launches are network/input kernels. A monotonic membership filter lets those bypass the
+   output registry's lock and scan. Collisions only take the old lookup path; a bit is never cleared,
+   so function-handle reuse, feature recreation and concurrent lookups cannot hide an output kernel. */
+static uint64_t output_kernel_bits[16];
+
+static unsigned output_kernel_bit(CUfunction function)
+{
+    uint64_t value = (uint64_t)(uintptr_t)function >> 4;
+    value ^= value >> 17;
+    return (unsigned)((value * UINT64_C(0x9e3779b97f4a7c15)) >> 54);
+}
 
 static void note_function_lookup(CUfunction function, const char* name)
 {
@@ -755,20 +800,34 @@ static void note_function_lookup(CUfunction function, const char* name)
     memmove(output_kernels, output_kernels + 1, sizeof(output_kernels) - sizeof(output_kernels[0]));
     output_kernels[OUTPUT_KERNEL_CAPACITY - 1].function = function;
     output_kernels[OUTPUT_KERNEL_CAPACITY - 1].native = native;
+    int rgba8 = 0;
+    for (size_t i = 0; i < native_kernel_count; ++i)
+        if (strcmp(native_kernels[i].name, name) == 0) { rgba8 = native_kernels[i].rgba8; break; }
+    output_kernels[OUTPUT_KERNEL_CAPACITY - 1].rgba8 = native && rgba8;
     pthread_mutex_unlock(&output_kernel_lock);
+    const unsigned bit = output_kernel_bit(function);
+    __atomic_fetch_or(&output_kernel_bits[bit >> 6], UINT64_C(1) << (bit & 63), __ATOMIC_RELEASE);
 }
 
-static void note_launch(CUfunction function)
+/* Returns 1 + native + 2 * rgba8 when `function` is an output kernel, 0 otherwise. */
+static int note_launch(CUfunction function)
 {
+    const unsigned bit = output_kernel_bit(function);
+    if ((__atomic_load_n(&output_kernel_bits[bit >> 6], __ATOMIC_ACQUIRE) & (UINT64_C(1) << (bit & 63))) == 0)
+        return 0;
+    int kind = 0;
     pthread_mutex_lock(&output_kernel_lock);
     /* newest first, so a recycled handle resolves with its latest lookup */
     for (int i = OUTPUT_KERNEL_CAPACITY - 1; i >= 0 && output_kernels[i].function != NULL; --i)
         if (output_kernels[i].function == function)
         {
             __atomic_store_n(&last_output_native, output_kernels[i].native, __ATOMIC_RELAXED);
+            __atomic_store_n(&last_output_rgba8, output_kernels[i].rgba8, __ATOMIC_RELAXED);
+            kind = 1 + output_kernels[i].native + 2 * output_kernels[i].rgba8;
             break;
         }
     pthread_mutex_unlock(&output_kernel_lock);
+    return kind;
 }
 
 /* 1 when the last output kernel launched was d4r's native one, 0 when it was ZLUDA's compile of
@@ -778,6 +837,11 @@ int WINAPI d4rOutputKernelNative(void)
     return __atomic_load_n(&last_output_native, __ATOMIC_RELAXED);
 }
 
+int WINAPI d4rOutputKernelRgba8(void)
+{
+    return __atomic_load_n(&last_output_rgba8, __ATOMIC_RELAXED);
+}
+
 static void load_zluda(void)
 {
     /* Explicit experimental backend. Never silently mix module/context/array
@@ -964,7 +1028,7 @@ CUresult WINAPI d4rDilateMotionVectors(const D4rMotionDilationParams* parameters
     D4rMotionDilationParams copy = *parameters;
     void* arguments[] = {&copy};
     const uint64_t pixels = (uint64_t)copy.motion_width * copy.motion_height;
-    return launch(function, (unsigned)((pixels + 127) / 128), 1, 1, 128, 1, 1, 0, NULL, arguments, NULL);
+    return launch(function, (unsigned)((pixels + 127) / 128), 1, 1, 128, 1, 1, 0, d4r_ngx_stream(NULL), arguments, NULL);
 #else
     return CUDA_ERROR_NOT_SUPPORTED;
 #endif
@@ -1127,6 +1191,16 @@ static void ensure_context(void)
     pthread_once(&context_once, create_default_zluda_context);
 }
 
+/* D4R_NGX_STREAM_PRIORITY=high|normal (default off): the null-stream work of NGX, the shim and this bridge
+   (kernels, async copies and memsets, events, stream waits, input waits, completion callbacks) moves from
+   HIP's null stream to one blocking HIP stream. "high" creates it at the device's greatest priority, so its
+   HSA queue (and the hardware queue it maps to) dispatches ahead of normal-priority queues; "normal" creates
+   it at the default priority, the control that isolates the stream change itself. Every caller passes NULL
+   and gets the same stream, so the order of the work is unchanged; being blocking, the stream also stays
+   ordered with ZLUDA's own synchronous null-stream operations. Off: NULL stays HIP's null stream. */
+static CUstream d4r_ngx_stream(CUstream stream);
+static void* hip_symbol(const char* name);
+
 /* Opt-in stage instrumentation. With D4R_CUDA_LAUNCH_STATS=1 every
    cuLaunchKernel is followed by a context synchronize and a summary of each
    device allocation, surface object and texture object named in the packed
@@ -1402,15 +1476,17 @@ static CUresult write_redirect_tail(CUsurfObject object, CUdeviceptr pointer, ui
     static unsigned int next;
     unsigned char* tail = ring[next++ % 256];
     // dword 21: 0x5232 ('R2', RGBA16F texels) << 16 | pitch / 8 (0 = off); dwords 22-23: pointer.
-    // Row pitches are multiples of 8, so bit 0 of `pitch` selects 'R3': R10G10B10A2_UNORM texels.
-    const uint32_t word = pitch != 0 ? ((pitch & 1u) != 0 ? 0x52330000u : 0x52320000u) | ((pitch >> 3) & 0xffffu) : 0u;
+    // Row pitches are multiples of 8: bit 0 selects R3 (RGB10A2), bit 1 selects R4 (RGBA8).
+    const uint32_t word = pitch != 0 ? ((pitch & 2u) != 0 ? 0x52340000u :
+                                      (pitch & 1u) != 0 ? 0x52330000u : 0x52320000u) |
+                                      ((pitch >> 3) & 0xffffu) : 0u;
     memset(tail, 0, 16);
     memcpy(tail, &word, 4);
     memcpy(tail + 4, &pointer, 8);
     CUMEMCPYHTODASYNC_FN function = (CUMEMCPYHTODASYNC_FN)find_zluda_symbol("cuMemcpyHtoDAsync_v2");
     if (function == NULL)
         return CUDA_ERROR_NOT_SUPPORTED;
-    CUresult result = function((CUdeviceptr)object + 84, tail, 12, NULL);
+    CUresult result = function((CUdeviceptr)object + 84, tail, 12, d4r_ngx_stream(NULL));
     if (result == CUDA_SUCCESS && !async)
     {
         CUCTX_SYNCHRONIZE_FN sync = (CUCTX_SYNCHRONIZE_FN)find_zluda_symbol("cuCtxSynchronize");
@@ -2431,6 +2507,33 @@ CUresult WINAPI cuModuleGetFunction(CUfunction* function_out, CUmodule module, c
     return result;
 }
 
+/* D4R_SHIM_GPU_COMPLETION, early form: the shim arms a completion word before NGX's evaluate. When the output
+   kernel that follows is d4r's native one (which writes the game-visible buffer directly; RGBA8-capable when
+   the frame needs the packed store), the word is queued right behind that kernel, ahead of NGX's trailing
+   bookkeeping kernels, which the game never reads. Any other output kernel leaves it to the shim's own write
+   after the output copy. Only the CUDA worker evaluates, so the arm state needs no lock. */
+static CUdeviceptr ngx_completion_pointer;
+static uint32_t ngx_completion_value;
+static int ngx_completion_rgba8, ngx_completion_fired;
+
+CUresult WINAPI d4rNgxArmCompletion(CUdeviceptr pointer, uint32_t value, int rgba8)
+{
+    ngx_completion_pointer = pointer;
+    ngx_completion_value = value;
+    ngx_completion_rgba8 = rgba8;
+    ngx_completion_fired = 0;
+    return CUDA_SUCCESS;
+}
+
+/* 1 when the armed word was queued behind the native output kernel; disarms. */
+int WINAPI d4rNgxCompletionFired(void)
+{
+    const int fired = ngx_completion_fired;
+    ngx_completion_pointer = 0;
+    ngx_completion_fired = 0;
+    return fired;
+}
+
 CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x, unsigned int grid_y,
                                unsigned int grid_z, unsigned int block_x, unsigned int block_y,
                                unsigned int block_z, unsigned int shared_bytes, CUstream stream,
@@ -2439,13 +2542,11 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
     static CULAUNCHKERNEL_FN function;
     if (function == NULL)
         function = (CULAUNCHKERNEL_FN)find_zluda_symbol("cuLaunchKernel");
-    unsigned int sequence = 0;
-    pthread_mutex_lock(&instrumentation_lock);
-    sequence = ++launch_sequence;
-    pthread_mutex_unlock(&instrumentation_lock);
+    stream = d4r_ngx_stream(stream);
+    const unsigned int sequence = __atomic_add_fetch(&launch_sequence, 1, __ATOMIC_RELAXED);
     replay_dump_launch(sequence, function_handle, grid_x, grid_y, grid_z, block_x, block_y, block_z,
                        shared_bytes, extra);
-    note_launch(function_handle);
+    const int output_kind = note_launch(function_handle);
     PendingKernelProfile* profile = function != NULL
         ? begin_kernel_profile(sequence, function_handle, stream) : NULL;
     CUfunction native_function = microcuda_k_lookup(function_handle);
@@ -2468,6 +2569,17 @@ CUresult WINAPI cuLaunchKernel(CUfunction function_handle, unsigned int grid_x,
         tracef("microcuda-k prep stats runs=%llu skips=%llu", (unsigned long long)prep, (unsigned long long)skips);
     }
     end_kernel_profile(profile, result, stream);
+    if (result == CUDA_SUCCESS && ngx_completion_pointer != 0 && output_kind != 0)
+    {
+        const int native = ((output_kind - 1) & 1) != 0, rgba8 = (output_kind - 1) >= 2;
+        typedef int (*WRITE_FN)(void*, void*, uint32_t, unsigned int);
+        static WRITE_FN write;
+        if (write == NULL)
+            write = (WRITE_FN)hip_symbol("hipStreamWriteValue32");
+        if (native && (!ngx_completion_rgba8 || rgba8) && write != NULL)
+            ngx_completion_fired = write(stream, (void*)(uintptr_t)ngx_completion_pointer, ngx_completion_value, 0) == 0;
+        ngx_completion_pointer = 0; /* one output kernel per evaluate: never fire on a later one */
+    }
     if (result != CUDA_SUCCESS || trace_verbose())
     {
         tracef("cuLaunchKernel[%u] %s", sequence, find_function_name(function_handle));
@@ -2569,6 +2681,7 @@ CUresult WINAPI cuMemcpyHtoDAsync(CUdeviceptr destination, const void* source, s
 {
     CUMEMCPYHTODASYNC_FN function = (CUMEMCPYHTODASYNC_FN)find_zluda_symbol("cuMemcpyHtoDAsync_v2");
     if (microcuda_k_write) microcuda_k_write(destination, bytes);
+    stream = d4r_ngx_stream(stream);
     CUresult result = function != NULL ? function(destination, source, bytes, stream) : missing("cuMemcpyHtoDAsync_v2");
     TRACE_CALL(result, "cuMemcpyHtoDAsync->v2 dst=0x%llx src=%p bytes=%zu stream=%p result=%d",
            (unsigned long long)destination, source, bytes, stream, result);
@@ -2906,7 +3019,6 @@ CUresult WINAPI cuTexObjectGetResourceDesc(void* descriptor, CUtexObject object)
     return result;
 }
 
-
 /* Direct pass-throughs for driver APIs whose arguments are all integers or
    pointers, so the Windows x64 call forwards unchanged to ZLUDA (SysV). Unversioned
    names resolve to the versions the CUDA 12 driver returns for them. */
@@ -2935,8 +3047,8 @@ static int d4r_forward_is_teardown(const char* target)
 D4R_FORWARD(cuEventCreate, cuEventCreate, 2, (D4rArg a0, D4rArg a1), (a0, a1))
 D4R_FORWARD(cuEventDestroy, cuEventDestroy_v2, 1, (D4rArg a0), (a0))
 D4R_FORWARD(cuEventDestroy_v2, cuEventDestroy_v2, 1, (D4rArg a0), (a0))
-D4R_FORWARD(cuEventRecord, cuEventRecord, 2, (D4rArg a0, D4rArg a1), (a0, a1))
-D4R_FORWARD(cuEventRecordWithFlags, cuEventRecordWithFlags, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
+D4R_FORWARD(cuEventRecord, cuEventRecord, 2, (D4rArg a0, D4rArg a1), (a0, (D4rArg)d4r_ngx_stream((CUstream)a1)))
+D4R_FORWARD(cuEventRecordWithFlags, cuEventRecordWithFlags, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, (D4rArg)d4r_ngx_stream((CUstream)a1), a2))
 CUresult WINAPI cuEventSynchronize(D4rArg a0)
 {
     if (before_teardown())
@@ -2995,9 +3107,9 @@ D4R_FORWARD(cuStreamCreate, cuStreamCreate, 2, (D4rArg a0, D4rArg a1), (a0, a1))
 D4R_FORWARD(cuStreamCreateWithPriority, cuStreamCreateWithPriority, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
 D4R_FORWARD(cuStreamDestroy, cuStreamDestroy_v2, 1, (D4rArg a0), (a0))
 D4R_FORWARD(cuStreamDestroy_v2, cuStreamDestroy_v2, 1, (D4rArg a0), (a0))
-D4R_FORWARD(cuStreamSynchronize, cuStreamSynchronize, 1, (D4rArg a0), (a0))
-D4R_FORWARD(cuStreamQuery, cuStreamQuery, 1, (D4rArg a0), (a0))
-D4R_FORWARD(cuStreamWaitEvent, cuStreamWaitEvent, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
+D4R_FORWARD(cuStreamSynchronize, cuStreamSynchronize, 1, (D4rArg a0), ((D4rArg)d4r_ngx_stream((CUstream)a0)))
+D4R_FORWARD(cuStreamQuery, cuStreamQuery, 1, (D4rArg a0), ((D4rArg)d4r_ngx_stream((CUstream)a0)))
+D4R_FORWARD(cuStreamWaitEvent, cuStreamWaitEvent, 3, (D4rArg a0, D4rArg a1, D4rArg a2), ((D4rArg)d4r_ngx_stream((CUstream)a0), a1, a2))
 D4R_FORWARD(cuStreamGetFlags, cuStreamGetFlags, 2, (D4rArg a0, D4rArg a1), (a0, a1))
 D4R_FORWARD(cuStreamGetPriority, cuStreamGetPriority, 2, (D4rArg a0, D4rArg a1), (a0, a1))
 D4R_FORWARD(cuCtxGetCurrent, cuCtxGetCurrent, 1, (D4rArg a0), (a0))
@@ -3014,14 +3126,14 @@ D4R_FORWARD(cuMemGetInfo_v2, cuMemGetInfo_v2, 2, (D4rArg a0, D4rArg a1), (a0, a1
 D4R_FORWARD(cuMemcpyHtoD, cuMemcpyHtoD_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
 D4R_FORWARD(cuMemcpyHtoD_v2, cuMemcpyHtoD_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
 D4R_FORWARD(cuMemcpyDtoD, cuMemcpyDtoD_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
-D4R_FORWARD(cuMemcpyDtoDAsync, cuMemcpyDtoDAsync_v2, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, a3))
-D4R_FORWARD(cuMemcpyDtoHAsync, cuMemcpyDtoHAsync_v2, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, a3))
+D4R_FORWARD(cuMemcpyDtoDAsync, cuMemcpyDtoDAsync_v2, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, (D4rArg)d4r_ngx_stream((CUstream)a3)))
+D4R_FORWARD(cuMemcpyDtoHAsync, cuMemcpyDtoHAsync_v2, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, (D4rArg)d4r_ngx_stream((CUstream)a3)))
 D4R_FORWARD(cuMemsetD8, cuMemsetD8_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
 D4R_FORWARD(cuMemsetD16, cuMemsetD16_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
 D4R_FORWARD(cuMemsetD32, cuMemsetD32_v2, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
-D4R_FORWARD(cuMemsetD8Async, cuMemsetD8Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, a3))
-D4R_FORWARD(cuMemsetD16Async, cuMemsetD16Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, a3))
-D4R_FORWARD(cuMemsetD32Async, cuMemsetD32Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, a3))
+D4R_FORWARD(cuMemsetD8Async, cuMemsetD8Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, (D4rArg)d4r_ngx_stream((CUstream)a3)))
+D4R_FORWARD(cuMemsetD16Async, cuMemsetD16Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, (D4rArg)d4r_ngx_stream((CUstream)a3)))
+D4R_FORWARD(cuMemsetD32Async, cuMemsetD32Async, 4, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3), (a0, a1, a2, (D4rArg)d4r_ngx_stream((CUstream)a3)))
 D4R_FORWARD(cuMemsetD2D8, cuMemsetD2D8_v2, 5, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3, D4rArg a4), (a0, a1, a2, a3, a4))
 D4R_FORWARD(cuMemsetD2D32, cuMemsetD2D32_v2, 5, (D4rArg a0, D4rArg a1, D4rArg a2, D4rArg a3, D4rArg a4), (a0, a1, a2, a3, a4))
 D4R_FORWARD(cuFuncSetAttribute, cuFuncSetAttribute, 3, (D4rArg a0, D4rArg a1, D4rArg a2), (a0, a1, a2))
@@ -3116,6 +3228,59 @@ static void* hip_symbol(const char* name)
     return hip != NULL ? dlsym(hip, name) : NULL;
 }
 
+static CUstream ngx_priority_stream;
+static pthread_once_t ngx_priority_once = PTHREAD_ONCE_INIT;
+
+static void create_ngx_priority_stream(void)
+{
+    const char* setting = getenv("D4R_NGX_STREAM_PRIORITY");
+    const int high = setting != NULL && strcmp(setting, "high") == 0;
+    if (!high && (setting == NULL || strcmp(setting, "normal") != 0))
+        return;
+    typedef int (*STREAM_PRIORITY_RANGE_FN)(int*, int*);
+    typedef int (*STREAM_CREATE_PRIORITY_FN)(void**, unsigned int, int);
+    STREAM_PRIORITY_RANGE_FN range = (STREAM_PRIORITY_RANGE_FN)hip_symbol("hipDeviceGetStreamPriorityRange");
+    STREAM_CREATE_PRIORITY_FN create = (STREAM_CREATE_PRIORITY_FN)hip_symbol("hipStreamCreateWithPriority");
+    int least = 0, greatest = 0;
+    ensure_context();
+    if (context_setup_result != CUDA_SUCCESS || range == NULL || create == NULL || range(&least, &greatest) != 0)
+    {
+        tracef("D4R_NGX_STREAM_PRIORITY=%s: HIP stream priorities unavailable; NGX stays on the null stream", setting);
+        return;
+    }
+    void* stream = NULL;
+    const int priority = high ? greatest : 0;
+    const int result = create(&stream, 0 /* hipStreamDefault: blocking */, priority);
+    tracef("D4R_NGX_STREAM_PRIORITY=%s: NGX stream %p priority %d (range %d..%d) result=%d", setting, stream,
+           priority, least, greatest, result);
+    if (result == 0)
+        ngx_priority_stream = (CUstream)stream;
+}
+
+static CUstream d4r_ngx_stream(CUstream stream)
+{
+    if (stream != NULL)
+        return stream;
+    pthread_once(&ngx_priority_once, create_ngx_priority_stream);
+    return ngx_priority_stream; /* NULL unless D4R_NGX_STREAM_PRIORITY created one */
+}
+
+CUresult WINAPI d4rNgxStreamPriority(int* priority)
+{
+    /* For the shim's log: 1 when NGX runs on the substituted stream, with its HIP priority. */
+    typedef int (*STREAM_GET_PRIORITY_FN)(void*, int*);
+    CUstream stream = d4r_ngx_stream(NULL);
+    STREAM_GET_PRIORITY_FN get = (STREAM_GET_PRIORITY_FN)hip_symbol("hipStreamGetPriority");
+    if (priority == NULL)
+        return CUDA_ERROR_INVALID_VALUE;
+    *priority = 0;
+    if (stream == NULL)
+        return CUDA_ERROR_NOT_SUPPORTED;
+    if (get != NULL)
+        get(stream, priority);
+    return CUDA_SUCCESS;
+}
+
 CUresult WINAPI d4rImportVulkanMemory(void* client_device, uint64_t client_memory, uint64_t bytes, CUdeviceptr* pointer,
                                       void** memory)
 {
@@ -3282,6 +3447,146 @@ typedef int (*VK_SIGNAL_SEMAPHORE_FN)(void*, const D4rVkSemaphoreSignalInfo*);
 typedef int (*VK_GET_SEMAPHORE_COUNTER_FN)(void*, uint64_t, uint64_t*);
 typedef int (*HIP_LAUNCH_HOST_FN)(void*, void (*)(void*), void*);
 
+/* NGX's null-stream completion, independent of Wine scheduling/event polling. CPU cancellations
+   and native callbacks share one timeline lock so a late callback never signals a lower value.
+   No HIP or Wine APIs may run from the callback. The shim queues this only for the completed-prefix
+   head, after every output write; resource retirement waits for its native completion condition. */
+typedef struct
+{
+    void* device;
+    uint64_t semaphore, value, pending;
+    VK_SIGNAL_SEMAPHORE_FN signal;
+    HIP_LAUNCH_HOST_FN launch;
+    pthread_mutex_t lock;
+    pthread_cond_t completed;
+    int busy, cancelled, detached;
+} D4rNgxTimeline;
+
+int WINAPI d4rNgxTimelineSignal(void* handle, uint64_t value)
+{
+    D4rNgxTimeline* timeline = handle;
+    pthread_mutex_lock(&timeline->lock);
+    int result = 0;
+    if (value > timeline->value)
+    {
+        D4rVkSemaphoreSignalInfo info = {1000207005, NULL, timeline->semaphore, value};
+        result = timeline->signal(timeline->device, &info);
+        if (result == 0) timeline->value = value;
+    }
+    pthread_mutex_unlock(&timeline->lock);
+    return result;
+}
+
+static void d4r_ngx_completed(void* data)
+{
+    D4rNgxTimeline* timeline = data;
+    pthread_mutex_lock(&timeline->lock);
+    if (!__atomic_load_n(&timeline->cancelled, __ATOMIC_ACQUIRE) && timeline->pending > timeline->value)
+    {
+        D4rVkSemaphoreSignalInfo info = {1000207005, NULL, timeline->semaphore, timeline->pending};
+        if (timeline->signal(timeline->device, &info) == 0) timeline->value = timeline->pending;
+        /* A failed signal leaves the worker's normal retirement to retry. */
+    }
+    timeline->busy = 0;
+    pthread_cond_signal(&timeline->completed);
+    const int detached = timeline->detached;
+    pthread_mutex_unlock(&timeline->lock);
+    if (detached) { pthread_cond_destroy(&timeline->completed); pthread_mutex_destroy(&timeline->lock); free(timeline); }
+}
+
+CUresult WINAPI d4rNgxTimelineCreate(void* client_device, uint64_t client_semaphore, void** handle)
+{
+    if (client_device == NULL || client_semaphore == 0 || handle == NULL) return CUDA_ERROR_INVALID_VALUE;
+    const uint64_t* device_object = (const uint64_t*)(uintptr_t)((const uint64_t*)client_device)[1];
+    const uint64_t* semaphore_object = (const uint64_t*)(uintptr_t)client_semaphore;
+    if (device_object == NULL || device_object[1] != (uint64_t)(uintptr_t)client_device ||
+        semaphore_object[1] != client_semaphore) return CUDA_ERROR_NOT_SUPPORTED;
+    void* vulkan = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_NOLOAD);
+    VK_GET_DEVICE_PROC_ADDR_FN get = vulkan != NULL ? dlsym(vulkan, "vkGetDeviceProcAddr") : NULL;
+    HIP_LAUNCH_HOST_FN launch = (HIP_LAUNCH_HOST_FN)hip_symbol("hipLaunchHostFunc");
+    if (get == NULL || launch == NULL) return CUDA_ERROR_NOT_SUPPORTED;
+    void* device = (void*)(uintptr_t)device_object[0];
+    VK_SIGNAL_SEMAPHORE_FN signal = (VK_SIGNAL_SEMAPHORE_FN)get(device, "vkSignalSemaphore");
+    VK_GET_SEMAPHORE_COUNTER_FN counter = (VK_GET_SEMAPHORE_COUNTER_FN)get(device, "vkGetSemaphoreCounterValue");
+    uint64_t value = 0;
+    if (signal == NULL || counter == NULL || counter(device, semaphore_object[0], &value) != 0)
+        return CUDA_ERROR_NOT_SUPPORTED;
+    D4rNgxTimeline* timeline = calloc(1, sizeof(*timeline));
+    if (timeline == NULL) return CUDA_ERROR_OUT_OF_MEMORY;
+    if (pthread_mutex_init(&timeline->lock, NULL) != 0) { free(timeline); return CUDA_ERROR_UNKNOWN; }
+    pthread_condattr_t attr;
+    if (pthread_condattr_init(&attr) != 0) { pthread_mutex_destroy(&timeline->lock); free(timeline); return CUDA_ERROR_UNKNOWN; }
+    const int condition = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) != 0 ||
+                          pthread_cond_init(&timeline->completed, &attr) != 0;
+    pthread_condattr_destroy(&attr);
+    if (condition) { pthread_mutex_destroy(&timeline->lock); free(timeline); return CUDA_ERROR_UNKNOWN; }
+    timeline->device = device; timeline->semaphore = semaphore_object[0]; timeline->value = value;
+    timeline->signal = signal; timeline->launch = launch; *handle = timeline;
+    return CUDA_SUCCESS;
+}
+
+CUresult WINAPI d4rNgxTimelinePrepare(void* handle)
+{
+    D4rNgxTimeline* timeline = handle;
+    pthread_mutex_lock(&timeline->lock);
+    const int busy = timeline->busy;
+    if (!busy) __atomic_store_n(&timeline->cancelled, 0, __ATOMIC_RELEASE);
+    pthread_mutex_unlock(&timeline->lock);
+    return busy ? CUDA_ERROR_NOT_READY : CUDA_SUCCESS;
+}
+
+void WINAPI d4rNgxTimelineCancel(void* handle)
+{
+    /* Called by the independent input watchdog before its coherent release store: no queue calls/locks. */
+    __atomic_store_n(&((D4rNgxTimeline*)handle)->cancelled, 1, __ATOMIC_RELEASE);
+}
+
+CUresult WINAPI d4rNgxTimelineQueue(void* handle, uint64_t value)
+{
+    D4rNgxTimeline* timeline = handle;
+    pthread_mutex_lock(&timeline->lock);
+    if (timeline->busy) { pthread_mutex_unlock(&timeline->lock); return CUDA_ERROR_NOT_READY; }
+    timeline->pending = value; timeline->busy = 1;
+    pthread_mutex_unlock(&timeline->lock);
+    const int result = timeline->launch(d4r_ngx_stream(NULL), d4r_ngx_completed, timeline);
+    if (result != 0)
+    {
+        pthread_mutex_lock(&timeline->lock); timeline->busy = 0; pthread_mutex_unlock(&timeline->lock);
+    }
+    return result == 0 ? CUDA_SUCCESS : CUDA_ERROR_NOT_SUPPORTED;
+}
+
+void WINAPI d4rNgxTimelineDestroy(void* handle)
+{
+    D4rNgxTimeline* timeline = handle;
+    pthread_mutex_lock(&timeline->lock);
+    __atomic_store_n(&timeline->cancelled, 1, __ATOMIC_RELEASE);
+    const int busy = timeline->busy;
+    timeline->detached = busy;
+    pthread_mutex_unlock(&timeline->lock);
+    /* Normally the native condition drained the callback. If a device error prevented that drain, retain
+       only this self-owned ticket until its cancelled callback returns, never a pointer into Feature. */
+    if (!busy) { pthread_cond_destroy(&timeline->completed); pthread_mutex_destroy(&timeline->lock); free(timeline); }
+}
+
+CUresult WINAPI d4rNgxTimelineWait(void* handle)
+{
+    D4rNgxTimeline* timeline = handle;
+    struct timespec deadline;
+    if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) return CUDA_ERROR_UNKNOWN;
+    deadline.tv_sec += 5;
+    pthread_mutex_lock(&timeline->lock);
+    int result = 0;
+    while (timeline->busy && result == 0)
+        result = pthread_cond_timedwait(&timeline->completed, &timeline->lock, &deadline);
+    const int busy = timeline->busy;
+    pthread_mutex_unlock(&timeline->lock);
+    /* The callback runs after every output write on the null stream. Its condition is sufficient to
+       retire those resources, without polling/recording another GPU event. A timeout keeps the ticket
+       owned; the caller cancels, disables this mode and falls back to its normal output synchronization. */
+    return !busy ? CUDA_SUCCESS : result == ETIMEDOUT ? CUDA_ERROR_NOT_READY : CUDA_ERROR_UNKNOWN;
+}
+
 typedef struct D4rEngineNet
 {
     int k_preset;  /* D4RKHIP1: the token grid comes from the output dimensions and the cycle has eight graphs */
@@ -3333,7 +3638,6 @@ typedef int (*HIP_CAPTURE_BEGIN_FN)(void*, int);
 typedef int (*HIP_CAPTURE_END_FN)(void*, void**);
 typedef int (*HIP_GRAPH_INSTANTIATE_FN)(void**, void*, void**, char*, size_t);
 
-
 CUresult WINAPI d4rEngineNetReleaseInput(void* handle, uint32_t value)
 {
     D4rEngineNet* net = (D4rEngineNet*)handle;
@@ -4175,7 +4479,7 @@ CUresult WINAPI d4rMemcpy2DAsync(const D4rMemcpy2D* copy, CUstream stream)
     D4rMemcpy2D hip = *copy;
     hip.srcMemoryType = (unsigned int)src;
     hip.dstMemoryType = (unsigned int)dst;
-    const int result = function(&hip, NULL);
+    const int result = function(&hip, d4r_ngx_stream(NULL));
     if (result != 0 || trace_verbose())
         tracef("d4rMemcpy2DAsync %zux%zu result=%d", copy->WidthInBytes, copy->Height, result);
     return result == 0 ? CUDA_SUCCESS : CUDA_ERROR_INVALID_VALUE;
@@ -4194,12 +4498,68 @@ CUresult WINAPI d4rStreamWaitValue32(CUdeviceptr pointer, uint32_t value)
         function = (HIP_STREAM_WAIT_VALUE32_FN)hip_symbol("hipStreamWaitValue32");
     if (function == NULL)
         return CUDA_ERROR_NOT_SUPPORTED;
-    const int result = function(NULL, (void*)(uintptr_t)pointer, value, 0 /* hipStreamWaitValueGte */, 0xffffffffu);
+    const int result = function(d4r_ngx_stream(NULL), (void*)(uintptr_t)pointer, value, 0 /* hipStreamWaitValueGte */,
+                                0xffffffffu);
     if (result != 0 || trace_verbose())
         tracef("d4rStreamWaitValue32 %p >= %u result=%d", (void*)(uintptr_t)pointer, value, result);
     return result == 0 ? CUDA_SUCCESS : CUDA_ERROR_INVALID_VALUE;
 }
 
+/* Exact per-slot tags, rather than the engine's monotonic Gte word: an old or later frame must never
+   satisfy NGX's wait, including across u32 wrap. ZLUDA uses HIP's null stream for null CUstream kernel
+   launches; d4rMemcpy2DAsync uses that same stream. This export therefore orders both input copies and
+   NGX kernels without adding an unsupported CUDA driver entry point to ZLUDA. The caller owns a coherent
+   CPU mapping and arms its watchdog before entering here, so a blocked HIP call can still be released. */
+CUresult WINAPI d4rStreamWaitValue32Eq(CUdeviceptr pointer, uint32_t value)
+{
+    typedef int (*GET_ATTRIBUTE_FN)(int*, int, int);
+    typedef int (*GET_DEVICE_FN)(int*);
+    ensure_context();
+    if (context_setup_result != CUDA_SUCCESS)
+        return context_setup_result;
+    HIP_STREAM_WAIT_VALUE32_FN wait = (HIP_STREAM_WAIT_VALUE32_FN)hip_symbol("hipStreamWaitValue32");
+    GET_ATTRIBUTE_FN attribute = (GET_ATTRIBUTE_FN)hip_symbol("hipDeviceGetAttribute");
+    GET_DEVICE_FN device = (GET_DEVICE_FN)hip_symbol("hipGetDevice");
+    int current = 0, supported = 0;
+    if (pointer == 0 || wait == NULL || attribute == NULL || device == NULL || device(&current) != 0 ||
+        attribute(&supported, 10013 /* hipDeviceAttributeCanUseStreamWaitValue */, current) != 0 || !supported)
+        return CUDA_ERROR_NOT_SUPPORTED;
+    const int result = wait(d4r_ngx_stream(NULL), (void*)(uintptr_t)pointer, value, 1 /* hipStreamWaitValueEq */,
+                            0xffffffffu);
+    if (result != 0 || trace_verbose())
+        tracef("d4rStreamWaitValue32Eq %p == %u result=%d", (void*)(uintptr_t)pointer, value, result);
+    return result == 0 ? CUDA_SUCCESS : CUDA_ERROR_NOT_SUPPORTED; /* failure: no wait queued */
+}
+
+double WINAPI d4rThreadCpuTimeMs(void)
+{
+    struct timespec resolution, now;
+    if (clock_getres(CLOCK_THREAD_CPUTIME_ID, &resolution) != 0 || resolution.tv_sec != 0 ||
+        resolution.tv_nsec > 100000 /* 100 us: too coarse for these sub-ms submission/wait costs */ ||
+        clock_gettime(CLOCK_THREAD_CPUTIME_ID, &now) != 0)
+        return -1.0;
+    return (double)now.tv_sec * 1000.0 + (double)now.tv_nsec / 1000000.0;
+}
+
+/* D4R_SHIM_GPU_COMPLETION: queues a 32-bit write of `value` to `pointer` on the stream NGX runs on, behind the
+   frame's output writes. The game's queue polls that word (kernels/support/vulkan_wait.comp) instead of waiting
+   for a CPU-signalled timeline value, so DLSS's completion reaches the game without a host round trip. */
+CUresult WINAPI d4rNgxWriteValue32(CUdeviceptr pointer, uint32_t value)
+{
+    static HIP_STREAM_WRITE_VALUE32_FN write;
+    ensure_context();
+    if (context_setup_result != CUDA_SUCCESS)
+        return context_setup_result;
+    if (write == NULL)
+        write = (HIP_STREAM_WRITE_VALUE32_FN)hip_symbol("hipStreamWriteValue32");
+    if (write == NULL || pointer == 0)
+        return CUDA_ERROR_NOT_SUPPORTED;
+    const int result = write(d4r_ngx_stream(NULL), (void*)(uintptr_t)pointer, value, 0);
+    if (result != 0 || trace_verbose())
+        tracef("d4rNgxWriteValue32 %p = %u result=%d", (void*)(uintptr_t)pointer, value, result);
+    return result == 0 ? CUDA_SUCCESS : CUDA_ERROR_NOT_SUPPORTED;
+}
+
 CUresult WINAPI d4rWriteValue32(CUdeviceptr pointer, uint32_t value)
 {
     static HIP_STREAM_WRITE_VALUE32_FN write;

tools/wine_nvcuda_bridge.spec

@@ -104,10 +104,24 @@
 @ stdcall d4rSetShutdownHook(ptr) d4rSetShutdownHook
 @ stdcall d4rEventSynchronize(ptr) d4rEventSynchronize
 @ stdcall d4rStreamWaitValue32(int64 long) d4rStreamWaitValue32
+@ stdcall d4rStreamWaitValue32Eq(int64 long) d4rStreamWaitValue32Eq
+@ stdcall d4rNgxStreamPriority(ptr) d4rNgxStreamPriority
+@ stdcall d4rNgxWriteValue32(int64 long) d4rNgxWriteValue32
+@ stdcall d4rNgxArmCompletion(int64 long long) d4rNgxArmCompletion
+@ stdcall d4rNgxCompletionFired() d4rNgxCompletionFired
+@ stdcall d4rThreadCpuTimeMs() d4rThreadCpuTimeMs
 @ stdcall d4rWriteValue32(int64 long) d4rWriteValue32
 @ stdcall d4rSetArrayRedirect(ptr int64 long) d4rSetArrayRedirect
 @ stdcall d4rRegisterLinearTexture(int64 int64 int64 long long) d4rRegisterLinearTexture
 @ stdcall d4rSetEnv(str str long) d4rSetEnv
 @ stdcall d4rLoadError() d4rLoadError
 @ stdcall d4rOutputKernelNative() d4rOutputKernelNative
+@ stdcall d4rOutputKernelRgba8() d4rOutputKernelRgba8
+@ stdcall d4rNgxTimelineCreate(ptr int64 ptr) d4rNgxTimelineCreate
+@ stdcall d4rNgxTimelinePrepare(ptr) d4rNgxTimelinePrepare
+@ stdcall d4rNgxTimelineCancel(ptr) d4rNgxTimelineCancel
+@ stdcall d4rNgxTimelineQueue(ptr int64) d4rNgxTimelineQueue
+@ stdcall d4rNgxTimelineSignal(ptr int64) d4rNgxTimelineSignal
+@ stdcall d4rNgxTimelineDestroy(ptr) d4rNgxTimelineDestroy
+@ stdcall d4rNgxTimelineWait(ptr) d4rNgxTimelineWait
 @ stdcall d4rDilateMotionVectors(ptr) d4rDilateMotionVectors