Batch Windows DLSS GPU inputs and add deferred kernel diagnostics

tools/windows/cuda_driver_probe.cpp

@@ -80,6 +80,21 @@ int main(int argc, char** argv)
                 if (primary) api.cuDevicePrimaryCtxRelease_v2(device); else api.cuCtxDestroy_v2(context); } }
         } cleanup{api, context, device, primary};
         api.check(api.cuCtxSetCurrent(context), "cuCtxSetCurrent");
+        void* stream = nullptr;
+        using CreateStream = int(WINAPI*)(void**, unsigned);
+        using DestroyStream = int(WINAPI*)(void*);
+        DestroyStream destroyStream = nullptr;
+        if (std::getenv("D4R_DIAG_NONBLOCKING_STREAM")) {
+            const auto createStream = api.library.symbol<CreateStream>("cuStreamCreate");
+            destroyStream = api.library.symbol<DestroyStream>("cuStreamDestroy_v2");
+            api.check(createStream(&stream, 1), "cuStreamCreate(NON_BLOCKING)");
+        }
+        struct StreamCleanup {
+            void*& stream; DestroyStream destroy;
+            ~StreamCleanup() { if (stream && destroy) destroy(stream); }
+        } streamCleanup{stream, destroyStream};
+        const bool queued = std::getenv("D4R_DIAG_QUEUE_PTX") != nullptr;
+        if (queued && !stream) throw std::runtime_error("Queued PTX diagnostic requires D4R_DIAG_NONBLOCKING_STREAM=1");
         CUmodule module = nullptr;
         std::printf("STAGE cuModuleLoadData (PTX JIT)\n");
         api.check(api.cuModuleLoadData(&module, ptx), "cuModuleLoadData");
@@ -93,25 +108,30 @@ int main(int argc, char** argv)
         CUdeviceptr pointer = 0;
         api.check(api.cuMemAlloc_v2(&pointer, bytes), "cuMemAlloc_v2");
         struct MemCleanup { CudaApi& api; CUdeviceptr p; ~MemCleanup() { if (p) api.cuMemFree_v2(p); } } ac{api, pointer};
+        if (queued) api.check(api.cuMemcpyHtoD_v2(pointer, result.data(), bytes), "cuMemcpyHtoD_v2(queued sentinel)");
         for (unsigned i = 0; i < args.iterations; ++i) {
-            uint32_t n = nmax - i % 129u, seed = 0x12345678u + i * 31u;
-            std::fill(result.begin(), result.end(), 0xdeadbeefu);
-            api.check(api.cuMemcpyHtoD_v2(pointer, result.data(), bytes), "cuMemcpyHtoD_v2(sentinel)");
+            uint32_t n = queued ? nmax : nmax - i % 129u, seed = 0x12345678u + i * 31u;
+            if (!queued) {
+                std::fill(result.begin(), result.end(), 0xdeadbeefu);
+                api.check(api.cuMemcpyHtoD_v2(pointer, result.data(), bytes), "cuMemcpyHtoD_v2(sentinel)");
+            }
             void* parameters[] = {&pointer, &n, &seed};
             api.check(api.cuLaunchKernel(function, (n + 127u) / 128u, 1, 1, 128, 1, 1,
-                0, nullptr, parameters, nullptr), "cuLaunchKernel");
+                0, stream, parameters, nullptr), "cuLaunchKernel");
+            if (queued && i + 1 != args.iterations) continue;
             api.check(api.cuCtxSynchronize(), "cuCtxSynchronize");
             api.check(api.cuMemcpyDtoH_v2(result.data(), pointer, bytes), "cuMemcpyDtoH_v2");
             if (!verify(result, n, seed)) return 5;
         }
         api.check(api.cuMemFree_v2(pointer), "cuMemFree_v2"); ac.p = 0;
         api.check(api.cuModuleUnload(module), "cuModuleUnload"); mc.m = nullptr;
+        if (stream) { api.check(destroyStream(stream), "cuStreamDestroy_v2"); stream = nullptr; }
         api.check(api.cuCtxSetCurrent(nullptr), "cuCtxSetCurrent(NULL)");
         if (primary) api.check(api.cuDevicePrimaryCtxRelease_v2(device), "cuDevicePrimaryCtxRelease_v2");
         else api.check(api.cuCtxDestroy_v2(context), "cuCtxDestroy_v2");
         cleanup.context = nullptr;
-        std::printf("PASS CUDA architecture=gfx1201 context=%s iterations=%u guard_verified=1\n",
-            args.context.c_str(), args.iterations);
+        std::printf("PASS CUDA architecture=gfx1201 context=%s iterations=%u guard_verified=1 queued=%u\n",
+            args.context.c_str(), args.iterations, unsigned(queued));
         return 0;
     } catch (const std::exception& e) {
         std::fprintf(stderr, "FAIL CUDA %s\n", e.what());

tools/windows/cuda_image_api.h

@@ -47,6 +47,7 @@ struct ImageApi {
     D4R_IMAGE(cuArray3DGetDescriptor_v2, Array3DDescriptor*, Array);
     D4R_IMAGE(cuArrayDestroy, Array);
     D4R_IMAGE(cuMemcpy2D_v2, const Copy2D*);
+    D4R_IMAGE(cuMemcpy2DAsync_v2, const Copy2D*, void*);
     D4R_IMAGE(cuTexObjectCreate, Texture*, const ResourceDescriptor*, const TextureDescriptor*, const void*);
     D4R_IMAGE(cuTexObjectDestroy, Texture);
     D4R_IMAGE(cuTexObjectGetResourceDesc, ResourceDescriptor*, Texture);
@@ -131,13 +132,18 @@ public:
         copy.widthBytes = rowBytes_; copy.height = height_;
         api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(diagnostic array to host)");
     }
-    void upload_device(CUdeviceptr source, size_t pitch) {
+    void upload_device(CUdeviceptr source, size_t pitch, bool deferred = false) {
         if (pitch < rowBytes_) throw std::runtime_error("Device input pitch is smaller than its row");
         Copy2D copy{};
         copy.srcType = 2; copy.srcDevice = source; copy.srcPitch = pitch;
         copy.dstType = 3; copy.dstArray = array_;
         copy.widthBytes = rowBytes_; copy.height = height_;
-        api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(external VRAM to array)");
+        if (deferred) {
+            // The Windows ZLUDA backend and HIP conversion use the same
+            // legacy default stream. Caller must complete the whole input
+            // batch before NGX may consume arrays on its own streams.
+            api_.cuda.check(api_.cuMemcpy2DAsync_v2(&copy, nullptr), "cuMemcpy2DAsync_v2(external VRAM to array)");
+        } else api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(external VRAM to array)");
     }
     void download_device(CUdeviceptr destination, size_t pitch) {
         if (pitch < rowBytes_) throw std::runtime_error("Device output pitch is smaller than its row");

tools/windows/cuda_image_probe.cpp

@@ -30,22 +30,25 @@ int main(int argc, char** argv) {
         struct Cleanup { CudaApi& api; CUcontext context; ~Cleanup() { (void)api.cuCtxDestroy_v2(context); } } cleanup{cuda, context};
         d4r::cuda::ImageApi images(cuda);
         for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
-            for (unsigned channels : {1u, 2u, 4u}) for (bool surface : {false, true}) {
-                d4r::cuda::Image image(images, 17, 9, 16, channels, surface);
+            for (unsigned format : {16u, 32u}) for (unsigned channels : {1u, 2u, 4u}) for (bool surface : {false, true}) {
+                const unsigned halvesPerPixel = channels * (format == 16 ? 1 : 2);
+                d4r::cuda::Image image(images, 17, 9, format, channels, surface);
                 image.verify_wrong_object_kind();
-                std::vector<uint16_t> input(17 * 9 * channels), output(input.size());
+                std::vector<uint16_t> input(17 * 9 * halvesPerPixel), output(input.size());
                 for (size_t i = 0; i < input.size(); ++i) input[i] = uint16_t((i + iteration) % 8 * 0x100);
                 image.upload(input.data());
                 image.download(output.data());
                 if (input != output) throw std::runtime_error("CUDA array storage roundtrip mismatch");
-                const size_t rowBytes = 17 * channels * sizeof(uint16_t), pitch = (rowBytes + 255) & ~size_t(255);
+                const size_t rowBytes = 17 * halvesPerPixel * sizeof(uint16_t), pitch = (rowBytes + 255) & ~size_t(255);
                 CUdeviceptr devicePixels = 0;
                 cuda.check(cuda.cuMemAlloc_v2(&devicePixels, pitch * 9), "cuMemAlloc_v2(pitched image regression)");
                 struct PixelsCleanup { CudaApi& cuda; CUdeviceptr pointer; ~PixelsCleanup() { (void)cuda.cuMemFree_v2(pointer); } } pixelsCleanup{cuda, devicePixels};
                 std::vector<uint8_t> padded(pitch * 9, 0xcd);
-                for (unsigned y = 0; y < 9; ++y) std::memcpy(padded.data() + y * pitch, input.data() + y * 17 * channels, rowBytes);
+                for (unsigned y = 0; y < 9; ++y) std::memcpy(padded.data() + y * pitch, input.data() + y * 17 * halvesPerPixel, rowBytes);
                 cuda.check(cuda.cuMemcpyHtoD_v2(devicePixels, padded.data(), padded.size()), "cuMemcpyHtoD_v2(pitched pattern)");
-                image.upload_device(devicePixels, pitch);
+                const bool deferred = std::getenv("D4R_DIAG_ASYNC_ARRAY_COPY") != nullptr;
+                image.upload_device(devicePixels, pitch, deferred);
+                if (deferred) cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize(input batch regression)");
                 image.download(output.data());
                 if (input != output) throw std::runtime_error("CUDA device-to-array pitched copy mismatch");
                 std::fill(padded.begin(), padded.end(), 0xee);
@@ -53,7 +56,7 @@ int main(int argc, char** argv) {
                 image.download_device(devicePixels, pitch);
                 cuda.check(cuda.cuMemcpyDtoH_v2(padded.data(), devicePixels, padded.size()), "cuMemcpyDtoH_v2(device array output)");
                 for (unsigned y = 0; y < 9; ++y) {
-                    if (std::memcmp(padded.data() + y * pitch, input.data() + y * 17 * channels, rowBytes))
+                    if (std::memcmp(padded.data() + y * pitch, input.data() + y * 17 * halvesPerPixel, rowBytes))
                         throw std::runtime_error("CUDA array-to-device pitched copy mismatch");
                     if (!std::all_of(padded.begin() + y * pitch + rowBytes, padded.begin() + (y + 1) * pitch, [](uint8_t value) { return value == 0xee; }))
                         throw std::runtime_error("CUDA array-to-device copy overwrote pitch padding");
@@ -61,7 +64,8 @@ int main(int argc, char** argv) {
             }
         }
         cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize");
-        std::printf("PASS CUDA_IMAGES architecture=gfx1201 iterations=%u descriptor_and_storage=1 pitched_device_copies=1\n", args.iterations);
+        std::printf("PASS CUDA_IMAGES architecture=gfx1201 iterations=%u descriptor_and_storage=1 pitched_device_copies=1 fp16_fp32=1 async_array_copy=%u\n",
+            args.iterations, unsigned(std::getenv("D4R_DIAG_ASYNC_ARRAY_COPY") != nullptr));
         return 0;
     } catch (const std::exception& e) {
         std::fprintf(stderr, "FAIL CUDA_IMAGES %s\n", e.what());

tools/windows/ngx_windows_runtime.h

@@ -112,6 +112,8 @@ struct Runtime {
             cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain");
             current();
             timeline = std::make_unique<SharedTimeline>(external, d3d);
+            std::printf("D4R_INTEROP async=%u batch_inputs=%u cpu_image_copies=0\n",
+                unsigned(std::getenv("D4R_ASYNC_INTEROP") != nullptr), unsigned(std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr));
             // NGX's loader can probe CUDA during DLL initialization: establish
             // the correct primary context before loading either NVIDIA DLL.
             nvapi = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NVAPI_DLL")));
@@ -386,6 +388,10 @@ private:
             report_completed_copies();
         }
         if (selected) {
+            // The fence can advance between report_completed_copies() and
+            // slot selection. Collect that now-complete ticket before Reset
+            // and before submitted() overwrites its timing identity.
+            if (selected->timing) selected->timing->report(queue_.Get(), this);
             dx(selected->input_allocator->Reset(), "Async input allocator Reset");
             dx(selected->output_allocator->Reset(), "Async output allocator Reset");
             dx(selected->input->Reset(selected->input_allocator.Get(), nullptr), "Async input list Reset");
@@ -580,16 +586,23 @@ private:
         if (ticket) rt_->timeline->wait_ready(ticket->input); else rt_->timeline->wait_input(queue);
         rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input producer completion");
         mark("input_fence_wait");
+        const bool batchInputs = std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr;
         for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
             auto& plane = planes_[i];
             void* data = plane.shared->mapped; uint64_t pitch = plane.shared->footprint.Footprint.RowPitch;
             if (plane.canonical) {
-                rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
+                if (batchInputs) rt_->pixels->convert_deferred(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
+                    unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
+                else rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
                     unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
                 data = plane.canonical->data; pitch = plane.canonical->pitch;
             }
-            plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch);
+            plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch, batchInputs);
         }
+        // HIP conversion and CUDA array copies are ordered on the shared
+        // legacy default stream. Retain an all-stream completion before NGX,
+        // including any internal nonblocking streams it might choose to use.
+        if (batchInputs) rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input batch completion");
         mark("input_conversion_array_upload");
         if (std::getenv("D4R_INTEROP_VERIFY")) {
             for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {

tools/windows/pixel_conversion.h

@@ -77,15 +77,19 @@ public:
         hip_.check(hip_.hipMemcpy(result.data(), counters_, sizeof(result), hipMemcpyDeviceToHost), "Read validation counters");
         return result;
     }
-    void convert(bool encode, const void* source, uint64_t sourcePitch, void* dest, uint64_t destPitch,
+    void convert_deferred(bool encode, const void* source, uint64_t sourcePitch, void* dest, uint64_t destPitch,
         unsigned width, unsigned height, pixel::Storage format, unsigned plane) {
         if (!width || !height || uint64_t(width) * height > UINT32_MAX) throw std::runtime_error("Invalid GPU conversion dimensions");
         unsigned storage = unsigned(format);
         void* args[] = {&source, &sourcePitch, &dest, &destPitch, &width, &height, &storage, &plane};
         hip_.check(hip_.hipModuleLaunchKernel(encode ? encode_ : decode_, unsigned((uint64_t(width) * height + 255) / 256), 1, 1,
             256, 1, 1, 0, nullptr, args, nullptr), encode ? "GPU pixel encode" : "GPU pixel decode");
-        // CUDA may use other nonblocking streams. Establish completion before
-        // giving the converted allocation to that API; images remain in VRAM.
+    }
+    void convert(bool encode, const void* source, uint64_t sourcePitch, void* dest, uint64_t destPitch,
+        unsigned width, unsigned height, pixel::Storage format, unsigned plane) {
+        convert_deferred(encode, source, sourcePitch, dest, destPitch, width, height, format, plane);
+        // CUDA may use other nonblocking streams. Standalone conversions must
+        // establish completion before handing the allocation to that API.
         hip_.check(hip_.hipDeviceSynchronize(), "GPU conversion completion");
     }
 };

tools/windows/profile_report.py

@@ -1,4 +1,4 @@
-"""Summarize native Windows d4r CPU stages and optional serializing HIP events.
+"""Summarize native Windows d4r CPU stages and optional HIP event diagnostics.
 
 CPU completion time is not GPU execution time. HIP occupancy API predictions
 are not hardware counters. No bandwidth / WMMA utilization is inferred here.
@@ -83,6 +83,8 @@ def report(directory, metadata_directory=None):
     kernels = collections.defaultdict(list)
     hook_threads = collections.defaultdict(list)
     apis = collections.defaultdict(list)
+    sample_skips = []
+    invalid_samples = []
     boundaries = collections.defaultdict(list)
     replay = collections.defaultdict(lambda: collections.defaultdict(dict))
     for path in sorted(directory.glob('*.log')):
@@ -115,13 +117,25 @@ def report(directory, metadata_directory=None):
                                                    'access_sum_ms', 'capture_sum_ms', 'driver_sum_ms')):
                     hook_threads[fields['thread']].append(fields)
             if not line.startswith('D4R_KERNEL_PROFILE '):
+                if line.startswith('D4R_KERNEL_PROFILE_SKIPPED '):
+                    sample_skips.append(dict(re.findall(r'(\w+)=(\S+)', line)))
+                if line.startswith('D4R_KERNEL_PROFILE_INVALID '):
+                    invalid_samples.append(dict(re.findall(r'(\w+)=("[^"]+"|\S+)', line)))
                 continue
             fields = dict(re.findall(r'(\w+)=("[^"]+"|\S+)', line))
-            kernels[(fields['kernel'].strip('"'), fields['backend'], fields['phase'])].append(fields)
+            required = ('kernel', 'backend', 'phase', 'gpu_ms', 'enqueue_ms', 'completion_ms',
+                        'grid', 'block', 'static_lds_bytes', 'dynamic_lds_bytes', 'regs', 'private_bytes',
+                        'max_threads', 'predicted_blocks_per_multiprocessor', 'serializing')
+            if not all(key in fields for key in required):
+                continue  # A bounded process stop can leave a partial final line.
+            kernels[(fields['kernel'].strip('"'), fields['backend'], fields['phase'],
+                     fields['serializing'])].append(fields)
     result = dict(cpuStages={name: stats(values) for name, values in stages.items()},
                   commandStages={name: stats(values) for name, values in commands.items()},
                   commandHookSamples=[],
                   cudaApi=[],
+                  kernelSampleSkips=sample_skips,
+                  invalidKernelSamples=invalid_samples,
                   gpuBoundaryStages={},
                   presentMon=presentmon(directory),
                   replayPairs=[],
@@ -134,7 +148,8 @@ def report(directory, metadata_directory=None):
                       'Command record intervals are between NGX recording calls on one thread, not Present/FPS.',
                       'Hook estimates use random one-in-64 samples; driver timing covers generated forwarding methods only.',
                       'Hook access includes lock waits; summing threads does not measure serial frame latency or CPU execution time.',
-                      'HIP-event profiling synchronizes every sampled launch and changes scheduling.',
+                      'Serializing HIP-event profiles wait after every launch. Deferred profiles query ready events and never add completion waits; timing events still add overhead.',
+                      'Deferred completionMs is host collection lag, not kernel completion latency. Legacy-stream sampling requires an opt-in because its timing markers retain legacy cross-stream ordering.',
                       'Occupancy is an API prediction, not a measured hardware counter.',
                       'Bandwidth and WMMA utilization require additional supported hardware tooling.'])
     boundary_stages = collections.defaultdict(list)
@@ -178,8 +193,11 @@ def report(directory, metadata_directory=None):
             accessMeanUs=1000 * sums['access_sum_ms'] / sums['samples'],
             captureMeanUsPerSample=1000 * sums['capture_sum_ms'] / sums['samples'],
             estimatedAccessCaptureMs=estimate, estimatedAccessCaptureMsPerSecond=1000 * estimate / sums['period_ms']))
-    for (name, backend, phase), rows in kernels.items():
+    for (name, backend, phase, serializing), rows in kernels.items():
         item = dict(kernel=name, backend=backend, phase=phase,
+                    serializing=serializing == '1',
+                    legacyStream=any(row.get('legacy_stream') == '1' for row in rows),
+                    completionMeaning='host_completion_wait' if serializing == '1' else 'host_collection_lag',
                     gpuMs=stats([float(row['gpu_ms']) for row in rows]),
                     enqueueMs=stats([float(row['enqueue_ms']) for row in rows]),
                     completionMs=stats([float(row['completion_ms']) for row in rows]),
@@ -224,7 +242,7 @@ def main():
               f"candidate_median={row['candidateGpuMs']['median']:.6f} ms ratio_median={row['candidateToControl']['median']:.6f}")
     for row in result['kernels']:
         times = row['gpuMs']
-        print(f"GPU {row['kernel']} {row['backend']} {row['phase']}: n={times['samples']} mean={times['mean']:.6f} ms total={times['total']:.3f} ms")
+        print(f"GPU {row['kernel']} {row['backend']} {row['phase']} serializing={int(row['serializing'])}: n={times['samples']} mean={times['mean']:.6f} ms total={times['total']:.3f} ms")
 
 
 if __name__ == '__main__':