Merge upstream accuracy mode and artifact fixes into Windows RDNA4

tools/d4r_native_selection.h

@@ -0,0 +1,91 @@
+/* Native directory selection shared with the CPU-only configuration tests. */
+#pragma once
+#include <dirent.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int d4r_apply_accuracy_policy(void)
+{
+    const char* prefer = getenv("D4R_PREFER_ACCURACY");
+    if (prefer == NULL || strcmp(prefer, "1") != 0)
+        return 0;
+    setenv("D4R_ZLUDA_IGNORE_DENORMAL", "0", 1);
+    setenv("D4R_ZLUDA_WMMA_F32ACC", "0", 1);
+    setenv("D4R_ZLUDA_FAST_MATH", "0", 1);
+    setenv("D4R_ZLUDA_WAVE64", "0", 1);
+    setenv("D4R_ELIDE_NGX_SYNC", "0", 1);
+    return 1;
+}
+
+static int d4r_accuracy_directory(const char* directory)
+{
+    char path[1200], version[16];
+    if (snprintf(path, sizeof(path), "%s/d4r-accuracy.txt", directory) >= (int)sizeof(path))
+        return 0;
+    FILE* file = fopen(path, "r");
+    if (file == NULL)
+        return 0;
+    const int valid = fgets(version, sizeof(version), file) != NULL && strcmp(version, "1\n") == 0;
+    fclose(file);
+    return valid;
+}
+
+/* 1 = hash-checked set, 2 = unmanifested developer set, 0 = no usable set.
+   Accuracy mode never selects an unmarked directory, including flat legacy developer builds. */
+static int d4r_native_candidate(const char* directory, int accuracy)
+{
+    if (accuracy && !d4r_accuracy_directory(directory))
+        return 0;
+    char path[1200];
+    if (snprintf(path, sizeof(path), "%s/d4r-kernels.txt", directory) >= (int)sizeof(path))
+        return 0;
+    FILE* file = fopen(path, "r");
+    if (file != NULL)
+    {
+        fclose(file);
+        return 1;
+    }
+    return accuracy ? 2 : 0;
+}
+
+static int d4r_select_native_source(const char* configured, const char* architecture, int accuracy,
+                                  int fp8, char* source, size_t size)
+{
+    char base[1024];
+    const int length = accuracy && !d4r_accuracy_directory(configured)
+        ? snprintf(base, sizeof(base), "%s/accuracy", configured)
+        : snprintf(base, sizeof(base), "%s", configured);
+    if (length < 0 || length >= (int)sizeof(base))
+        return 0;
+    int kind;
+    if (fp8 && strncmp(architecture, "gfx12", 5) == 0)
+    {
+        if (snprintf(source, size, "%s/%s-fp8", base, architecture) >= (int)size)
+            return 0;
+        if ((kind = d4r_native_candidate(source, accuracy)) != 0)
+            return kind;
+    }
+    if (architecture[0] != '\0')
+    {
+        if (snprintf(source, size, "%s/%s", base, architecture) >= (int)size)
+            return 0;
+        if ((kind = d4r_native_candidate(source, accuracy)) != 0)
+            return kind;
+    }
+    if (snprintf(source, size, "%s", base) >= (int)size)
+        return 0;
+    if ((kind = d4r_native_candidate(source, accuracy)) != 0)
+        return kind;
+    if (accuracy)
+        return 0;
+    /* Keep legacy flat developer directories working, but do not hand a release root to ZLUDA. */
+    DIR* directory = opendir(base);
+    if (directory == NULL)
+        return 0;
+    int targets = 0;
+    for (struct dirent* entry; (entry = readdir(directory)) != NULL;)
+        targets += strncmp(entry->d_name, "gfx", 3) == 0;
+    closedir(directory);
+    return targets ? 0 : 2;
+}

tools/d4r_nvngx_shim.cpp

@@ -221,6 +221,7 @@ struct CudaApi
     int(WINAPI* ctxSynchronize)() = nullptr;
     int(WINAPI* eventCreate)(CudaEvent*, unsigned int) = nullptr;
     int(WINAPI* eventRecord)(CudaEvent, void*) = nullptr;
+    int(WINAPI* eventSynchronize)(CudaEvent) = nullptr;
     int(WINAPI* eventElapsedTime)(float*, CudaEvent, CudaEvent) = nullptr;
     int(WINAPI* eventDestroy)(CudaEvent) = nullptr;
     // Optional: overlapped staging (see async_copies()).
@@ -528,6 +529,8 @@ static void load_portable_config()
 
     // Native kernels: d4r\kernels, each used only when DLSS's PTX for it matches (see the bridge).
     const std::string native = ascii_lower(ini_value(ini, "kernels", "NativeKernels"));
+    // The bridge selects accuracy variants and enforces their compiler/sync policy before cuInit.
+    portable_set_unix("D4R_PREFER_ACCURACY", ini_flag(ini, "kernels", "PreferAccuracy", 0) ? "1" : "0");
     const bool nativeOn = native.empty() || native == "on" || native == "true" || native == "1" || native == "fast";
     if (!nativeOn && native != "off" && native != "false" && native != "0")
         g_portable.notes.push_back("d4r.ini: [Kernels] NativeKernels must be on or off, not '" + native + "'");
@@ -656,6 +659,16 @@ static double profile_ms(ProfileClock::time_point start, ProfileClock::time_poin
     return std::chrono::duration<double, std::milli>(end - start).count();
 }
 
+static double profile_thread_cpu_ms()
+{
+    FILETIME created = {}, exited = {}, kernel = {}, user = {};
+    if (!GetThreadTimes(GetCurrentThread(), &created, &exited, &kernel, &user))
+        return -1.0;
+    const uint64_t kernelTicks = (static_cast<uint64_t>(kernel.dwHighDateTime) << 32) | kernel.dwLowDateTime;
+    const uint64_t userTicks = (static_cast<uint64_t>(user.dwHighDateTime) << 32) | user.dwLowDateTime;
+    return static_cast<double>(kernelTicks + userTicks) / 10000.0;
+}
+
 // --- CUDA worker thread --------------------------------------------------------
 // Every NGX-core and CUDA call runs on this thread so the ZLUDA context that
 // the nvcuda bridge creates on first use stays current.
@@ -1292,14 +1305,18 @@ static bool load_libraries()
         g.cu.outputKernelNative = nullptr;
     if (!load_export(g.cuda, "d4rRegisterLinearTexture", g.cu.registerLinearTexture))
         g.cu.registerLinearTexture = nullptr;
-    if (profile_enabled())
+    if (profile_enabled() || env_uint("D4R_SHIM_BLOCKING_SYNC", 0) != 0)
     {
-        const bool events = load_export(g.cuda, "cuEventCreate", g.cu.eventCreate) &&
-                            load_export(g.cuda, "cuEventRecord", g.cu.eventRecord) &&
-                            load_export(g.cuda, "cuEventElapsedTime", g.cu.eventElapsedTime) &&
-                            load_export(g.cuda, "cuEventDestroy", g.cu.eventDestroy);
-        if (!events)
+        const bool create = load_export(g.cuda, "cuEventCreate", g.cu.eventCreate);
+        const bool record = load_export(g.cuda, "cuEventRecord", g.cu.eventRecord);
+        const bool wait = env_uint("D4R_SHIM_BLOCKING_SYNC", 0) != 0
+                              ? load_export(g.cuda, "d4rEventSynchronize", g.cu.eventSynchronize) : true;
+        const bool destroy = load_export(g.cuda, "cuEventDestroy", g.cu.eventDestroy);
+        const bool elapsed = profile_enabled() ? load_export(g.cuda, "cuEventElapsedTime", g.cu.eventElapsedTime) : true;
+        if (profile_enabled() && !(create && record && destroy && elapsed))
             logf("D4R_PROFILE: CUDA event timing unavailable; CPU stage timings remain enabled");
+        if (env_uint("D4R_SHIM_BLOCKING_SYNC", 0) != 0 && !(create && record && wait && destroy))
+            logf("blocking output wait unavailable; using context sync");
     }
     logf("loaded NGX core %ls and nvcuda.dll: %s", corePath, ok ? "all exports present" : "exports missing");
     return ok;
@@ -1484,6 +1501,9 @@ struct FrameTiming
     double download = 0, outputConvert = 0;
     double prep = 0, worker = 0, workerWait = 0, finishWait = 0, markerWait = 0;
     double ngxHost = 0, ctxSync = 0, gpuEval = -1;
+    bool gpuEventsRecorded = false;
+    double outputSyncWall = 0, outputSyncCpu = -1;
+    bool outputSyncBlocking = false;
     double h2dTotal = 0, d2hIssue = 0, d2hWait = 0;
     ProfileClock::time_point prepStart, workerQueued, finishQueued;
     size_t queueDepth = 0;
@@ -1536,6 +1556,7 @@ struct Feature
     int quality = 0, flags = 0;
     unsigned int preset = 0;
     CudaEvent profileStart = nullptr, profileEnd = nullptr;
+    CudaEvent outputReadyEvent = nullptr;
     bool profileEventsReady = false;
 
     CudaImage color, depth, motion, exposure, output;
@@ -1590,6 +1611,7 @@ struct Feature
     ID3D12Resource* marker = nullptr;
     volatile uint32_t* markerValue = nullptr;
     uint32_t frame = 0;
+    uint32_t rejectedFormatCount = 0; // game thread; rate-limits format diagnostics
     std::atomic<int> latestOutput{-1};
     std::atomic<uint32_t> completedFrames{0};
     uint32_t evaluatedFrames = 0; // worker only
@@ -3613,7 +3635,40 @@ static void publish_vram(Feature* feature, uint32_t frame, const FrameParams& pa
     copy.Height = feature->output.height;
     int result = rowBytes * feature->output.height > buffer.bytes ? -1 : feature->outputRedirected ? 0 : copy_2d(copy);
     if (result == 0)
-        result = g.cu.ctxSynchronize();
+    {
+        const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
+        const double cpuStart = timing.enabled ? profile_thread_cpu_ms() : -1.0;
+        if (feature->outputReadyEvent != nullptr)
+        {
+            const int recordResult = g.cu.eventRecord(feature->outputReadyEvent, nullptr);
+            const int waitResult = recordResult == 0 ? g.cu.eventSynchronize(feature->outputReadyEvent) : recordResult;
+            if (waitResult == 0)
+                timing.outputSyncBlocking = true;
+            else
+            {
+                logf("frame %u: blocking output wait failed (record %d, wait %d); using context sync", frame,
+                     recordResult, waitResult);
+                result = g.cu.ctxSynchronize();
+                g.cu.eventDestroy(feature->outputReadyEvent);
+                feature->outputReadyEvent = nullptr;
+            }
+        }
+        else
+            result = g.cu.ctxSynchronize();
+        if (timing.enabled)
+        {
+            timing.outputSyncWall = profile_ms(syncStart, ProfileClock::now());
+            const double cpuEnd = profile_thread_cpu_ms();
+            if (cpuStart >= 0.0 && cpuEnd >= 0.0)
+                timing.outputSyncCpu = std::max(0.0, cpuEnd - cpuStart);
+        }
+    }
+    if (result == 0 && timing.gpuEventsRecorded)
+    {
+        float gpuEvalMs = -1.0f;
+        if (g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) == 0)
+            timing.gpuEval = gpuEvalMs;
+    }
     if (result != 0)
     {
         logf("frame %u: VRAM output copy failed: %d", frame, result);
@@ -3802,8 +3857,9 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram
     static const bool evalSync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0;
     const int syncResult = params.vram && !evalSync ? 0 : g.cu.ctxSynchronize();
     const auto evaluated = ProfileClock::now();
+    const bool gpuEventsRecorded = eventStartResult == 0 && eventEndResult == 0;
     float gpuEvalMs = -1.0f;
-    if (eventStartResult == 0 && eventEndResult == 0 && syncResult == 0 &&
+    if (gpuEventsRecorded && syncResult == 0 && (!params.vram || evalSync) &&
         g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) != 0)
         gpuEvalMs = -1.0f;
     if (params.vram && !gpuWait)
@@ -3825,6 +3881,7 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram
         timing.ngxHost = profile_ms(evalStart, evalReturned);
         timing.ctxSync = profile_ms(syncStart, evaluated);
         timing.gpuEval = gpuEvalMs;
+        timing.gpuEventsRecorded = gpuEventsRecorded && params.vram && !evalSync;
     }
     if (params.vram)
     {
@@ -3988,7 +4045,8 @@ static void finish_publish(Feature* feature, int target, uint32_t frame, const F
              "d3d_input_record=%.3f d3d_output_record=%.3f game_call=%.3f "
              "color_convert=%.3f depth_convert=%.3f motion_convert=%.3f exposure_convert=%.3f "
              "color_h2d=%.3f depth_h2d=%.3f motion_h2d=%.3f exposure_h2d=%.3f "
-             "ngx_host=%.3f ctx_sync=%.3f gpu_eval=%.3f d2h=%.3f output_convert=%.3f "
+             "ngx_host=%.3f ctx_sync=%.3f gpu_eval=%.3f output_sync_wall=%.3f "
+             "output_sync_cpu=%.3f output_sync_blocking=%d d2h=%.3f output_convert=%.3f "
              "h2d_total=%.3f d2h_issue=%.3f d2h_wait=%.3f "
              "prep=%.3f worker_wait=%.3f worker=%.3f finish_wait=%.3f finish=%.3f pipeline=%.3f "
              "throughput_interval=%.3f presented=%u age=%d",
@@ -4001,7 +4059,8 @@ static void finish_publish(Feature* feature, int target, uint32_t frame, const F
              timing.inputRecord, timing.outputRecord, timing.gameCall,
              timing.convert[0], timing.convert[1], timing.convert[2], timing.convert[3],
              timing.upload[0], timing.upload[1], timing.upload[2], timing.upload[3],
-             timing.ngxHost, timing.ctxSync, timing.gpuEval, timing.download, timing.outputConvert,
+             timing.ngxHost, timing.ctxSync, timing.gpuEval, timing.outputSyncWall, timing.outputSyncCpu,
+             timing.outputSyncBlocking, timing.download, timing.outputConvert,
              timing.h2dTotal, timing.d2hIssue, timing.d2hWait,
              timing.prep, timing.workerWait, timing.worker, timing.finishWait, profile_ms(start, finished),
              profile_ms(timing.prepStart, finished), interval,
@@ -4296,6 +4355,22 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, u
         else
             feature->scratch = 0;
         const NgxResult created = g.ngx.createFeature(NGX_FEATURE_SUPER_SAMPLING, p, &feature->cudaHandle);
+        if (created == NGX_SUCCESS && env_uint("D4R_SHIM_BLOCKING_SYNC", 0) != 0 &&
+            g.cu.eventCreate != nullptr && g.cu.eventRecord != nullptr &&
+            g.cu.eventSynchronize != nullptr && g.cu.eventDestroy != nullptr)
+        {
+            // All VRAM-path input waits, NGX kernels and output copies use the null stream.
+            // A blocking event yields the CPU while the GPU finishes that stream.
+            constexpr unsigned int kBlockingSyncNoTiming = 0x1u | 0x2u;
+            const int eventResult = g.cu.eventCreate(&feature->outputReadyEvent, kBlockingSyncNoTiming);
+            if (eventResult != 0)
+            {
+                logf("blocking output wait unavailable (cuEventCreate %d); using context sync", eventResult);
+                feature->outputReadyEvent = nullptr;
+            }
+            else
+                logf("blocking output wait enabled for this feature");
+        }
         if (created == NGX_SUCCESS && profile_enabled() && g.cu.eventCreate != nullptr &&
             g.cu.eventRecord != nullptr && g.cu.eventElapsedTime != nullptr && g.cu.eventDestroy != nullptr)
         {
@@ -4396,11 +4471,23 @@ static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* hand
              colorDesc.Width, colorDesc.Height, colorDesc.Format, depthDesc.Width, depthDesc.Height, depthDesc.Format,
              motionDesc.Width, motionDesc.Height, motionDesc.Format, outputDesc.Width, outputDesc.Height,
              outputDesc.Format, exposure);
-    if (!supported_input(Plane::Color, colorDesc.Format) || !supported_input(Plane::Depth, depthDesc.Format) ||
-        !supported_input(Plane::Motion, motionDesc.Format) || !supported_output(outputDesc.Format))
+    const bool colorSupported = supported_input(Plane::Color, colorDesc.Format);
+    const bool depthSupported = supported_input(Plane::Depth, depthDesc.Format);
+    const bool motionSupported = supported_input(Plane::Motion, motionDesc.Format);
+    const bool outputSupported = supported_output(outputDesc.Format);
+    if (!colorSupported || !depthSupported || !motionSupported || !outputSupported)
     {
-        if (frame <= 3)
-            logf("evaluate frame %u: unsupported format", frame);
+        // Log the first failures even if they occur well after startup, then periodically.
+        const uint32_t rejected = ++feature->rejectedFormatCount;
+        if (rejected <= 3 || rejected % 120 == 0)
+            logf("evaluate feature %u frame %u: unsupported format (rejection %u): "
+                 "color %llux%u fmt=%d%s, depth %llux%u fmt=%d%s, "
+                 "mv %llux%u fmt=%d%s, output %llux%u fmt=%d%s",
+                 feature->handle.Id, frame, rejected,
+                 colorDesc.Width, colorDesc.Height, colorDesc.Format, colorSupported ? "" : " (unsupported)",
+                 depthDesc.Width, depthDesc.Height, depthDesc.Format, depthSupported ? "" : " (unsupported)",
+                 motionDesc.Width, motionDesc.Height, motionDesc.Format, motionSupported ? "" : " (unsupported)",
+                 outputDesc.Width, outputDesc.Height, outputDesc.Format, outputSupported ? "" : " (unsupported)");
         return NGX_FAIL_UNSUPPORTED_FORMAT;
     }
     if (exposure != nullptr)
@@ -4710,6 +4797,8 @@ static void release_feature(Feature* feature)
             g.cu.eventDestroy(feature->profileStart);
         if (feature->profileEnd != nullptr)
             g.cu.eventDestroy(feature->profileEnd);
+        if (feature->outputReadyEvent != nullptr)
+            g.cu.eventDestroy(feature->outputReadyEvent);
         if (feature->scratch != 0)
             g.cu.memFree(feature->scratch);
         for (CudaDevicePtr buffer : feature->planeLinear)

tools/wine_nvcuda_bridge.c

@@ -11,6 +11,7 @@
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include "d4r_native_selection.h"
 
 typedef int CUresult;
 typedef int CUdevice;
@@ -443,39 +444,28 @@ static void prepare_native_kernels(const char* cache_home)
     /* RDNA4 with native FP8 WMMA (D4R_ZLUDA_WMMA_FP8_NATIVE=1, d4r.ini NativeFp8): the <target>-fp8 variant,
        whose kernels match ZLUDA's FP8 lowering; no other GPU has such a directory */
     const char* fp8 = getenv("D4R_ZLUDA_WMMA_FP8_NATIVE");
-    if (fp8 != NULL && strcmp(fp8, "1") == 0 && strncmp(architecture, "gfx12", 5) == 0)
+    const char* prefer = getenv("D4R_PREFER_ACCURACY");
+    const int accuracy = prefer != NULL && strcmp(prefer, "1") == 0;
+    const int kind = d4r_select_native_source(configured, architecture, accuracy,
+        fp8 != NULL && strcmp(fp8, "1") == 0, source, sizeof(source));
+    if (kind == 0)
     {
-        snprintf(source, sizeof(source), "%s/%s-fp8", configured, architecture);
-        snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
-        manifest = fopen(path, "r");
+        tracef("native kernels: no %sset for this GPU (%s) in %s; ZLUDA compiles every DLSS kernel",
+               accuracy ? "accuracy " : "", architecture[0] != '\0' ? architecture : "unknown target", configured);
+        unsetenv("D4R_ZLUDA_NATIVE_DIR");
+        return;
     }
+    if (kind == 2)
+    {
+        setenv("D4R_ZLUDA_NATIVE_DIR", source, 1);
+        tracef("native kernels: %sdeveloper set in %s", accuracy ? "accuracy " : "", source);
+        return;
+    }
+    snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
+    manifest = fopen(path, "r");
     if (manifest == NULL)
     {
-        snprintf(source, sizeof(source), "%s/%s", configured, architecture);
-        snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
-        manifest = architecture[0] != '\0' ? fopen(path, "r") : NULL;
-    }
-    if (manifest == NULL)
-    {
-        snprintf(source, sizeof(source), "%s", configured);
-        snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
-        manifest = fopen(path, "r");
-    }
-    if (manifest == NULL)
-    {
-        /* a developer directory is used as it is; a release one without this GPU's target is not */
-        DIR* directory = opendir(configured);
-        int targets = 0;
-        for (struct dirent* entry; directory != NULL && (entry = readdir(directory)) != NULL;)
-            targets += strncmp(entry->d_name, "gfx", 3) == 0;
-        if (directory != NULL)
-            closedir(directory);
-        if (targets != 0)
-        {
-            tracef("native kernels: none for this GPU (%s) in %s; ZLUDA compiles every DLSS kernel",
-                   architecture[0] != '\0' ? architecture : "unknown target", configured);
-            unsetenv("D4R_ZLUDA_NATIVE_DIR");
-        }
+        unsetenv("D4R_ZLUDA_NATIVE_DIR");
         return;
     }
     char line[512];
@@ -767,6 +757,13 @@ int WINAPI d4rOutputKernelNative(void)
 
 static void load_zluda(void)
 {
+    /* A policy switch, applied before ZLUDA loads or reads its compilation/cache settings. This
+       also covers native-off and missing/version-mismatched native kernels. */
+    const int accuracy = d4r_apply_accuracy_policy();
+    if (accuracy)
+    {
+        tracef("PreferAccuracy on: original denormal handling, per-MMA rounding, wave32 and NGX synchronizations");
+    }
     /* D4R_ZLUDA_LIBCUDA may name ZLUDA's libcuda.so directly, for processes
        (such as Proton games) whose library search path is not ours. */
     const char* configured = getenv("D4R_ZLUDA_LIBCUDA");
@@ -796,6 +793,28 @@ static void load_zluda(void)
         expand_home(cache, cache_home, sizeof(cache_home));
         make_directories(cache_home);
     }
+    if (accuracy)
+    {
+        /* FAST_MATH is not fingerprinted by older ZLUDA runtimes. Use a separate cache even
+           when a previous run explicitly enabled that experimental compiler switch. */
+        char base[1024];
+        const char* xdg = getenv("XDG_CACHE_HOME");
+        if (cache_home[0] != '\0')
+            snprintf(base, sizeof(base), "%s", cache_home);
+        else if (xdg != NULL && xdg[0] != '\0')
+            snprintf(base, sizeof(base), "%s", xdg);
+        else
+        {
+            const char* home = getenv("HOME");
+            snprintf(base, sizeof(base), "%s/.cache", home != NULL ? home : "/tmp");
+        }
+        if (snprintf(cache_home, sizeof(cache_home), "%s/d4r-accuracy", base) >= (int)sizeof(cache_home))
+        {
+            set_load_error("accuracy cache path is too long");
+            return;
+        }
+        make_directories(cache_home);
+    }
     prepare_native_kernels(cache_home[0] != '\0' ? cache_home : NULL);
     if (cache_home[0] != '\0')
     {
@@ -2263,7 +2282,18 @@ CUresult WINAPI cuArrayDestroy(CUarray array)
     CUARRAYDESTROY_FN function = (CUARRAYDESTROY_FN)find_zluda_symbol("cuArrayDestroy");
     CUresult result = function != NULL ? function(array) : missing("cuArrayDestroy");
     if (result == CUDA_SUCCESS)
+    {
+        /* A recycled array handle must not inherit a released feature's output target. */
+        pthread_mutex_lock(&instrumentation_lock);
+        if (redirect_array == array)
+        {
+            redirect_array = NULL;
+            redirect_pointer = 0;
+            redirect_pitch = 0;
+        }
+        pthread_mutex_unlock(&instrumentation_lock);
         forget_array_descriptor(array);
+    }
     TRACE_CALL(result, "cuArrayDestroy array=%p result=%d", array, result);
     return result;
 }
@@ -2537,6 +2567,18 @@ CUresult WINAPI cuEventSynchronize(D4rArg a0)
     TRACE_CALL(result, "cuEventSynchronize->cuEventSynchronize result=%d", result);
     return result;
 }
+/* The shim's output wait must not inherit D4R_ELIDE_NGX_SYNC: it releases the
+   game's split command list only after this event has actually completed. */
+CUresult WINAPI d4rEventSynchronize(D4rArg a0)
+{
+    typedef CUresult(__attribute__((sysv_abi)) * function_type)(D4rArg);
+    function_type function = (function_type)find_zluda_symbol("cuEventSynchronize");
+    CUresult result = function != NULL ? function(a0) : missing("cuEventSynchronize");
+    if (result == CUDA_SUCCESS)
+        flush_kernel_profiles();
+    TRACE_CALL(result, "d4rEventSynchronize result=%d", result);
+    return result;
+}
 D4R_FORWARD(cuEventQuery, cuEventQuery, 1, (D4rArg a0), (a0))
 /* With elided event waits an event may still be pending: report 0 ms instead of CUDA_ERROR_NOT_READY. */
 CUresult WINAPI cuEventElapsedTime(D4rArg a0, D4rArg a1, D4rArg a2)

tools/wine_nvcuda_bridge.spec

@@ -92,6 +92,7 @@
 @ stdcall d4rReleaseVulkanMemory(ptr) d4rReleaseVulkanMemory
 @ stdcall d4rMemcpy2DAsync(ptr ptr) d4rMemcpy2DAsync
 @ stdcall d4rCtxSynchronize() d4rCtxSynchronize
+@ stdcall d4rEventSynchronize(ptr) d4rEventSynchronize
 @ stdcall d4rStreamWaitValue32(int64 long) d4rStreamWaitValue32
 @ stdcall d4rWriteValue32(int64 long) d4rWriteValue32
 @ stdcall d4rSetArrayRedirect(ptr int64 long) d4rSetArrayRedirect