// d4r_nvngx.dll: an NGX core replacement that implements the D3D12 DLSS
// entry points on top of the official NGX core's CUDA path (which runs on AMD
// through the Wine nvcuda bridge and ZLUDA). OptiScaler loads it through its
// NvngxPath setting.
//
// A D3D12 EvaluateFeature call only records into the game's still-open command
// list, while DLSS runs on a separate CUDA context. Each evaluation therefore:
//   1. records copies of the inputs into readback buffers plus a GPU-written
//      frame marker (WriteBufferImmediate) on the game's command list,
//   2. records a copy of the most recent finished DLSS result from an upload
//      buffer into the output texture,
//   3. queues a CUDA job that waits for the marker, uploads the inputs into
//      CUDA arrays, runs the official DLSS evaluation, and stores the result
//      into an upload buffer for a later frame.
// The output is therefore one or more frames behind the game's other passes.
//
// Environment:
//   D4R_NGX_CORE        Windows path of the official _nvngx.dll (required)
//   D4R_NGX_FEATURE_DIR directory holding nvngx_dlss.dll (optional, added to
//                       the feature search paths)
//   D4R_SHIM_LOG        log file path (default: d4r_nvngx.log next to the DLL)
//   D4R_PROFILE=1       log per-frame CPU stages and default-stream GPU event time
//   D4R_SHIM_OUTPUT_HASH=1 log an FNV-1a hash of every DLSS output as produced
//   D4R_SHIM_OUTPUT_DUMP_DIR=<Windows path> save selected raw RGBA16F DLSS frames
//   D4R_SHIM_OUTPUT_DUMP_START / _COUNT / _EVERY select frames (defaults 1/1/1)
//   D4R_SHIM_INPUT_DUMP_DIR=<Windows path> save selected canonical input planes
//                       (use D4R_SHIM_VRAM_VERIFY=1 with VRAM interop)
//   D4R_SHIM_CAPTURE_TRIGGER=<Windows path> select the next frame when this file appears
//                       (D4R_SHIM_CAPTURE_COUNT=N: that frame and the N-1 after it;
//                       D4R_SHIM_CAPTURE_MEMORY=1 buffers them in memory, written afterwards)
//   D4R_SHIM_INPUT_STATE / D4R_SHIM_DEPTH_STATE / D4R_SHIM_OUTPUT_STATE
//                       D3D12 resource states the inputs/output are in when
//                       EvaluateFeature is called (defaults 0x40, 0x40, 0x8)
//   D4R_SHIM_VRAM_INTEROP=1 keep inputs and output in VRAM (see "VRAM interop")
//   D4R_SHIM_SPLIT_FRAME=1 with VRAM interop and the d4r vkd3d-proton patch,
//                       present each frame's own DLSS result (see "Split frames")
#define WIDL_EXPLICIT_AGGREGATE_RETURNS
#include <windows.h>
#include <d3d12.h>
#include <vulkan/vulkan_core.h>

#include <algorithm>
#include <bit>
#include <atomic>
#include <chrono>
#include <cmath>
#include <condition_variable>
#include <cstdarg>
#include <cstdint>
#include <cstdlib>
#include <cstdio>
#include <cstring>
#include <deque>
#include <functional>
#include <future>
#include <mutex>
#include <string>
#include <thread>
#include <vector>

// --- NGX types (declared locally: the core's exports differ from the SDK) --

using NgxResult = unsigned int;
constexpr NgxResult NGX_SUCCESS = 0x1;
constexpr NgxResult NGX_FAIL = 0xBAD00000;
constexpr NgxResult NGX_FAIL_FEATURE_NOT_SUPPORTED = NGX_FAIL | 1;
constexpr NgxResult NGX_FAIL_PLATFORM_ERROR = NGX_FAIL | 2;
constexpr NgxResult NGX_FAIL_INVALID_PARAMETER = NGX_FAIL | 5;
constexpr NgxResult NGX_FAIL_NOT_INITIALIZED = NGX_FAIL | 7;
constexpr NgxResult NGX_FAIL_UNSUPPORTED_FORMAT = NGX_FAIL | 14;
constexpr unsigned int NGX_FEATURE_SUPER_SAMPLING = 1;

struct NgxHandle
{
    unsigned int Id;
};

struct NgxPathListInfo
{
    const wchar_t* const* Path;
    unsigned int Length;
};

struct NgxLoggingInfo
{
    void* LoggingCallback;
    int MinimumLoggingLevel;
    bool DisableOtherLoggingSinks;
};

struct NgxFeatureCommonInfo
{
    NgxPathListInfo PathListInfo;
    void* InternalData;
    NgxLoggingInfo LoggingInfo;
};

struct NgxFeatureRequirement
{
    unsigned int FeatureSupported;
    unsigned int MinHWArchitecture;
    char MinOSVersion[255];
};

// MSVC-ABI parameter accessors (d4r_ngx_param_msvc.cpp, built with clang-cl).
extern "C"
{
    void d4r_ngx_set_ull(void* parameters, const char* name, unsigned long long value);
    void d4r_ngx_set_float(void* parameters, const char* name, float value);
    void d4r_ngx_set_uint(void* parameters, const char* name, unsigned int value);
    void d4r_ngx_set_int(void* parameters, const char* name, int value);
    void d4r_ngx_set_void(void* parameters, const char* name, void* value);
    NgxResult d4r_ngx_get_ull(void* parameters, const char* name, unsigned long long* value);
    NgxResult d4r_ngx_get_float(void* parameters, const char* name, float* value);
    NgxResult d4r_ngx_get_uint(void* parameters, const char* name, unsigned int* value);
    NgxResult d4r_ngx_get_int(void* parameters, const char* name, int* value);
    NgxResult d4r_ngx_get_void(void* parameters, const char* name, void** value);
    NgxResult d4r_ngx_get_d3d12_resource(void* parameters, const char* name, ID3D12Resource** value);
    void* d4r_ngx_parameters_create();
    void d4r_ngx_parameters_destroy(void* parameters);
}

// --- CUDA driver API (through nvcuda.dll, the Wine bridge to ZLUDA) --------

using CudaDevicePtr = unsigned long long;
using CudaArray = void*;
using CudaObject = unsigned long long;
using CudaEvent = void*;

struct CudaArrayDescriptor
{
    size_t Width;
    size_t Height;
    uint32_t Format;
    uint32_t NumChannels;
};

struct CudaArray3DDescriptor
{
    size_t Width;
    size_t Height;
    size_t Depth;
    uint32_t Format;
    uint32_t NumChannels;
    uint32_t Flags;
};

struct CudaMemcpy2D
{
    size_t srcXInBytes;
    size_t srcY;
    uint32_t srcMemoryType;
    uint32_t srcAlignment;
    const void* srcHost;
    CudaDevicePtr srcDevice;
    CudaArray srcArray;
    size_t srcPitch;
    size_t dstXInBytes;
    size_t dstY;
    uint32_t dstMemoryType;
    uint32_t dstAlignment;
    void* dstHost;
    CudaDevicePtr dstDevice;
    CudaArray dstArray;
    size_t dstPitch;
    size_t WidthInBytes;
    size_t Height;
};
static_assert(sizeof(CudaMemcpy2D) == 128);

struct CudaResourceDesc
{
    uint32_t resType;
    uint32_t alignment;
    union
    {
        struct
        {
            CudaArray hArray;
        } array;
        int reserved[32];
    } res;
    uint32_t flags;
    uint32_t reserved;
};
static_assert(sizeof(CudaResourceDesc) == 144);

struct CudaTextureDesc
{
    uint32_t addressMode[3];
    uint32_t filterMode;
    uint32_t flags;
    uint32_t maxAnisotropy;
    uint32_t mipmapFilterMode;
    float mipmapLevelBias;
    float minMipmapLevelClamp;
    float maxMipmapLevelClamp;
    float borderColor[4];
    int32_t reserved[12];
};
static_assert(sizeof(CudaTextureDesc) == 104);

constexpr uint32_t CUDA_FORMAT_HALF = 0x10;
constexpr uint32_t CUDA_FORMAT_FLOAT = 0x20;
constexpr uint32_t CUDA_MEMORY_HOST = 1;
constexpr uint32_t CUDA_MEMORY_DEVICE = 2;
constexpr uint32_t CUDA_MEMORY_ARRAY = 3;

struct CudaApi
{
    int(WINAPI* memAlloc)(CudaDevicePtr*, size_t) = nullptr;
    int(WINAPI* memFree)(CudaDevicePtr) = nullptr;
    int(WINAPI* arrayCreate)(CudaArray*, const CudaArrayDescriptor*) = nullptr;
    int(WINAPI* array3DCreate)(CudaArray*, const CudaArray3DDescriptor*) = nullptr;
    int(WINAPI* arrayDestroy)(CudaArray) = nullptr;
    int(WINAPI* memcpy2D)(const CudaMemcpy2D*) = nullptr;
    int(WINAPI* texObjectCreate)(CudaObject*, const CudaResourceDesc*, const CudaTextureDesc*, const void*) = nullptr;
    int(WINAPI* texObjectDestroy)(CudaObject) = nullptr;
    int(WINAPI* surfObjectCreate)(CudaObject*, const CudaResourceDesc*) = nullptr;
    int(WINAPI* surfObjectDestroy)(CudaObject) = nullptr;
    int(WINAPI* ctxSynchronize)() = nullptr;
    int(WINAPI* eventCreate)(CudaEvent*, unsigned int) = nullptr;
    int(WINAPI* eventRecord)(CudaEvent, void*) = nullptr;
    int(WINAPI* eventElapsedTime)(float*, CudaEvent, CudaEvent) = nullptr;
    int(WINAPI* eventDestroy)(CudaEvent) = nullptr;
    // Optional: overlapped staging (see async_copies()).
    int(WINAPI* memAllocHost)(void**, size_t) = nullptr;
    int(WINAPI* memFreeHost)(void*) = nullptr;
    int(WINAPI* memcpyHtoDAsync)(CudaDevicePtr, const void*, size_t, void*) = nullptr;
    int(WINAPI* memcpyDtoHAsync)(void*, CudaDevicePtr, size_t, void*) = nullptr;
    int(WINAPI* streamCreate)(void**, unsigned int) = nullptr;
    int(WINAPI* streamSynchronize)(void*) = nullptr;
    // Optional (d4r nvcuda bridge): device <-> array copies queued on the null stream.
    int(WINAPI* memcpy2DAsync)(const CudaMemcpy2D*, void*) = nullptr;
    // Optional (d4r nvcuda bridge): null-stream wait for a device u32, and a write from another stream.
    int(WINAPI* streamWaitValue32)(CudaDevicePtr, uint32_t) = nullptr;
    int(WINAPI* writeValue32)(CudaDevicePtr, uint32_t) = nullptr;
    // Optional (d4r nvcuda bridge): surfaces of an array store to linear memory instead (d4r native kernels).
    int(WINAPI* setArrayRedirect)(CudaArray, CudaDevicePtr, uint32_t) = nullptr;
    int(WINAPI* outputKernelNative)() = nullptr;
    // Optional (d4r nvcuda bridge): report a pitch-linear texture to NGX as an array of that size/format.
    int(WINAPI* registerLinearTexture)(CudaObject, size_t, size_t, uint32_t, uint32_t) = nullptr;
};

// --- official NGX core CUDA API --------------------------------------------

struct CoreApi
{
    NgxResult (*init)(unsigned long long, const wchar_t*, const NgxFeatureCommonInfo*, unsigned int) = nullptr;
    NgxResult (*initProjectId)(const char*, int, const char*, const wchar_t*, unsigned int,
                               const NgxFeatureCommonInfo*) = nullptr;
    NgxResult (*shutdown)() = nullptr;
    NgxResult (*getParameters)(void**) = nullptr;
    NgxResult (*allocateParameters)(void**) = nullptr;
    NgxResult (*getCapabilityParameters)(void**) = nullptr;
    NgxResult (*destroyParameters)(void*) = nullptr;
    NgxResult (*createFeature)(unsigned int, void*, NgxHandle**) = nullptr;
    NgxResult (*releaseFeature)(NgxHandle*) = nullptr;
    NgxResult (*evaluateFeature)(const NgxHandle*, void*, void*) = nullptr;
};

// --- logging -----------------------------------------------------------------

static std::mutex g_logMutex;
static FILE* g_log = nullptr;
static HMODULE g_selfModule = nullptr;

// --- portable install ----------------------------------------------------------
// The drag-in release keeps this DLL (as nvngx.dll, which OptiScaler finds through its OptiDllPath),
// the nvcuda bridge, ZLUDA and the native kernels in an d4r folder next to the game's executable,
// with that game's settings in d4r\d4r.ini. The file's settings become the environment variables the
// developer launcher sets (config/d4r.ini.default documents them), and a variable that is already set,
// for example in the game's launch options, wins. Settings for the Linux side (ZLUDA, the bridge) are
// handed to the bridge right after it is loaded, before its first CUDA call. Without an d4r.ini next
// to the DLL nothing changes.

struct PortableInstall
{
    bool active = false;
    std::wstring dir;                                         // the d4r folder
    std::string unixDir;                                      // the same folder as a Linux path
    std::vector<std::pair<std::string, std::string>> unixEnv; // for the bridge's d4rSetEnv
    std::vector<std::string> notes;                           // logged once the log is open
};
static PortableInstall g_portable;
static std::once_flag g_portableOnce;
static bool g_logFresh = false; // a portable install starts a new log for every launch

struct IniEntry
{
    std::string section, key, value;
};

static std::string ascii_lower(std::string text)
{
    for (char& c : text)
        c = static_cast<char>(tolower(static_cast<unsigned char>(c)));
    return text;
}

static std::string trim(const std::string& text)
{
    const size_t first = text.find_first_not_of(" \t\r\n");
    if (first == std::string::npos)
        return std::string();
    return text.substr(first, text.find_last_not_of(" \t\r\n") - first + 1);
}

// INI in the configparser dialect d4r_config.py reads: [Section], key = value, full-line comments
// with ; or #, inline comments after whitespace.
static std::vector<IniEntry> parse_ini(const std::string& text)
{
    std::vector<IniEntry> entries;
    std::string section;
    size_t start = text.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0;
    while (start < text.size())
    {
        size_t end = text.find('\n', start);
        if (end == std::string::npos)
            end = text.size();
        const std::string line = trim(text.substr(start, end - start));
        start = end + 1;
        if (line.empty() || line[0] == ';' || line[0] == '#')
            continue;
        if (line[0] == '[')
        {
            section = ascii_lower(trim(line.substr(1, line.find(']') - 1)));
            continue;
        }
        const size_t equals = line.find('=');
        if (equals == std::string::npos)
            continue;
        std::string value = line.substr(equals + 1);
        for (size_t i = 1; i < value.size(); ++i)
            if ((value[i] == ';' || value[i] == '#') && (value[i - 1] == ' ' || value[i - 1] == '\t'))
            {
                value.resize(i);
                break;
            }
        entries.push_back({section, trim(line.substr(0, equals)), trim(value)});
    }
    return entries;
}

// The last value for [section] key; "" when it is missing, empty or "auto".
static std::string ini_value(const std::vector<IniEntry>& ini, const char* section, const char* key)
{
    std::string result;
    for (const IniEntry& entry : ini)
        if (entry.section == section && ascii_lower(entry.key) == ascii_lower(key))
            result = entry.value;
    return ascii_lower(result) == "auto" ? std::string() : result;
}

// 1 or 0 for a boolean setting, fallback when it is unset or not a boolean.
static int ini_flag(const std::vector<IniEntry>& ini, const char* section, const char* key, int fallback)
{
    const std::string value = ascii_lower(ini_value(ini, section, key));
    if (value.empty())
        return fallback;
    if (value == "1" || value == "true" || value == "yes" || value == "on")
        return 1;
    if (value == "0" || value == "false" || value == "no" || value == "off")
        return 0;
    g_portable.notes.push_back("d4r.ini: [" + std::string(section) + "] " + key + " must be true or false, not '" +
                               value + "'; using " + (fallback ? "true" : "false"));
    return fallback;
}

static bool read_whole_file(const std::wstring& path, std::string& text)
{
    HANDLE file = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING,
                              FILE_ATTRIBUTE_NORMAL, nullptr);
    if (file == INVALID_HANDLE_VALUE)
        return false;
    LARGE_INTEGER size = {};
    bool ok = GetFileSizeEx(file, &size) && size.QuadPart < (1 << 20);
    if (ok)
    {
        text.resize(static_cast<size_t>(size.QuadPart));
        DWORD read = 0;
        ok = text.empty() || (ReadFile(file, text.data(), static_cast<DWORD>(text.size()), &read, nullptr) &&
                              read == text.size());
    }
    CloseHandle(file);
    return ok;
}

static bool file_exists(const std::wstring& path)
{
    const DWORD attributes = GetFileAttributesW(path.c_str());
    return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) == 0;
}

// Wine's kernel32 maps a Windows path to the Linux one.
static std::string unix_path(const std::wstring& path)
{
    using UnixFileNameFn = char*(CDECL*)(const wchar_t*);
    const auto function = reinterpret_cast<UnixFileNameFn>(
        reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "wine_get_unix_file_name")));
    char* result = function != nullptr ? function(path.c_str()) : nullptr;
    if (result == nullptr)
        return std::string();
    std::string text(result);
    HeapFree(GetProcessHeap(), 0, result);
    return text;
}

static std::wstring widen(const std::string& text)
{
    std::wstring wide(text.size() + 1, L'\0');
    const int length = MultiByteToWideChar(CP_UTF8, 0, text.c_str(), -1, wide.data(), static_cast<int>(wide.size()));
    wide.resize(length > 0 ? static_cast<size_t>(length - 1) : 0);
    return wide;
}

// A Windows path from d4r.ini: absolute (C:\..., \\...) or relative to the d4r folder.
static std::wstring portable_path(const std::string& value)
{
    const std::wstring path = widen(value);
    if ((path.size() > 2 && path[1] == L':') || (path.size() > 1 && path[0] == L'\\' && path[1] == L'\\'))
        return path;
    return g_portable.dir + L"\\" + path;
}

static void portable_set(const char* name, const std::string& value)
{
    if (GetEnvironmentVariableA(name, nullptr, 0) == 0)
        SetEnvironmentVariableA(name, value.c_str());
}

static void portable_set(const wchar_t* name, const std::wstring& value)
{
    if (GetEnvironmentVariableW(name, nullptr, 0) == 0)
        SetEnvironmentVariableW(name, value.c_str());
}

static void portable_set_unix(const char* name, const std::string& value)
{
    g_portable.unixEnv.emplace_back(name, value);
}

// [DLSS] Model: a preset letter, a friendly name or a render-preset number; "" leaves the choice to the game.
static std::string preset_number(const std::string& model)
{
    std::string name;
    for (char c : model)
        name += static_cast<char>(toupper(static_cast<unsigned char>(c)));
    if (name.empty() || name == "GAME" || name == "DEFAULT")
        return std::string();
    if (name.find_first_not_of("0123456789") == std::string::npos)
        return name;
    static const std::pair<const char*, const char*> aliases[] = {
        {"CNN", "E"}, {"DLSS3", "E"}, {"DLSS4", "K"}, {"DLSS4.5", "M"}, {"TRANSFORMER", "K"}};
    for (const auto& [alias, letter] : aliases)
        if (name == alias)
            name = letter;
    static const std::pair<const char*, int> letters[] = {{"A", 1}, {"B", 2}, {"C", 3}, {"D", 4},  {"E", 5},  {"F", 6},
                                                          {"G", 7}, {"J", 10}, {"K", 11}, {"L", 12}, {"M", 13}};
    for (const auto& [letter, number] : letters)
        if (name == letter)
            return std::to_string(number);
    g_portable.notes.push_back("d4r.ini: [DLSS] Model '" + model + "' is not a preset (use K, E, M or a number); "
                               "leaving the preset to the game");
    return std::string();
}

static void load_portable_config()
{
    wchar_t module[MAX_PATH] = {};
    const DWORD length = GetModuleFileNameW(g_selfModule, module, MAX_PATH);
    if (length == 0 || length >= MAX_PATH)
        return;
    std::wstring dir(module, length);
    dir.resize(dir.find_last_of(L"\\/"));
    std::string text;
    if (!read_whole_file(dir + L"\\d4r.ini", text))
        return; // developer layout: the launcher's environment has everything
    const std::vector<IniEntry> ini = parse_ini(text);
    g_portable.active = true;
    g_portable.dir = dir;
    g_portable.unixDir = unix_path(dir);
    if (g_portable.unixDir.empty())
        g_portable.notes.push_back("cannot map the d4r folder to a Linux path (not running under Wine?)");

    // Files of the install; NGX looks for nvngx_dlss.dll next to this DLL.
    portable_set(L"D4R_NVCUDA_BRIDGE", dir + L"\\nvcuda.dll");
    const std::string core = ini_value(ini, "paths", "NgxCore");
    portable_set(L"D4R_NGX_CORE", core.empty() ? dir + L"\\ngx\\_nvngx.dll" : portable_path(core));
    portable_set(L"D4R_NGX_FEATURE_DIR", dir);
    const std::string log = ini_value(ini, "debug", "Log");
    g_logFresh = log.empty();
    portable_set(L"D4R_SHIM_LOG", log.empty() ? dir + L"\\d4r_nvngx.log" : portable_path(log));
    portable_set("D4R_CUDA_CAPTURE", "0"); // the bridge would otherwise save every DLSS module it loads

    const std::string zluda = ini_value(ini, "paths", "ZludaDir");
    portable_set_unix("D4R_ZLUDA_LIBCUDA", (zluda.empty() ? g_portable.unixDir + "/zluda" : zluda) + "/libcuda.so");
    if (const std::string rocm = ini_value(ini, "paths", "RocmDir"); !rocm.empty())
        portable_set_unix("D4R_ROCM_DIR", rocm);
    const std::string cache = ini_value(ini, "paths", "CacheDir");
    portable_set_unix("D4R_ZLUDA_CACHE_HOME", cache.empty() ? "~/.cache/d4r" : cache);

    std::string model = ini_value(ini, "dlss", "Model");
    if (ascii_lower(model) != "game")
        model = preset_number(model.empty() ? "K" : model);
    else
        model.clear();
    if (!model.empty())
        portable_set("D4R_DLSS_PRESET", model);

    const std::string age = ini_value(ini, "latency", "FrameAge");
    if (age.empty() || age == "0")
    {
        portable_set("D4R_SHIM_SPLIT_FRAME", "1");
        portable_set("D4R_SHIM_MAX_IN_FLIGHT", "3");
    }
    else if (age == "1" || age == "2" || age == "3")
    {
        portable_set("D4R_SHIM_SPLIT_FRAME", "0");
        portable_set("D4R_SHIM_MAX_IN_FLIGHT", age);
    }
    else
        g_portable.notes.push_back("d4r.ini: [Latency] FrameAge must be 0-3, not '" + age + "'");

    // Native kernels: d4r\kernels, each used only when DLSS's PTX for it matches (see the bridge).
    const std::string native = ascii_lower(ini_value(ini, "kernels", "NativeKernels"));
    const bool nativeOn = native.empty() || native == "on" || native == "true" || native == "1" || native == "fast";
    if (!nativeOn && native != "off" && native != "false" && native != "0")
        g_portable.notes.push_back("d4r.ini: [Kernels] NativeKernels must be on or off, not '" + native + "'");
    if (nativeOn && !g_portable.unixDir.empty())
        portable_set_unix("D4R_ZLUDA_NATIVE_DIR", g_portable.unixDir + "/kernels");
    portable_set_unix("D4R_ZLUDA_WMMA", ini_flag(ini, "kernels", "Wmma", 1) ? "1" : "0");
    portable_set_unix("D4R_ZLUDA_WMMA_FP8", ini_flag(ini, "kernels", "Fp8Wmma", 1) ? "1" : "0");
    portable_set_unix("D4R_ZLUDA_IGNORE_DENORMAL", ini_flag(ini, "kernels", "IgnoreDenormals", 1) ? "1" : "0");
    const std::string maxBlock = ini_value(ini, "kernels", "ImplicitMaxBlock");
    portable_set_unix("D4R_ZLUDA_IMPLICIT_MAX_BLOCK", maxBlock.empty() ? "256" : maxBlock);

    static const struct
    {
        const char* key;
        const char* variable;
        int fallback;
    } interop[] = {{"VramInterop", "D4R_SHIM_VRAM_INTEROP", 1}, {"InputSync", "D4R_SHIM_INPUT_SYNC", 1},
                   {"GpuWait", "D4R_SHIM_GPU_WAIT", 1},         {"LinearInputs", "D4R_SHIM_LINEAR_INPUTS", 1},
                   {"EvalSync", "D4R_SHIM_EVAL_SYNC", 0}};
    for (const auto& setting : interop)
        portable_set(setting.variable, ini_flag(ini, "interop", setting.key, setting.fallback) ? "1" : "0");
    portable_set_unix("D4R_ELIDE_NGX_SYNC", ini_flag(ini, "interop", "ElideNgxSync", 1) ? "1" : "0");
    const std::string poll = ini_value(ini, "interop", "MarkerPollUs");
    portable_set("D4R_SHIM_MARKER_POLL_US", poll.empty() ? "20" : poll);
    // Direct output takes effect only while NGX runs the native output kernel (the bridge reports it); the
    // release cannot ship that kernel, which is built from NVIDIA's PTX.
    portable_set("D4R_SHIM_OUTPUT_DIRECT", ini_flag(ini, "interop", "DirectOutput", nativeOn) ? "1" : "0");

    portable_set("D4R_SHIM_WATERMARK", ini_flag(ini, "dlss", "ShowWatermark", 0) ? "1" : "0");
    if (ini_flag(ini, "debug", "Profile", 0))
        portable_set("D4R_PROFILE", "1");
    if (const std::string level = ini_value(ini, "debug", "NgxLogLevel"); !level.empty())
        portable_set("D4R_NGX_LOG_LEVEL", level);

    // [Env]: any other variable, for both sides
    for (const IniEntry& entry : ini)
        if (entry.section == "env" && !entry.key.empty())
        {
            portable_set(entry.key.c_str(), entry.value);
            portable_set_unix(entry.key.c_str(), entry.value);
        }

    if (!file_exists(dir + L"\\nvngx_dlss.dll"))
        g_portable.notes.push_back("nvngx_dlss.dll is missing: copy NVIDIA's DLSS library (310.7 or 310.9 recommended) "
                                   "into the d4r folder");
    wchar_t corePath[MAX_PATH] = {};
    const DWORD coreLength = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
    if (coreLength > 0 && coreLength < MAX_PATH && !file_exists(corePath))
        g_portable.notes.push_back("the NGX core is missing: copy NVIDIA's _nvngx.dll into d4r\\ngx");
}

static void ensure_portable_config()
{
    std::call_once(g_portableOnce, load_portable_config);
}

static void log_open()
{
    ensure_portable_config();
    wchar_t path[MAX_PATH] = {};
    DWORD length = GetEnvironmentVariableW(L"D4R_SHIM_LOG", path, MAX_PATH);
    if (length == 0 || length >= MAX_PATH)
    {
        GetModuleFileNameW(g_selfModule, path, MAX_PATH);
        wchar_t* slash = wcsrchr(path, L'\\');
        if (slash != nullptr)
            wcscpy(slash + 1, L"d4r_nvngx.log");
    }
    g_log = _wfopen(path, g_logFresh ? L"w" : L"a");
}

static void logf(const char* format, ...)
{
    std::lock_guard<std::mutex> lock(g_logMutex);
    if (g_log == nullptr)
        log_open();
    if (g_log == nullptr)
        return;
    std::fprintf(g_log, "[%10.3f] ", GetTickCount64() / 1000.0);
    va_list arguments;
    va_start(arguments, format);
    std::vfprintf(g_log, format, arguments);
    va_end(arguments);
    std::fputc('\n', g_log);
    std::fflush(g_log);
}

// NGX core and feature diagnostics, routed into the shim log.
static void ngx_log_callback(const char* message, int level, unsigned int component)
{
    if (message == nullptr)
        return;
    std::string line(message);
    while (!line.empty() && (line.back() == '\n' || line.back() == '\r'))
        line.pop_back();
    logf("NGX[%d/%u] %s", level, component, line.c_str());
}

static unsigned int env_uint(const char* name, unsigned int fallback)
{
    char value[64] = {};
    DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
    if (length == 0 || length >= sizeof(value))
        return fallback;
    return static_cast<unsigned int>(std::strtoul(value, nullptr, 0));
}

static std::string env_string(const char* name)
{
    char value[1024] = {};
    const DWORD length = GetEnvironmentVariableA(name, value, sizeof(value));
    return length > 0 && length < sizeof(value) ? std::string(value, length) : std::string();
}

static bool profile_enabled()
{
    static const bool enabled = env_uint("D4R_PROFILE", 0) != 0;
    return enabled;
}

using ProfileClock = std::chrono::steady_clock;
static double profile_ms(ProfileClock::time_point start, ProfileClock::time_point end)
{
    return std::chrono::duration<double, std::milli>(end - start).count();
}

// --- CUDA worker thread --------------------------------------------------------
// Every NGX-core and CUDA call runs on this thread so the ZLUDA context that
// the nvcuda bridge creates on first use stays current.

class Worker
{
public:
    void start()
    {
        // Detached: the process may exit without shutting NGX down.
        std::thread([this] { run(); }).detach();
    }

    void post(std::function<void()> job)
    {
        {
            std::lock_guard<std::mutex> lock(mutex_);
            jobs_.push_back(std::move(job));
        }
        condition_.notify_one();
    }

    size_t pending()
    {
        std::lock_guard<std::mutex> lock(mutex_);
        return jobs_.size();
    }

    template <typename Function> auto call(Function function) -> decltype(function())
    {
        using Result = decltype(function());
        auto task = std::make_shared<std::packaged_task<Result()>>(std::move(function));
        auto future = task->get_future();
        post([task] { (*task)(); });
        return future.get();
    }

private:
    void run()
    {
        for (;;)
        {
            std::function<void()> job;
            {
                std::unique_lock<std::mutex> lock(mutex_);
                condition_.wait(lock, [this] { return !jobs_.empty(); });
                job = std::move(jobs_.front());
                jobs_.pop_front();
            }
            job();
        }
    }

    std::mutex mutex_;
    std::condition_variable condition_;
    std::deque<std::function<void()>> jobs_;
};

// --- half/float and packed format conversions --------------------------------

static uint16_t float_to_half(float value)
{
    uint32_t bits;
    std::memcpy(&bits, &value, sizeof(bits));
    const uint32_t sign = (bits >> 16) & 0x8000u;
    const uint32_t exponent = (bits >> 23) & 0xffu;
    uint32_t mantissa = bits & 0x7fffffu;
    if (exponent == 0xff)
        return static_cast<uint16_t>(sign | 0x7c00u | (mantissa ? 0x200u : 0u));
    int halfExponent = static_cast<int>(exponent) - 112;
    if (halfExponent >= 31)
        return static_cast<uint16_t>(sign | 0x7c00u);
    uint32_t shift = 13;
    if (halfExponent <= 0)
    {
        if (halfExponent < -10)
            return static_cast<uint16_t>(sign);
        mantissa |= 0x800000u;
        shift = static_cast<uint32_t>(14 - halfExponent);
        halfExponent = 0;
    }
    const uint32_t halfMantissa = mantissa >> shift;
    const uint32_t remainder = mantissa & ((1u << shift) - 1u);
    const uint32_t halfway = 1u << (shift - 1u);
    uint32_t result = sign | (static_cast<uint32_t>(halfExponent) << 10);
    result += halfMantissa & (halfExponent == 0 ? 0x7ffu : 0x3ffu);
    if (remainder > halfway || (remainder == halfway && (halfMantissa & 1u)))
        ++result;
    return static_cast<uint16_t>(result);
}

static float half_to_float(uint16_t bits)
{
    const uint32_t sign = static_cast<uint32_t>(bits & 0x8000u) << 16;
    const uint32_t exponent = (bits >> 10) & 0x1fu;
    const uint32_t mantissa = bits & 0x3ffu;
    uint32_t result;
    if (exponent == 0)
    {
        float magnitude = static_cast<float>(mantissa) * (1.0f / 16777216.0f);
        std::memcpy(&result, &magnitude, sizeof(result));
        result |= sign;
    }
    else if (exponent == 0x1f)
        result = sign | 0x7f800000u | (mantissa << 13);
    else
        result = sign | ((exponent + 112u) << 23) | (mantissa << 13);
    float value;
    std::memcpy(&value, &result, sizeof(value));
    return value;
}

// Unsigned small floats (R11G11B10): 5-bit exponent, 6 or 5 bit mantissa.
static float small_float_to_float(uint32_t bits, int mantissaBits)
{
    const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
    const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
    if (exponent == 0)
        return std::ldexp(static_cast<float>(mantissa), -14 - mantissaBits);
    if (exponent == 0x1f)
        return mantissa ? NAN : INFINITY;
    return std::ldexp(1.0f + static_cast<float>(mantissa) / static_cast<float>(1u << mantissaBits),
                      static_cast<int>(exponent) - 15);
}

static uint32_t float_to_small_float(float value, int mantissaBits)
{
    if (!(value > 0.0f))
        return 0; // negatives and NaN clamp to zero
    const float maximum = std::ldexp(2.0f - std::ldexp(1.0f, -mantissaBits), 15);
    if (value >= maximum)
        value = maximum;
    int exponent;
    const float fraction = std::frexp(value, &exponent); // value = fraction * 2^exponent
    int biased = exponent - 1 + 15;
    if (biased <= 0)
    {
        const uint32_t mantissa = static_cast<uint32_t>(std::lround(std::ldexp(value, 14 + mantissaBits)));
        return mantissa;
    }
    uint32_t mantissa = static_cast<uint32_t>(std::lround((fraction * 2.0f - 1.0f) * static_cast<float>(1u << mantissaBits)));
    if (mantissa >= (1u << mantissaBits))
    {
        mantissa = 0;
        ++biased;
    }
    if (biased >= 31)
        return (30u << mantissaBits) | ((1u << mantissaBits) - 1u);
    return (static_cast<uint32_t>(biased) << mantissaBits) | mantissa;
}

// Canonical CUDA-side layouts: color RGBA16F, depth R32F, motion RG16F,
// exposure R32F, output RGBA16F.
enum class Plane
{
    Color,
    Depth,
    Motion,
    Exposure
};

static bool supported_input(Plane plane, DXGI_FORMAT format)
{
    switch (plane)
    {
    case Plane::Color:
        return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
               format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
               format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
               format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_UNORM_SRGB ||
               format == DXGI_FORMAT_R8G8B8A8_TYPELESS || format == DXGI_FORMAT_B8G8R8A8_UNORM ||
               format == DXGI_FORMAT_B8G8R8A8_UNORM_SRGB || format == DXGI_FORMAT_B8G8R8A8_TYPELESS;
    case Plane::Depth:
        return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
               format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
               format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS ||
               format == DXGI_FORMAT_D24_UNORM_S8_UINT || format == DXGI_FORMAT_R24G8_TYPELESS ||
               format == DXGI_FORMAT_R24_UNORM_X8_TYPELESS || format == DXGI_FORMAT_D16_UNORM ||
               format == DXGI_FORMAT_R16_UNORM || format == DXGI_FORMAT_R16_TYPELESS;
    case Plane::Motion:
        return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS ||
               format == DXGI_FORMAT_R32G32_FLOAT || format == DXGI_FORMAT_R32G32_TYPELESS ||
               format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R32G32B32A32_FLOAT;
    case Plane::Exposure:
        return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS ||
               format == DXGI_FORMAT_R16_FLOAT || format == DXGI_FORMAT_R16_TYPELESS ||
               format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R16G16B16A16_FLOAT;
    }
    return false;
}

static bool supported_output(DXGI_FORMAT format)
{
    return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS ||
           format == DXGI_FORMAT_R32G32B32A32_FLOAT || format == DXGI_FORMAT_R11G11B10_FLOAT ||
           format == DXGI_FORMAT_R10G10B10A2_UNORM || format == DXGI_FORMAT_R10G10B10A2_TYPELESS ||
           format == DXGI_FORMAT_R8G8B8A8_UNORM || format == DXGI_FORMAT_R8G8B8A8_TYPELESS ||
           format == DXGI_FORMAT_B8G8R8A8_UNORM || format == DXGI_FORMAT_B8G8R8A8_TYPELESS;
}

static float unorm(uint32_t value, int bits)
{
    return static_cast<float>(value) / static_cast<float>((1u << bits) - 1u);
}

static uint32_t to_unorm(float value, int bits)
{
    const float clamped = value < 0.0f ? 0.0f : (value > 1.0f ? 1.0f : value);
    return static_cast<uint32_t>(std::lround(clamped * static_cast<float>((1u << bits) - 1u)));
}

// Converts one row of a D3D12 texel format into the canonical layout.
static void convert_row_in(Plane plane, DXGI_FORMAT format, const uint8_t* source, void* destination, UINT width)
{
    uint16_t* half = static_cast<uint16_t*>(destination);
    float* single = static_cast<float*>(destination);
    for (UINT x = 0; x < width; ++x)
    {
        switch (plane)
        {
        case Plane::Color:
        {
            float rgba[4] = {0.0f, 0.0f, 0.0f, 1.0f};
            switch (format)
            {
            case DXGI_FORMAT_R16G16B16A16_FLOAT:
            case DXGI_FORMAT_R16G16B16A16_TYPELESS:
                std::memcpy(half + x * 4, source + x * 8, 8);
                continue;
            case DXGI_FORMAT_R32G32B32A32_FLOAT:
                std::memcpy(rgba, source + x * 16, 16);
                break;
            case DXGI_FORMAT_R11G11B10_FLOAT:
            {
                uint32_t packed;
                std::memcpy(&packed, source + x * 4, 4);
                rgba[0] = small_float_to_float(packed & 0x7ffu, 6);
                rgba[1] = small_float_to_float((packed >> 11) & 0x7ffu, 6);
                rgba[2] = small_float_to_float((packed >> 22) & 0x3ffu, 5);
                break;
            }
            case DXGI_FORMAT_R10G10B10A2_UNORM:
            case DXGI_FORMAT_R10G10B10A2_TYPELESS:
            {
                uint32_t packed;
                std::memcpy(&packed, source + x * 4, 4);
                rgba[0] = unorm(packed & 0x3ffu, 10);
                rgba[1] = unorm((packed >> 10) & 0x3ffu, 10);
                rgba[2] = unorm((packed >> 20) & 0x3ffu, 10);
                rgba[3] = unorm(packed >> 30, 2);
                break;
            }
            case DXGI_FORMAT_B8G8R8A8_UNORM:
            case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB:
            case DXGI_FORMAT_B8G8R8A8_TYPELESS:
                rgba[0] = unorm(source[x * 4 + 2], 8);
                rgba[1] = unorm(source[x * 4 + 1], 8);
                rgba[2] = unorm(source[x * 4 + 0], 8);
                rgba[3] = unorm(source[x * 4 + 3], 8);
                break;
            default: // RGBA8 variants
                for (int c = 0; c < 4; ++c)
                    rgba[c] = unorm(source[x * 4 + c], 8);
                break;
            }
            for (int c = 0; c < 4; ++c)
                half[x * 4 + c] = float_to_half(rgba[c]);
            break;
        }
        case Plane::Depth:
        {
            switch (format)
            {
            case DXGI_FORMAT_D24_UNORM_S8_UINT:
            case DXGI_FORMAT_R24G8_TYPELESS:
            case DXGI_FORMAT_R24_UNORM_X8_TYPELESS:
            {
                uint32_t packed;
                std::memcpy(&packed, source + x * 4, 4);
                single[x] = unorm(packed & 0xffffffu, 24);
                break;
            }
            case DXGI_FORMAT_D16_UNORM:
            case DXGI_FORMAT_R16_UNORM:
            case DXGI_FORMAT_R16_TYPELESS:
            {
                uint16_t value;
                std::memcpy(&value, source + x * 2, 2);
                single[x] = unorm(value, 16);
                break;
            }
            default: // 32-bit float depth (the depth plane of D32S8 copies as 4 bytes)
                std::memcpy(single + x, source + x * 4, 4);
                break;
            }
            break;
        }
        case Plane::Motion:
            switch (format)
            {
            case DXGI_FORMAT_R16G16_FLOAT:
            case DXGI_FORMAT_R16G16_TYPELESS:
                std::memcpy(half + x * 2, source + x * 4, 4);
                break;
            case DXGI_FORMAT_R16G16B16A16_FLOAT:
                std::memcpy(half + x * 2, source + x * 8, 4);
                break;
            case DXGI_FORMAT_R32G32B32A32_FLOAT:
            case DXGI_FORMAT_R32G32_FLOAT:
            case DXGI_FORMAT_R32G32_TYPELESS:
            {
                const size_t stride = format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 8;
                float mv[2];
                std::memcpy(mv, source + x * stride, 8);
                half[x * 2] = float_to_half(mv[0]);
                half[x * 2 + 1] = float_to_half(mv[1]);
                break;
            }
            default:
                break;
            }
            break;
        case Plane::Exposure:
            switch (format)
            {
            case DXGI_FORMAT_R16_FLOAT:
            case DXGI_FORMAT_R16_TYPELESS:
            case DXGI_FORMAT_R16G16B16A16_FLOAT:
            {
                uint16_t value;
                std::memcpy(&value, source + x * (format == DXGI_FORMAT_R16G16B16A16_FLOAT ? 8 : 2), 2);
                single[x] = half_to_float(value);
                break;
            }
            default:
                std::memcpy(single + x, source + x * (format == DXGI_FORMAT_R32G32B32A32_FLOAT ? 16 : 4), 4);
                break;
            }
            break;
        }
    }
}

// Formats whose texels are bit-identical to the canonical CUDA layout (the
// cases convert_row_in handles as a plain copy), so the mapped staging rows can
// be copied straight into the CUDA array with their D3D12 row pitch.
static bool canonical_input(Plane plane, DXGI_FORMAT format)
{
    switch (plane)
    {
    case Plane::Color:
        return format == DXGI_FORMAT_R16G16B16A16_FLOAT || format == DXGI_FORMAT_R16G16B16A16_TYPELESS;
    case Plane::Depth:
        return format == DXGI_FORMAT_D32_FLOAT || format == DXGI_FORMAT_R32_FLOAT ||
               format == DXGI_FORMAT_R32_TYPELESS || format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT ||
               format == DXGI_FORMAT_R32G8X24_TYPELESS || format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS;
    case Plane::Motion:
        return format == DXGI_FORMAT_R16G16_FLOAT || format == DXGI_FORMAT_R16G16_TYPELESS;
    case Plane::Exposure:
        return format == DXGI_FORMAT_R32_FLOAT || format == DXGI_FORMAT_R32_TYPELESS;
    }
    return false;
}

// R11G11B10 floats share binary16's 5-bit exponent and bias, so each channel
// widens to half exactly by shifting its mantissa; NaN becomes the quiet NaN
// float_to_half produces. Matches convert_row_in bit for bit, without libm.
static uint16_t small_float_to_half(uint32_t bits, int mantissaBits)
{
    const uint32_t exponent = (bits >> mantissaBits) & 0x1fu;
    const uint32_t mantissa = bits & ((1u << mantissaBits) - 1u);
    if (exponent == 0x1f && mantissa != 0)
        return 0x7e00u;
    return static_cast<uint16_t>((exponent << 10) | (mantissa << (10 - mantissaBits)));
}

static void convert_r11g11b10_row(const uint8_t* source, uint16_t* half, UINT width)
{
    for (UINT x = 0; x < width; ++x)
    {
        uint32_t packed;
        std::memcpy(&packed, source + x * 4, 4);
        half[x * 4 + 0] = small_float_to_half(packed & 0x7ffu, 6);
        half[x * 4 + 1] = small_float_to_half((packed >> 11) & 0x7ffu, 6);
        half[x * 4 + 2] = small_float_to_half((packed >> 22) & 0x3ffu, 5);
        half[x * 4 + 3] = 0x3c00u; // 1.0
    }
}

// Converts one row of RGBA16F DLSS output into the game's output format.
static void convert_row_out(DXGI_FORMAT format, const uint16_t* source, uint8_t* destination, UINT width)
{
    for (UINT x = 0; x < width; ++x)
    {
        const uint16_t* texel = source + x * 4;
        switch (format)
        {
        case DXGI_FORMAT_R16G16B16A16_FLOAT:
        case DXGI_FORMAT_R16G16B16A16_TYPELESS:
            std::memcpy(destination + x * 8, texel, 8);
            break;
        case DXGI_FORMAT_R32G32B32A32_FLOAT:
            for (int c = 0; c < 4; ++c)
            {
                const float value = half_to_float(texel[c]);
                std::memcpy(destination + x * 16 + c * 4, &value, 4);
            }
            break;
        case DXGI_FORMAT_R11G11B10_FLOAT:
        {
            const uint32_t packed = float_to_small_float(half_to_float(texel[0]), 6) |
                                    (float_to_small_float(half_to_float(texel[1]), 6) << 11) |
                                    (float_to_small_float(half_to_float(texel[2]), 5) << 22);
            std::memcpy(destination + x * 4, &packed, 4);
            break;
        }
        case DXGI_FORMAT_R10G10B10A2_UNORM:
        case DXGI_FORMAT_R10G10B10A2_TYPELESS:
        {
            const uint32_t packed = to_unorm(half_to_float(texel[0]), 10) |
                                    (to_unorm(half_to_float(texel[1]), 10) << 10) |
                                    (to_unorm(half_to_float(texel[2]), 10) << 20) |
                                    (to_unorm(half_to_float(texel[3]), 2) << 30);
            std::memcpy(destination + x * 4, &packed, 4);
            break;
        }
        case DXGI_FORMAT_B8G8R8A8_UNORM:
        case DXGI_FORMAT_B8G8R8A8_TYPELESS:
            destination[x * 4 + 0] = static_cast<uint8_t>(to_unorm(half_to_float(texel[2]), 8));
            destination[x * 4 + 1] = static_cast<uint8_t>(to_unorm(half_to_float(texel[1]), 8));
            destination[x * 4 + 2] = static_cast<uint8_t>(to_unorm(half_to_float(texel[0]), 8));
            destination[x * 4 + 3] = static_cast<uint8_t>(to_unorm(half_to_float(texel[3]), 8));
            break;
        default: // RGBA8
            for (int c = 0; c < 4; ++c)
                destination[x * 4 + c] = static_cast<uint8_t>(to_unorm(half_to_float(texel[c]), 8));
            break;
        }
    }
}

// --- global state ----------------------------------------------------------------

struct Global
{
    std::mutex mutex;
    bool started = false;
    Worker worker;        // every NGX-core and CUDA call
    std::thread::id workerThread;
    Worker prep, finish;  // CPU-only pipeline stages around it
    HMODULE core = nullptr;
    CoreApi ngx;
    HMODULE cuda = nullptr;
    CudaApi cu;
    bool ngxInitialized = false;
    ID3D12Device* device = nullptr;
    std::vector<std::wstring> paths;
    std::vector<const wchar_t*> pathPointers;
    NgxFeatureCommonInfo featureInfo = {};
    std::wstring dataPath;
    std::atomic<unsigned int> nextHandleId{0x7200};
};

static Global g;

template <typename Function> static bool load_export(HMODULE module, const char* name, Function& function)
{
    function = reinterpret_cast<Function>(reinterpret_cast<void*>(GetProcAddress(module, name)));
    if (function == nullptr)
        logf("missing export %s", name);
    return function != nullptr;
}

// OptiScaler's DLSS-input hook answers kernel32 LoadLibrary* calls for any
// *nvngx*.dll outside the game directory with OptiScaler itself, which would
// hide the official core's CUDA exports. ntdll's LdrLoadDll sits below those
// hooks (OptiScaler's own proxies load libraries the same way).
static HMODULE load_library_below_hooks(const wchar_t* path)
{
    struct UnicodeString
    {
        USHORT Length;
        USHORT MaximumLength;
        PWSTR Buffer;
    };
    using LdrLoadDllFn = LONG(NTAPI*)(PWSTR, PULONG, UnicodeString*, PHANDLE);
    auto ldrLoadDll = reinterpret_cast<LdrLoadDllFn>(
        reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "LdrLoadDll")));
    if (ldrLoadDll == nullptr)
        return LoadLibraryW(path);
    std::wstring copy(path);
    UnicodeString name;
    name.Length = static_cast<USHORT>(copy.size() * sizeof(wchar_t));
    name.MaximumLength = static_cast<USHORT>(name.Length + sizeof(wchar_t));
    name.Buffer = copy.data();
    HANDLE module = nullptr;
    const LONG status = ldrLoadDll(nullptr, nullptr, &name, &module);
    if (status < 0)
    {
        logf("LdrLoadDll(%ls) failed: NTSTATUS 0x%08lx", path, static_cast<unsigned long>(status));
        return nullptr;
    }
    return static_cast<HMODULE>(module);
}

// With DLSS inputs enabled, OptiScaler also hooks ntdll-level loads and
// answers every *nvngx*.dll request with itself unless the requested path
// contains the game EXE's directory in lowercase (its test for "the game's own
// NGX core"). The official core must keep its _nvngx.dll name, so request it
// as <lowercase exe dir>\..\..\<core path>: the hook sees the game
// directory, and Wine collapses the ".." segments to the staged core without
// anything being written into the game folder.
static std::wstring core_path_inside_exe_dir(const wchar_t* corePath)
{
    std::wstring core(corePath);
    wchar_t exe[MAX_PATH] = {};
    const DWORD length = GetModuleFileNameW(nullptr, exe, MAX_PATH);
    if (length == 0 || length >= MAX_PATH || core.size() < 3 || core[1] != L':' || exe[1] != L':' ||
        towlower(core[0]) != towlower(exe[0]))
        return core;
    std::wstring directory(exe);
    directory.resize(directory.find_last_of(L"\\/"));
    for (wchar_t& c : directory)
        c = static_cast<wchar_t>(towlower(c));
    std::wstring path = directory;
    size_t depth = 0;
    for (size_t index = 3; index <= directory.size(); ++index)
        if (index == directory.size() || directory[index] == L'\\' || directory[index] == L'/')
            ++depth;
    for (size_t level = 0; level < depth; ++level)
        path += L"\\..";
    path += L"\\";
    path += core.substr(3);
    logf("requesting NGX core as %ls", path.c_str());
    return path;
}

static bool load_libraries()
{
    DWORD length = 0;
    // Under Proton, nvcuda.dll resolves to Proton's own builtin, which skips the
    // bridge's fixups. Loading the bridge by path first makes every later
    // LoadLibrary("nvcuda.dll"), including the NGX core's, reuse it.
    wchar_t bridgePath[MAX_PATH] = {};
    length = GetEnvironmentVariableW(L"D4R_NVCUDA_BRIDGE", bridgePath, MAX_PATH);
    if (length > 0 && length < MAX_PATH)
    {
        HMODULE bridge = LoadLibraryW(bridgePath);
        logf("preloaded nvcuda bridge %ls: %p (error %lu)", bridgePath, static_cast<void*>(bridge),
             bridge != nullptr ? 0ul : GetLastError());
        // d4r.ini's Linux-side settings, before the bridge's first CUDA call loads ZLUDA
        if (bridge != nullptr && g_portable.active)
        {
            using SetEnvFn = int(WINAPI*)(const char*, const char*, int);
            const auto setEnv = reinterpret_cast<SetEnvFn>(reinterpret_cast<void*>(GetProcAddress(bridge, "d4rSetEnv")));
            if (setEnv == nullptr)
                logf("the nvcuda bridge has no d4rSetEnv; d4r.ini's ZLUDA settings are not applied");
            for (const auto& [name, value] : g_portable.unixEnv)
                if (setEnv != nullptr && setEnv(name.c_str(), value.c_str(), 0))
                    logf("  %s=%s%s", name.c_str(), value.c_str(),
                         getenv(name.c_str()) != nullptr && value != getenv(name.c_str()) ? " (set by the environment)" : "");
        }
    }
    wchar_t corePath[MAX_PATH] = {};
    length = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, MAX_PATH);
    if (length == 0 || length >= MAX_PATH)
    {
        logf("D4R_NGX_CORE is not set; cannot locate the official NGX core");
        return false;
    }
    g.core = load_library_below_hooks(core_path_inside_exe_dir(corePath).c_str());
    if (g.core == nullptr)
    {
        logf("loading D4R_NGX_CORE=%ls failed: %lu", corePath, GetLastError());
        return false;
    }
    bool ok = true;
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_Init", g.ngx.init);
    if (!load_export(g.core, "NVSDK_NGX_CUDA_Init_ProjectID", g.ngx.initProjectId))
        g.ngx.initProjectId = nullptr;
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_Shutdown", g.ngx.shutdown);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetParameters", g.ngx.getParameters);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_AllocateParameters", g.ngx.allocateParameters);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_GetCapabilityParameters", g.ngx.getCapabilityParameters);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_DestroyParameters", g.ngx.destroyParameters);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_CreateFeature", g.ngx.createFeature);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_ReleaseFeature", g.ngx.releaseFeature);
    ok &= load_export(g.core, "NVSDK_NGX_CUDA_EvaluateFeature", g.ngx.evaluateFeature);

    g.cuda = LoadLibraryA("nvcuda.dll");
    if (g.cuda == nullptr)
    {
        logf("LoadLibraryA(nvcuda.dll) failed: %lu", GetLastError());
        return false;
    }
    ok &= load_export(g.cuda, "cuMemAlloc", g.cu.memAlloc);
    ok &= load_export(g.cuda, "cuMemFree", g.cu.memFree);
    ok &= load_export(g.cuda, "cuArrayCreate", g.cu.arrayCreate);
    ok &= load_export(g.cuda, "cuArray3DCreate", g.cu.array3DCreate);
    ok &= load_export(g.cuda, "cuArrayDestroy", g.cu.arrayDestroy);
    ok &= load_export(g.cuda, "cuMemcpy2D", g.cu.memcpy2D);
    ok &= load_export(g.cuda, "cuTexObjectCreate", g.cu.texObjectCreate);
    ok &= load_export(g.cuda, "cuTexObjectDestroy", g.cu.texObjectDestroy);
    ok &= load_export(g.cuda, "cuSurfObjectCreate", g.cu.surfObjectCreate);
    ok &= load_export(g.cuda, "cuSurfObjectDestroy", g.cu.surfObjectDestroy);
    ok &= load_export(g.cuda, "cuCtxSynchronize", g.cu.ctxSynchronize);
    // The d4r bridge's own export keeps waiting when the application's syncs are elided
    // (D4R_ELIDE_NGX_SYNC); cuCtxSynchronize above stays the fallback.
    {
        decltype(g.cu.ctxSynchronize) ownSync = nullptr;
        if (load_export(g.cuda, "d4rCtxSynchronize", ownSync))
            g.cu.ctxSynchronize = ownSync;
    }
    if (!load_export(g.cuda, "cuMemAllocHost", g.cu.memAllocHost) ||
        !load_export(g.cuda, "cuMemFreeHost", g.cu.memFreeHost) ||
        !load_export(g.cuda, "cuMemcpyHtoDAsync", g.cu.memcpyHtoDAsync) ||
        !load_export(g.cuda, "cuMemcpyDtoHAsync", g.cu.memcpyDtoHAsync) ||
        !load_export(g.cuda, "cuStreamCreate", g.cu.streamCreate) ||
        !load_export(g.cuda, "cuStreamSynchronize", g.cu.streamSynchronize))
    {
        g.cu.memAllocHost = nullptr;
        logf("overlapped staging unavailable; using synchronous copies");
    }
    if (!load_export(g.cuda, "d4rMemcpy2DAsync", g.cu.memcpy2DAsync))
        g.cu.memcpy2DAsync = nullptr;
    if (!load_export(g.cuda, "d4rStreamWaitValue32", g.cu.streamWaitValue32) ||
        !load_export(g.cuda, "d4rWriteValue32", g.cu.writeValue32))
        g.cu.streamWaitValue32 = nullptr, g.cu.writeValue32 = nullptr;
    if (!load_export(g.cuda, "d4rSetArrayRedirect", g.cu.setArrayRedirect))
        g.cu.setArrayRedirect = nullptr;
    if (!load_export(g.cuda, "d4rOutputKernelNative", g.cu.outputKernelNative))
        g.cu.outputKernelNative = nullptr;
    if (!load_export(g.cuda, "d4rRegisterLinearTexture", g.cu.registerLinearTexture))
        g.cu.registerLinearTexture = nullptr;
    if (profile_enabled())
    {
        const bool events = load_export(g.cuda, "cuEventCreate", g.cu.eventCreate) &&
                            load_export(g.cuda, "cuEventRecord", g.cu.eventRecord) &&
                            load_export(g.cuda, "cuEventElapsedTime", g.cu.eventElapsedTime) &&
                            load_export(g.cuda, "cuEventDestroy", g.cu.eventDestroy);
        if (!events)
            logf("D4R_PROFILE: CUDA event timing unavailable; CPU stage timings remain enabled");
    }
    logf("loaded NGX core %ls and nvcuda.dll: %s", corePath, ok ? "all exports present" : "exports missing");
    return ok;
}

// A game that initialises NGX with its project ID (engine integrations such as Unreal's) is identified
// to the NGX core the same way.
struct ProjectIdentity
{
    std::string id;
    int engineType = 0;
    std::string engineVersion;
};

static NgxResult initialize(unsigned long long applicationId, const wchar_t* dataPath, ID3D12Device* device,
                            const NgxFeatureCommonInfo* featureInfo, unsigned int sdkVersion,
                            const ProjectIdentity* project = nullptr)
{
    std::lock_guard<std::mutex> lock(g.mutex);
    if (g.ngxInitialized)
        return NGX_SUCCESS;
    if (!g.started)
    {
        g.worker.start();
        g.workerThread = g.worker.call([] { return std::this_thread::get_id(); });
        g.prep.start();
        g.finish.start();
        g.started = true;
    }
    ensure_portable_config();
    if (g_portable.active && !g.core)
    {
        logf("portable install in %ls (settings from d4r.ini, Linux path %s)", g_portable.dir.c_str(),
             g_portable.unixDir.c_str());
        for (const std::string& note : g_portable.notes)
            logf("d4r: %s", note.c_str());
    }
    if (g.core == nullptr && !load_libraries())
        return NGX_FAIL_PLATFORM_ERROR;
    if (device != nullptr)
    {
        device->AddRef();
        g.device = device;
        // NGX only accepts a CUDA device whose LUID matches the D3D12 adapter;
        // the nvcuda bridge reports this LUID from cuDeviceGetLuid.
        LUID luid;
        device->GetAdapterLuid(&luid);
        char low[24], high[24];
        std::snprintf(low, sizeof(low), "0x%08lx", static_cast<unsigned long>(luid.LowPart));
        std::snprintf(high, sizeof(high), "0x%08lx", static_cast<unsigned long>(luid.HighPart));
        SetEnvironmentVariableA("D4R_CUDA_LUID_LOW", low);
        SetEnvironmentVariableA("D4R_CUDA_LUID_HIGH", high);
        SetEnvironmentVariableA("D4R_CUDA_NODE_MASK", "1");
        logf("CUDA device LUID set to the D3D12 adapter LUID %s:%s", high, low);
    }

    g.paths.clear();
    wchar_t featureDirectory[MAX_PATH] = {};
    DWORD length = GetEnvironmentVariableW(L"D4R_NGX_FEATURE_DIR", featureDirectory, MAX_PATH);
    if (length > 0 && length < MAX_PATH)
        g.paths.emplace_back(featureDirectory);
    if (featureInfo != nullptr && featureInfo->PathListInfo.Path != nullptr)
        for (unsigned int index = 0; index < featureInfo->PathListInfo.Length; ++index)
            if (featureInfo->PathListInfo.Path[index] != nullptr)
                g.paths.emplace_back(featureInfo->PathListInfo.Path[index]);
    g.pathPointers.clear();
    for (const std::wstring& path : g.paths)
    {
        g.pathPointers.push_back(path.c_str());
        logf("feature search path: %ls", path.c_str());
    }
    g.featureInfo = {};
    g.featureInfo.PathListInfo.Path = g.pathPointers.data();
    g.featureInfo.PathListInfo.Length = static_cast<unsigned int>(g.pathPointers.size());
    g.featureInfo.LoggingInfo.LoggingCallback = reinterpret_cast<void*>(&ngx_log_callback);
    g.featureInfo.LoggingInfo.MinimumLoggingLevel = static_cast<int>(env_uint("D4R_NGX_LOG_LEVEL", 1)); // 1=on, 2=verbose
    g.featureInfo.LoggingInfo.DisableOtherLoggingSinks = false;
    g.dataPath = dataPath != nullptr ? dataPath : L".";

    // The CUDA path takes the same application identity as the game's D3D12 init. (The SDK sample's id
    // is only a last resort: NGX stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output for it.)
    const unsigned long long id = applicationId != 0 ? applicationId : 241534723ULL;
    const unsigned int version = sdkVersion != 0 ? sdkVersion : 0x15;
    // Logging info is only read for API versions 0x14 and later.
    const unsigned int apiVersion = version < 0x14 ? 0x14 : version;
    NgxResult result;
    if (project != nullptr && !project->id.empty() && g.ngx.initProjectId != nullptr)
    {
        result = g.worker.call([&] {
            return g.ngx.initProjectId(project->id.c_str(), project->engineType, project->engineVersion.c_str(),
                                       g.dataPath.c_str(), apiVersion, &g.featureInfo);
        });
        logf("NVSDK_NGX_CUDA_Init_ProjectID(project=%s, engine=%d %s, sdk=0x%x) -> 0x%08x", project->id.c_str(),
             project->engineType, project->engineVersion.c_str(), version, result);
    }
    else
    {
        result = g.worker.call([&] { return g.ngx.init(id, g.dataPath.c_str(), &g.featureInfo, apiVersion); });
        logf("NVSDK_NGX_CUDA_Init(app=%llu, sdk=0x%x) -> 0x%08x", id, version, result);
    }
    using LoadErrorFn = const char*(WINAPI*)();
    const auto loadError = reinterpret_cast<LoadErrorFn>(reinterpret_cast<void*>(GetProcAddress(g.cuda, "d4rLoadError")));
    if (loadError != nullptr && loadError()[0] != '\0')
        logf("nvcuda bridge: %s", loadError());
    g.ngxInitialized = result == NGX_SUCCESS;
    return result;
}

// --- per-feature state ---------------------------------------------------------

constexpr int kSlots = 3;
// One more result slot than input slots: a slot is reused only once the GPU
// is past every command list that copies from it (see claim_output_slot).
constexpr int kOutputSlots = kSlots + 1;

// A VkBuffer in device-local memory on vkd3d-proton's VkDevice, exported to
// the CUDA side (see "VRAM interop").
struct VramBuffer
{
    VkBuffer buffer = VK_NULL_HANDLE;
    VkDeviceMemory memory = VK_NULL_HANDLE;
    CudaDevicePtr device = 0;
    void* external = nullptr;
    size_t bytes = 0;
};

// A temporary image for colour or exposure format conversion.
struct VramImage
{
    VkImage image = VK_NULL_HANDLE;
    VkDeviceMemory memory = VK_NULL_HANDLE;
    UINT width = 0, height = 0;
    VkFormat format = VK_FORMAT_UNDEFINED;
};

struct Staging
{
    ID3D12Resource* buffer = nullptr;
    uint8_t* mapped = nullptr;
    D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout = {};
    UINT rows = 0;
    UINT64 total = 0;
    DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN;
    UINT width = 0;
    UINT height = 0;

    void release()
    {
        if (buffer != nullptr)
            buffer->Release();
        *this = Staging{};
    }
};

struct CudaImage
{
    CudaArray array = nullptr;
    CudaObject object = 0; // texture or surface object
    UINT width = 0;
    UINT height = 0;
};

struct FrameParams
{
    float jitterX = 0, jitterY = 0, mvScaleX = 1, mvScaleY = 1, sharpness = 0;
    float preExposure = 1, exposureScale = 1, frameTime = 16.6f;
    int reset = 0, invertX = 0, invertY = 0;
    unsigned int renderWidth = 0, renderHeight = 0;
    unsigned int colorBaseX = 0, colorBaseY = 0, depthBaseX = 0, depthBaseY = 0;
    unsigned int mvBaseX = 0, mvBaseY = 0, outputBaseX = 0, outputBaseY = 0;
    bool hasExposure = false;
    bool vram = false; // inputs and output stay in VRAM (VRAM interop)
    bool split = false; // this frame presents its own result (split frames)
};

struct FrameTiming
{
    bool enabled = false;
    ProfileClock::time_point gameStart, queuedAt;
    double slotWait = 0, inputRecord = 0, outputRecord = 0, gameCall = 0, gameInterval = 0;
    double convert[4] = {}, upload[4] = {};
    double download = 0, outputConvert = 0;
    double prep = 0, worker = 0, workerWait = 0, finishWait = 0, markerWait = 0;
    double ngxHost = 0, ctxSync = 0, gpuEval = -1;
    double h2dTotal = 0, d2hIssue = 0, d2hWait = 0;
    ProfileClock::time_point prepStart, workerQueued, finishQueued;
    size_t queueDepth = 0;
    uint32_t presentedFrame = 0;
};

// A plane in the canonical CUDA layout, tightly packed, in ordinary host
// memory: filled from the D3D12 staging by one pipeline stage and consumed by
// the next (see prepare_inputs / run_evaluation / publish_output).
struct HostPlane
{
    uint8_t* bytes = nullptr; // page-locked when overlapped staging is on
    size_t capacity = 0;
    bool pinned = false;
    std::vector<uint8_t> fallback;
    UINT width = 0, height = 0;
    size_t rowBytes = 0;
    size_t size() const { return rowBytes * height; }
};

struct InputSlot
{
    Staging color, depth, motion, exposure;
    VramBuffer vram[4]; // VRAM interop: the planes in canonical layout
    // D4R_SHIM_LINEAR_INPUTS: pitch-linear texture objects on vram[] handed to NGX (worker only)
    CudaObject linearTexture[4] = {};
    CudaDevicePtr linearPointer[4] = {};
    UINT linearWidth[4] = {}, linearHeight[4] = {};
    size_t linearPitch[4] = {};
    std::atomic<bool> busy{false}; // readback staging owned by the pipeline
    HostPlane host[4];
    std::atomic<bool> hostBusy{false}; // host planes not yet uploaded by the worker
};

struct OutputSlot
{
    Staging staging;
    VramBuffer vram; // VRAM interop: the result in RGBA16F
    std::atomic<uint32_t> lastReadFrame{0}; // last frame whose command list copies from it
    std::atomic<uint32_t> producedFrame{0};
};

struct Feature
{
    NgxHandle handle = {}; // returned to the caller
    NgxHandle* cudaHandle = nullptr;
    void* cudaParams = nullptr;
    CudaDevicePtr scratch = 0;
    unsigned int width = 0, height = 0, outWidth = 0, outHeight = 0;
    int quality = 0, flags = 0;
    unsigned int preset = 0;
    CudaEvent profileStart = nullptr, profileEnd = nullptr;
    bool profileEventsReady = false;

    CudaImage color, depth, motion, exposure, output;
    CudaObject colorHandle = 0, depthHandle = 0, motionHandle = 0, exposureHandle = 0, outputHandle = 0;

    InputSlot inputs[kSlots];
    OutputSlot outputs[kOutputSlots];
    HostPlane outputHost[2]; // downloaded results waiting for publish_output
    std::atomic<bool> outputHostBusy[2] = {false, false};
    unsigned int nextOutputHost = 0; // worker only
    // Overlapped staging (worker only): linear device buffers the async copies
    // target, moved to/from the CUDA arrays with device-to-device copies.
    CudaDevicePtr planeLinear[4] = {}, outputLinear = 0;
    size_t planeLinearBytes[4] = {}, outputLinearBytes = 0;
    // Frames queued but not yet published (or dropped). evaluate() blocks the
    // game at D4R_SHIM_MAX_IN_FLIGHT (default 3) so a backlog cannot build up
    // between pipeline stages and age the presented result.
    std::atomic<int> inFlight{0};
    DXGI_FORMAT outputFormat = DXGI_FORMAT_UNKNOWN;
    UINT outputWidth = 0, outputHeight = 0;
    // VRAM interop: decided on the first evaluation, dropped for good if the
    // formats later stop qualifying.
    bool vram = false, vramDecided = false;
    VramImage colorConversion;
    VramImage exposureConversion;
    VramImage outputConversion;
    // Split frames: the rest of frame N's command list is submitted only once
    // splitSemaphore reaches N (signalled when N retires, or by the watchdog).
    bool split = false;
    VkSemaphore splitSemaphore = VK_NULL_HANDLE;
    std::mutex splitMutex;
    uint64_t splitSignalled = 0; // under splitMutex
    // GPU-side start (D4R_SHIM_GPU_WAIT, split frames): the input copies end with a vkCmdFillBuffer of the
    // frame number into gpuMarker, and the null stream waits for it on the GPU instead of the CPU polling
    // the frame marker before any DLSS work is queued.
    bool gpuWait = false;
    VramBuffer gpuMarker;
    // Frame the null stream currently waits for (0: none) and since when; the watchdog releases a wait for a
    // frame whose command list never reaches the GPU (e.g. recorded but not executed), so DLSS cannot stall.
    std::atomic<uint32_t> gpuWaitFrame{0};
    std::atomic<int64_t> gpuWaitSince{0};
    // D4R_SHIM_OUTPUT_DIRECT: this frame's result is stored by the native output kernel straight into its
    // destination buffer (worker only), so the array -> buffer copy is skipped.
    bool outputRedirected = false;
    bool outputDirectAllowed = false; // the forced preset's output kernel honours the redirect
    // D4R_SHIM_LINEAR_INPUTS (with gpuWait): NGX samples the interop buffers directly through pitch-linear
    // texture objects (rows padded to 256 bytes), so the buffer -> array copies disappear.
    bool linearInputs = false;
    std::vector<VramBuffer> retiredBuffers; // may still be read by queued command lists
    std::vector<VramImage> retiredImages;
    ID3D12Resource* marker = nullptr;
    volatile uint32_t* markerValue = nullptr;
    uint32_t frame = 0;
    std::atomic<int> latestOutput{-1};
    std::atomic<uint32_t> completedFrames{0};
    uint32_t evaluatedFrames = 0; // worker only
    // Serializes picking the slot to present (game thread) against picking
    // the slot to overwrite (worker).
    std::mutex outputMutex;
};

static std::mutex g_featuresMutex;
static std::vector<Feature*> g_features;

static bool create_buffer(D3D12_HEAP_TYPE heapType, UINT64 size, ID3D12Resource** buffer, uint8_t** mapped)
{
    D3D12_HEAP_PROPERTIES heap = {};
    heap.Type = heapType;
    D3D12_RESOURCE_DESC desc = {};
    desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
    desc.Width = size;
    desc.Height = 1;
    desc.DepthOrArraySize = 1;
    desc.MipLevels = 1;
    desc.SampleDesc.Count = 1;
    desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
    const D3D12_RESOURCE_STATES state =
        heapType == D3D12_HEAP_TYPE_READBACK ? D3D12_RESOURCE_STATE_COPY_DEST : D3D12_RESOURCE_STATE_GENERIC_READ;
    HRESULT hr = g.device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, state, nullptr,
                                                   __uuidof(ID3D12Resource), reinterpret_cast<void**>(buffer));
    if (FAILED(hr))
    {
        logf("CreateCommittedResource(heap=%d, %llu bytes) failed: 0x%08lx", heapType,
             static_cast<unsigned long long>(size), hr);
        return false;
    }
    D3D12_RANGE readRange = {0, heapType == D3D12_HEAP_TYPE_READBACK ? static_cast<SIZE_T>(size) : 0};
    hr = (*buffer)->Map(0, &readRange, reinterpret_cast<void**>(mapped));
    if (FAILED(hr))
    {
        logf("Map failed: 0x%08lx", hr);
        (*buffer)->Release();
        *buffer = nullptr;
        return false;
    }
    return true;
}

// Ensures staging matches the resource's first subresource; heap selects
// readback (inputs) or upload (output).
static bool ensure_staging(Staging& staging, ID3D12Resource* resource, D3D12_HEAP_TYPE heapType)
{
    D3D12_RESOURCE_DESC desc;
    resource->GetDesc(&desc);
    if (staging.buffer != nullptr && staging.format == desc.Format && staging.width == desc.Width &&
        staging.height == desc.Height)
        return true;
    staging.release();
    UINT64 rowSize = 0;
    g.device->GetCopyableFootprints(&desc, 0, 1, 0, &staging.layout, &staging.rows, &rowSize, &staging.total);
    if (!create_buffer(heapType, staging.total, &staging.buffer, &staging.mapped))
        return false;
    staging.format = desc.Format;
    staging.width = static_cast<UINT>(desc.Width);
    staging.height = desc.Height;
    logf("staging %s %ux%u format=%d rowPitch=%u total=%llu", heapType == D3D12_HEAP_TYPE_READBACK ? "readback" : "upload",
         staging.width, staging.height, staging.format, staging.layout.Footprint.RowPitch,
         static_cast<unsigned long long>(staging.total));
    return true;
}

static void transition(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES before,
                       D3D12_RESOURCE_STATES after)
{
    if (before == after)
        return;
    D3D12_RESOURCE_BARRIER barrier = {};
    barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
    barrier.Transition.pResource = resource;
    barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
    barrier.Transition.StateBefore = before;
    barrier.Transition.StateAfter = after;
    list->ResourceBarrier(1, &barrier);
}

static void copy_to_staging(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, Staging& staging,
                            D3D12_RESOURCE_STATES state)
{
    transition(list, resource, state, D3D12_RESOURCE_STATE_COPY_SOURCE);
    D3D12_TEXTURE_COPY_LOCATION destination = {};
    destination.pResource = staging.buffer;
    destination.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
    destination.PlacedFootprint = staging.layout;
    D3D12_TEXTURE_COPY_LOCATION source = {};
    source.pResource = resource;
    source.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
    source.SubresourceIndex = 0;
    list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
    transition(list, resource, D3D12_RESOURCE_STATE_COPY_SOURCE, state);
}

// --- CUDA-side resources (worker thread) -----------------------------------------

// Sampler of an input texture: clamp addressing, normalized coordinates (as NGX's own textures); bilinear
// for the colour input (D4R_SHIM_COLOR_LINEAR), optionally for motion / depth (experiments).
static CudaTextureDesc input_sampler(UINT width, uint32_t channels, bool linear)
{
    CudaTextureDesc sampler = {};
    sampler.addressMode[0] = 1;
    sampler.addressMode[1] = 1;
    static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
    sampler.filterMode = linear && colorLinear ? 1 : 0;
    static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
    static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
    if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
        sampler.filterMode = 1;
    sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
    return sampler;
}

static bool ensure_cuda_image(CudaImage& image, UINT width, UINT height, uint32_t format, uint32_t channels,
                              bool surface, bool linear)
{
    if (image.array != nullptr && image.width == width && image.height == height)
        return true;
    if (image.object != 0)
        surface ? g.cu.surfObjectDestroy(image.object) : g.cu.texObjectDestroy(image.object);
    if (image.array != nullptr)
        g.cu.arrayDestroy(image.array);
    image = CudaImage{};
    int result;
    if (surface)
    {
        const CudaArray3DDescriptor descriptor{width, height, 0, format, channels, 2u /* SURFACE_LDST */};
        result = g.cu.array3DCreate(&image.array, &descriptor);
    }
    else
    {
        const CudaArrayDescriptor descriptor{width, height, format, channels};
        result = g.cu.arrayCreate(&image.array, &descriptor);
    }
    if (result != 0)
    {
        logf("CUDA array create %ux%u fmt=0x%x ch=%u failed: %d", width, height, format, channels, result);
        return false;
    }
    CudaResourceDesc resource = {};
    resource.resType = 0; // CU_RESOURCE_TYPE_ARRAY
    resource.res.array.hArray = image.array;
    if (surface)
        result = g.cu.surfObjectCreate(&image.object, &resource);
    else
    {
        // Matches the texture objects NGX itself creates: clamp addressing,
        // normalized coordinates.
        CudaTextureDesc sampler = {};
        sampler.addressMode[0] = 1;
        sampler.addressMode[1] = 1;
        // D4R_SHIM_COLOR_LINEAR=0 samples the colour input with point filtering (experiment).
        static const bool colorLinear = env_uint("D4R_SHIM_COLOR_LINEAR", 1) != 0;
        sampler.filterMode = linear && colorLinear ? 1 : 0;
        // D4R_SHIM_MOTION_LINEAR / D4R_SHIM_DEPTH_LINEAR=1: bilinear filtering for
        // those planes (identified by format; experiment).
        static const bool motionLinear = env_uint("D4R_SHIM_MOTION_LINEAR", 0) != 0;
        static const bool depthLinear = env_uint("D4R_SHIM_DEPTH_LINEAR", 0) != 0;
        if ((motionLinear && channels == 2) || (depthLinear && channels == 1 && width > 1))
            sampler.filterMode = 1;
        sampler.flags = 2; // CU_TRSF_NORMALIZED_COORDINATES
        result = g.cu.texObjectCreate(&image.object, &resource, &sampler, nullptr);
    }
    if (result != 0)
    {
        logf("CUDA %s object create failed: %d", surface ? "surface" : "texture", result);
        return false;
    }
    image.width = width;
    image.height = height;
    return true;
}

// Device <-> array copy. Queued on the null stream, where NGX evaluates, when the bridge provides
// it (D4R_SHIM_ASYNC_2D=0: ZLUDA's cuMemcpy2D); callers synchronize before the data is used.
static int copy_2d(const CudaMemcpy2D& copy)
{
    static const bool async = env_uint("D4R_SHIM_ASYNC_2D", 1) != 0;
    if (async && g.cu.memcpy2DAsync != nullptr && g.cu.memcpy2DAsync(&copy, nullptr) == 0)
        return 0;
    return g.cu.memcpy2D(&copy);
}

// Splits row-wise staging work (tens of MB per frame at 1440p, which one core
// moves at ~11 GB/s) across helper threads plus the calling worker thread.
// D4R_SHIM_COPY_THREADS sets the total thread count (default 4, 1 = serial).
class RowPool
{
public:
    void run(UINT rows, size_t bytes, const std::function<void(UINT, UINT)>& task)
    {
        const unsigned int parts = threads();
        if (parts <= 1 || bytes < (2u << 20) || rows < parts * 8)
        {
            task(0, rows);
            return;
        }
        {
            std::lock_guard<std::mutex> lock(mutex_);
            task_ = &task;
            rows_ = rows;
            parts_ = parts;
            remaining_ = parts - 1;
            ++generation_;
        }
        start_.notify_all();
        task(0, rows / parts);
        std::unique_lock<std::mutex> lock(mutex_);
        done_.wait(lock, [this] { return remaining_ == 0; });
        task_ = nullptr;
    }

private:
    unsigned int threads()
    {
        std::call_once(once_, [this] {
            count_ = env_uint("D4R_SHIM_COPY_THREADS", 4);
            count_ = count_ < 1 ? 1 : (count_ > 16 ? 16 : count_);
            for (unsigned int index = 1; index < count_; ++index)
                std::thread([this, index] { helper(index); }).detach();
        });
        return count_;
    }

    void helper(unsigned int index)
    {
        uint64_t seen = 0;
        for (;;)
        {
            const std::function<void(UINT, UINT)>* task;
            UINT begin, end;
            {
                std::unique_lock<std::mutex> lock(mutex_);
                start_.wait(lock, [&] { return generation_ != seen; });
                seen = generation_;
                task = task_;
                begin = static_cast<UINT>(static_cast<uint64_t>(rows_) * index / parts_);
                end = static_cast<UINT>(static_cast<uint64_t>(rows_) * (index + 1) / parts_);
            }
            (*task)(begin, end);
            {
                std::lock_guard<std::mutex> lock(mutex_);
                --remaining_;
            }
            done_.notify_one();
        }
    }

    std::once_flag once_;
    unsigned int count_ = 1;
    std::mutex mutex_;
    std::condition_variable start_, done_;
    const std::function<void(UINT, UINT)>* task_ = nullptr;
    UINT rows_ = 0;
    unsigned int parts_ = 1, remaining_ = 0;
    uint64_t generation_ = 0;
};

// One pool per CPU pipeline stage; a pool serves one caller at a time.
static RowPool g_inputRows, g_outputRows;

// Copies rows between pitched buffers.
static void copy_rows(RowPool& pool, uint8_t* destination, size_t destinationPitch, const uint8_t* source,
                      size_t sourcePitch, size_t rowBytes, UINT rows)
{
    pool.run(rows, rowBytes * rows, [&](UINT begin, UINT end) {
        if (destinationPitch == rowBytes && sourcePitch == rowBytes)
            std::memcpy(destination + begin * rowBytes, source + begin * rowBytes, rowBytes * (end - begin));
        else
            for (UINT y = begin; y < end; ++y)
                std::memcpy(destination + y * destinationPitch, source + y * sourcePitch, rowBytes);
    });
}

// Overlapped staging: frame N's uploads run while frame N-1's result downloads
// (PCIe is full duplex), as async copies between page-locked host memory and
// linear device buffers on two non-blocking streams. D4R_SHIM_ASYNC_COPIES=0
// keeps the synchronous host<->array copies.
static bool async_copies()
{
    static const bool enabled = env_uint("D4R_SHIM_ASYNC_COPIES", 1) != 0;
    return enabled && g.cu.memAllocHost != nullptr;
}

struct CopyStreams
{
    void* up = nullptr;
    void* down = nullptr;
};

// Worker thread only.
static CopyStreams* copy_streams()
{
    static CopyStreams streams;
    static bool attempted = false;
    if (!attempted)
    {
        attempted = true;
        constexpr unsigned int kNonBlocking = 1; // CU_STREAM_NON_BLOCKING
        if (g.cu.streamCreate(&streams.up, kNonBlocking) != 0 || g.cu.streamCreate(&streams.down, kNonBlocking) != 0)
        {
            logf("cuStreamCreate failed; using synchronous copies");
            streams = {};
        }
    }
    return streams.up != nullptr ? &streams : nullptr;
}

// Grows a host plane's buffer. Page-locked memory is allocated on the CUDA
// worker thread (the only one with a current context).
static void reserve_host(HostPlane& host, size_t bytes)
{
    if (bytes <= host.capacity)
        return;
    auto allocate = [&host, bytes] {
        if (host.pinned)
            g.cu.memFreeHost(host.bytes);
        host = HostPlane{};
        void* pinned = nullptr;
        if (async_copies() && g.cu.memAllocHost(&pinned, bytes) == 0)
        {
            host.bytes = static_cast<uint8_t*>(pinned);
            host.pinned = true;
        }
        else
        {
            host.fallback.resize(bytes);
            host.bytes = host.fallback.data();
        }
        host.capacity = bytes;
        return 0;
    };
    if (async_copies() && std::this_thread::get_id() != g.workerThread)
        g.worker.call(allocate);
    else
        allocate();
}

static void release_host(HostPlane& host)
{
    if (host.pinned)
        g.cu.memFreeHost(host.bytes);
    host = HostPlane{};
}

// Worker thread only.
static bool ensure_linear(CudaDevicePtr& buffer, size_t& capacity, size_t bytes)
{
    if (bytes <= capacity)
        return true;
    if (buffer != 0)
        g.cu.memFree(buffer);
    buffer = 0;
    capacity = 0;
    if (g.cu.memAlloc(&buffer, bytes) != 0)
    {
        buffer = 0;
        logf("cuMemAlloc(%zu) for staging failed", bytes);
        return false;
    }
    capacity = bytes;
    return true;
}

static uint32_t plane_format(Plane plane)
{
    return (plane == Plane::Color || plane == Plane::Motion) ? CUDA_FORMAT_HALF : CUDA_FORMAT_FLOAT;
}

static uint32_t plane_channels(Plane plane)
{
    return plane == Plane::Color ? 4 : (plane == Plane::Motion ? 2 : 1);
}

// --- VRAM interop ------------------------------------------------------------------
//
// With D4R_SHIM_VRAM_INTEROP=1 the inputs and the result never leave VRAM.
// vkd3d-proton's interop interface hands out the game's VkDevice, the VkImage
// behind each D3D12 texture, and the raw VkCommandBuffer behind the game's
// D3D12 command list. evaluate() records Vulkan copies of the inputs into
// exportable device-local buffers (colour formats other than RGBA16F are
// converted by a blit first), and of the latest result out of one; the Wine
// nvcuda bridge (d4rImportVulkanMemory) maps those buffers into CUDA. The
// worker then only moves data between them and the CUDA arrays on the GPU.
// Frames whose formats do not qualify use the readback/upload path.

MIDL_INTERFACE("39da4e09-bd1c-4198-9fae-86bbe3be41fd")
ID3D12DXVKInteropDevice : public IUnknown
{
    virtual HRESULT STDMETHODCALLTYPE GetDXGIAdapter(REFIID iid, void** object) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetInstanceExtensions(UINT* count, const char** extensions) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetDeviceExtensions(UINT* count, const char** extensions) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetDeviceFeatures(const VkPhysicalDeviceFeatures2** features) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetVulkanHandles(VkInstance* instance, VkPhysicalDevice* physical,
                                                       VkDevice* device) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetVulkanQueueInfo(ID3D12CommandQueue* queue, VkQueue* vkQueue,
                                                         UINT32* family) = 0;
    virtual void STDMETHODCALLTYPE GetVulkanImageLayout(ID3D12Resource* resource, D3D12_RESOURCE_STATES state,
                                                        VkImageLayout* layout) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetVulkanResourceInfo(ID3D12Resource* resource, UINT64* handle,
                                                            UINT64* offset) = 0;
    virtual HRESULT STDMETHODCALLTYPE LockCommandQueue(ID3D12CommandQueue* queue) = 0;
    virtual HRESULT STDMETHODCALLTYPE UnlockCommandQueue(ID3D12CommandQueue* queue) = 0;
};

MIDL_INTERFACE("902d8115-59eb-4406-9518-fe00f991ee65")
ID3D12DXVKInteropDevice1 : public ID3D12DXVKInteropDevice
{
    virtual HRESULT STDMETHODCALLTYPE GetVulkanResourceInfo1(ID3D12Resource* resource, UINT64* handle, UINT64* offset,
                                                             VkFormat* format) = 0;
    virtual HRESULT STDMETHODCALLTYPE CreateInteropCommandQueue(const D3D12_COMMAND_QUEUE_DESC* desc, UINT32 family,
                                                                ID3D12CommandQueue** queue) = 0;
    virtual HRESULT STDMETHODCALLTYPE CreateInteropCommandAllocator(D3D12_COMMAND_LIST_TYPE type, UINT32 family,
                                                                    ID3D12CommandAllocator** allocator) = 0;
    virtual HRESULT STDMETHODCALLTYPE BeginVkCommandBufferInterop(ID3D12CommandList* list, VkCommandBuffer* buffer) = 0;
    virtual HRESULT STDMETHODCALLTYPE EndVkCommandBufferInterop(ID3D12CommandList* list) = 0;
};
__CRT_UUID_DECL(ID3D12DXVKInteropDevice1, 0x902d8115, 0x59eb, 0x4406, 0x95, 0x18, 0xfe, 0x00, 0xf9, 0x91, 0xee, 0x65)

// Added by the d4r vkd3d-proton patch (patches/vkd3d-proton/): ends the list's
// current Vulkan command buffer; what is recorded afterwards is submitted
// separately, with a queue-level wait for the timeline semaphore value.
MIDL_INTERFACE("5a7c8b3e-2f61-4d0e-9c1a-7e3b52d4a901")
ID3D12DXVKInteropDeviceD4R : public ID3D12DXVKInteropDevice1
{
    virtual HRESULT STDMETHODCALLTYPE LockVulkanQueue(ID3D12CommandQueue* queue) = 0;
    virtual HRESULT STDMETHODCALLTYPE UnlockVulkanQueue(ID3D12CommandQueue* queue) = 0;
    virtual HRESULT STDMETHODCALLTYPE GetVulkanHeapInfo(ID3D12Heap* heap, UINT64* memory, UINT64* offset,
                                                        UINT32* type) = 0;
    virtual HRESULT STDMETHODCALLTYPE SplitCommandListForExternalWait(ID3D12CommandList* list, UINT64 semaphore,
                                                                      UINT64 value) = 0;
};
__CRT_UUID_DECL(ID3D12DXVKInteropDeviceD4R, 0x5a7c8b3e, 0x2f61, 0x4d0e, 0x9c, 0x1a, 0x7e, 0x3b, 0x52, 0xd4, 0xa9, 0x01)

struct VulkanInterop
{
    std::once_flag once;
    bool ready = false;
    ID3D12DXVKInteropDevice1* interop = nullptr;
    ID3D12DXVKInteropDeviceD4R* split = nullptr; // only with the d4r vkd3d-proton patch
    VkDevice device = VK_NULL_HANDLE;
    VkPhysicalDeviceMemoryProperties memory = {};
    VkPhysicalDevice physical = VK_NULL_HANDLE;
    PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
    PFN_vkCreateBuffer createBuffer = nullptr;
    PFN_vkDestroyBuffer destroyBuffer = nullptr;
    PFN_vkGetBufferMemoryRequirements bufferRequirements = nullptr;
    PFN_vkBindBufferMemory bindBuffer = nullptr;
    PFN_vkCreateImage createImage = nullptr;
    PFN_vkDestroyImage destroyImage = nullptr;
    PFN_vkGetImageMemoryRequirements imageRequirements = nullptr;
    PFN_vkBindImageMemory bindImage = nullptr;
    PFN_vkAllocateMemory allocate = nullptr;
    PFN_vkFreeMemory free = nullptr;
    PFN_vkCmdPipelineBarrier barrier = nullptr;
    PFN_vkCmdCopyImageToBuffer copyImageToBuffer = nullptr;
    PFN_vkCmdCopyBufferToImage copyBufferToImage = nullptr;
    PFN_vkCmdBlitImage blit = nullptr;
    PFN_vkCmdFillBuffer fill = nullptr;
    PFN_vkCreateSemaphore createSemaphore = nullptr;
    PFN_vkDestroySemaphore destroySemaphore = nullptr;
    PFN_vkSignalSemaphore signalSemaphore = nullptr;
    int(WINAPI* import)(VkDevice, uint64_t, uint64_t, CudaDevicePtr*, void**) = nullptr;
    int(WINAPI* release)(void*) = nullptr;
};
static VulkanInterop g_vk;

static bool vram_interop_requested()
{
    static const bool requested = env_uint("D4R_SHIM_VRAM_INTEROP", 0) != 0;
    return requested;
}

// Game thread, once.
static void init_vram_interop()
{
    if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDevice1), reinterpret_cast<void**>(&g_vk.interop))))
    {
        logf("VRAM interop: ID3D12DXVKInteropDevice1 unavailable (not vkd3d-proton?)");
        return;
    }
    VkInstance instance = VK_NULL_HANDLE;
    VkPhysicalDevice physical = VK_NULL_HANDLE;
    if (FAILED(g_vk.interop->GetVulkanHandles(&instance, &physical, &g_vk.device)))
    {
        logf("VRAM interop: GetVulkanHandles failed");
        return;
    }
    UINT count = 0;
    g_vk.interop->GetDeviceExtensions(&count, nullptr);
    std::vector<const char*> extensions(count);
    g_vk.interop->GetDeviceExtensions(&count, extensions.data());
    bool exportable = false;
    for (const char* name : extensions)
        exportable |= std::strcmp(name, "VK_KHR_external_memory_win32") == 0;
    HMODULE vulkan = GetModuleHandleA("vulkan-1.dll");
    if (vulkan == nullptr)
        vulkan = LoadLibraryA("vulkan-1.dll");
    PFN_vkGetInstanceProcAddr instanceProc = nullptr;
    if (vulkan != nullptr)
        load_export(vulkan, "vkGetInstanceProcAddr", instanceProc);
    load_export(g.cuda, "d4rImportVulkanMemory", g_vk.import);
    load_export(g.cuda, "d4rReleaseVulkanMemory", g_vk.release);
    if (!exportable || instanceProc == nullptr || g_vk.import == nullptr || g_vk.release == nullptr)
    {
        logf("VRAM interop: unavailable (external_memory_win32 %d, vulkan-1 %p, bridge exports %d)", exportable,
             reinterpret_cast<void*>(instanceProc), g_vk.import != nullptr && g_vk.release != nullptr);
        return;
    }
    auto deviceProc = reinterpret_cast<PFN_vkGetDeviceProcAddr>(instanceProc(instance, "vkGetDeviceProcAddr"));
    auto memoryProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceMemoryProperties>(
        instanceProc(instance, "vkGetPhysicalDeviceMemoryProperties"));
    if (deviceProc == nullptr || memoryProperties == nullptr)
        return;
    memoryProperties(physical, &g_vk.memory);
    g_vk.physical = physical;
    g_vk.formatProperties = reinterpret_cast<PFN_vkGetPhysicalDeviceFormatProperties>(
        instanceProc(instance, "vkGetPhysicalDeviceFormatProperties"));
    bool ok = g_vk.formatProperties != nullptr;
    auto load = [&](auto& function, const char* name) {
        function = reinterpret_cast<std::remove_reference_t<decltype(function)>>(deviceProc(g_vk.device, name));
        ok &= function != nullptr;
    };
    load(g_vk.createBuffer, "vkCreateBuffer");
    load(g_vk.destroyBuffer, "vkDestroyBuffer");
    load(g_vk.bufferRequirements, "vkGetBufferMemoryRequirements");
    load(g_vk.bindBuffer, "vkBindBufferMemory");
    load(g_vk.createImage, "vkCreateImage");
    load(g_vk.destroyImage, "vkDestroyImage");
    load(g_vk.imageRequirements, "vkGetImageMemoryRequirements");
    load(g_vk.bindImage, "vkBindImageMemory");
    load(g_vk.allocate, "vkAllocateMemory");
    load(g_vk.free, "vkFreeMemory");
    load(g_vk.barrier, "vkCmdPipelineBarrier");
    load(g_vk.copyImageToBuffer, "vkCmdCopyImageToBuffer");
    load(g_vk.copyBufferToImage, "vkCmdCopyBufferToImage");
    load(g_vk.blit, "vkCmdBlitImage");
    g_vk.fill = reinterpret_cast<PFN_vkCmdFillBuffer>(deviceProc(g_vk.device, "vkCmdFillBuffer"));
    load(g_vk.createSemaphore, "vkCreateSemaphore");
    load(g_vk.destroySemaphore, "vkDestroySemaphore");
    load(g_vk.signalSemaphore, "vkSignalSemaphore");
    if (FAILED(g.device->QueryInterface(__uuidof(ID3D12DXVKInteropDeviceD4R), reinterpret_cast<void**>(&g_vk.split))))
        g_vk.split = nullptr;
    g_vk.ready = ok;
    logf("VRAM interop: %s (VkDevice %p), split frames %s", ok ? "ready" : "missing Vulkan entry points",
         static_cast<void*>(g_vk.device), g_vk.split != nullptr ? "available" : "unavailable (stock vkd3d-proton)");
}

static bool vram_interop_available()
{
    if (!vram_interop_requested())
        return false;
    std::call_once(g_vk.once, init_vram_interop);
    return g_vk.ready;
}

// Device-local memory; prefers memory the CPU cannot map (no BAR window).
static int device_local_type(uint32_t typeBits)
{
    for (int pass = 0; pass < 2; ++pass)
        for (uint32_t index = 0; index < g_vk.memory.memoryTypeCount; ++index)
        {
            const VkMemoryPropertyFlags flags = g_vk.memory.memoryTypes[index].propertyFlags;
            if ((typeBits & (1u << index)) && (flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) &&
                (pass == 1 || !(flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)))
                return static_cast<int>(index);
        }
    return -1;
}

// Game thread. The CUDA import runs on the worker, which owns the context.
static bool create_vram_buffer(VramBuffer& target, size_t bytes)
{
    VramBuffer buffer;
    VkExternalMemoryBufferCreateInfo external = {VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
    external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
    VkBufferCreateInfo info = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
    info.size = bytes;
    info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
    info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
    VkResult result = g_vk.createBuffer(g_vk.device, &info, nullptr, &buffer.buffer);
    VkMemoryRequirements requirements = {};
    if (result == VK_SUCCESS)
        g_vk.bufferRequirements(g_vk.device, buffer.buffer, &requirements);
    const int type = device_local_type(requirements.memoryTypeBits);
    VkMemoryDedicatedAllocateInfo dedicated = {VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
    dedicated.buffer = buffer.buffer;
    VkExportMemoryAllocateInfo exportInfo = {VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, &dedicated};
    exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT;
    VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo};
    allocation.allocationSize = requirements.size;
    allocation.memoryTypeIndex = static_cast<uint32_t>(type);
    if (result == VK_SUCCESS && type < 0)
        result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    if (result == VK_SUCCESS)
        result = g_vk.allocate(g_vk.device, &allocation, nullptr, &buffer.memory);
    if (result == VK_SUCCESS)
        result = g_vk.bindBuffer(g_vk.device, buffer.buffer, buffer.memory, 0);
    int imported = -1;
    if (result == VK_SUCCESS)
        imported = g.worker.call([&] {
            return g_vk.import(g_vk.device, reinterpret_cast<uint64_t>(buffer.memory), requirements.size,
                               &buffer.device, &buffer.external);
        });
    if (result != VK_SUCCESS || imported != 0)
    {
        logf("VRAM interop: exportable buffer of %zu bytes failed (VkResult %d, import %d)", bytes, result, imported);
        if (buffer.memory != VK_NULL_HANDLE)
            g_vk.free(g_vk.device, buffer.memory, nullptr);
        if (buffer.buffer != VK_NULL_HANDLE)
            g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
        return false;
    }
    buffer.bytes = bytes;
    target = buffer;
    return true;
}

// Only once no command list or CUDA work uses the buffer.
static void destroy_vram_buffer(VramBuffer& buffer)
{
    if (buffer.external != nullptr)
        g.worker.call([&buffer] {
            g.cu.memFree(buffer.device);
            return g_vk.release(buffer.external);
        });
    if (buffer.memory != VK_NULL_HANDLE)
        g_vk.free(g_vk.device, buffer.memory, nullptr);
    if (buffer.buffer != VK_NULL_HANDLE)
        g_vk.destroyBuffer(g_vk.device, buffer.buffer, nullptr);
    buffer = VramBuffer{};
}

static void destroy_vram_image(VramImage& image)
{
    if (image.memory != VK_NULL_HANDLE)
        g_vk.free(g_vk.device, image.memory, nullptr);
    if (image.image != VK_NULL_HANDLE)
        g_vk.destroyImage(g_vk.device, image.image, nullptr);
    image = VramImage{};
}

static bool create_vram_image(VramImage& image, UINT width, UINT height, VkFormat format)
{
    VkImageCreateInfo info = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
    info.imageType = VK_IMAGE_TYPE_2D;
    info.format = format;
    info.extent = {width, height, 1};
    info.mipLevels = 1;
    info.arrayLayers = 1;
    info.samples = VK_SAMPLE_COUNT_1_BIT;
    info.tiling = VK_IMAGE_TILING_OPTIMAL;
    info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
    info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
    info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
    VkResult result = g_vk.createImage(g_vk.device, &info, nullptr, &image.image);
    VkMemoryRequirements requirements = {};
    if (result == VK_SUCCESS)
        g_vk.imageRequirements(g_vk.device, image.image, &requirements);
    const int type = device_local_type(requirements.memoryTypeBits);
    VkMemoryAllocateInfo allocation = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
    allocation.allocationSize = requirements.size;
    allocation.memoryTypeIndex = static_cast<uint32_t>(type);
    if (result == VK_SUCCESS && type < 0)
        result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
    if (result == VK_SUCCESS)
        result = g_vk.allocate(g_vk.device, &allocation, nullptr, &image.memory);
    if (result == VK_SUCCESS)
        result = g_vk.bindImage(g_vk.device, image.image, image.memory, 0);
    if (result != VK_SUCCESS)
    {
        logf("VRAM interop: %ux%u conversion image failed: %d", width, height, result);
        destroy_vram_image(image);
        return false;
    }
    image.width = width;
    image.height = height;
    image.format = format;
    return true;
}

static size_t canonical_texel_bytes(Plane plane)
{
    return (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
}

static size_t vk_texel_bytes(VkFormat format)
{
    switch (format)
    {
    case VK_FORMAT_R16G16B16A16_SFLOAT:
    case VK_FORMAT_R16G16B16A16_UINT:
    case VK_FORMAT_R16G16B16A16_SINT:
    case VK_FORMAT_R16G16B16A16_UNORM:
        return 8;
    case VK_FORMAT_R32_SFLOAT:
    case VK_FORMAT_R32_UINT:
    case VK_FORMAT_R32_SINT:
    case VK_FORMAT_R16G16_SFLOAT:
    case VK_FORMAT_R16G16_UINT:
    case VK_FORMAT_R16G16_SINT:
    case VK_FORMAT_D32_SFLOAT:
    case VK_FORMAT_D32_SFLOAT_S8_UINT: // the depth aspect alone
        return 4;
    default:
        return 0;
    }
}

// How one input or the output moves between its D3D12 texture and a buffer.
struct VramCopy
{
    ID3D12Resource* resource = nullptr;
    VkImage image = VK_NULL_HANDLE;
    VkImageAspectFlags aspect = VK_IMAGE_ASPECT_COLOR_BIT;
    bool convert = false; // blit to/from the plane's canonical format
    UINT width = 0, height = 0;
};

static bool vram_blit_supported(VkFormat source, VkFormat destination)
{
    VkFormatProperties sourceProperties = {}, destinationProperties = {};
    g_vk.formatProperties(g_vk.physical, source, &sourceProperties);
    g_vk.formatProperties(g_vk.physical, destination, &destinationProperties);
    return (sourceProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) != 0 &&
           (destinationProperties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_DST_BIT) != 0;
}

// sRGB is deliberately excluded: the host staging path treats its bytes as
// UNORM and a Vulkan blit would apply an sRGB transfer function instead.
static bool vram_color_blit_supported(VkFormat format, bool output)
{
    switch (format)
    {
    case VK_FORMAT_B10G11R11_UFLOAT_PACK32:
    case VK_FORMAT_A2B10G10R10_UNORM_PACK32:
    case VK_FORMAT_R8G8B8A8_UNORM:
    case VK_FORMAT_B8G8R8A8_UNORM:
        break;
    default:
        return false;
    }
    return output ? vram_blit_supported(VK_FORMAT_R16G16B16A16_SFLOAT, format)
                  : vram_blit_supported(format, VK_FORMAT_R16G16B16A16_SFLOAT);
}

// The host path takes only the first (R) component of an exposure texel.
static bool vram_exposure_blit_supported(VkFormat format)
{
    switch (format)
    {
    case VK_FORMAT_R16_SFLOAT:
    case VK_FORMAT_R16G16B16A16_SFLOAT:
    case VK_FORMAT_R32G32B32A32_SFLOAT:
        return vram_blit_supported(format, VK_FORMAT_R32_SFLOAT);
    default:
        return false;
    }
}

// Whether `resource` can be copied raw (or, for colour/exposure, blitted)
// to/from the canonical layout of `plane`.
static bool describe_vram_copy(ID3D12Resource* resource, Plane plane, VramCopy& copy, bool output = false)
{
    UINT64 handle = 0, offset = 0;
    VkFormat format = VK_FORMAT_UNDEFINED;
    if (FAILED(g_vk.interop->GetVulkanResourceInfo1(resource, &handle, &offset, &format)) || handle == 0)
        return false;
    D3D12_RESOURCE_DESC desc;
    resource->GetDesc(&desc);
    if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1)
        return false;
    copy.resource = resource;
    copy.image = reinterpret_cast<VkImage>(handle);
    copy.width = static_cast<UINT>(desc.Width);
    copy.height = desc.Height;
    copy.aspect = (format == VK_FORMAT_D32_SFLOAT || format == VK_FORMAT_D32_SFLOAT_S8_UINT)
                      ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT;
    if (plane == Plane::Color)
    {
        copy.convert = vram_color_blit_supported(format, output);
        return copy.convert || format == VK_FORMAT_R16G16B16A16_SFLOAT;
    }
    if (plane == Plane::Exposure)
        copy.convert = vram_exposure_blit_supported(format);
    return copy.convert || vk_texel_bytes(format) == canonical_texel_bytes(plane);
}

static const VkMemoryBarrier kBeforeTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_MEMORY_WRITE_BIT,
                                                VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT};
static const VkMemoryBarrier kAfterTransfer = {VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT,
                                               VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT};

static VkBufferImageCopy full_region(const VramCopy& copy)
{
    VkBufferImageCopy region = {};
    region.imageSubresource.aspectMask = copy.aspect;
    region.imageSubresource.layerCount = 1;
    region.imageExtent = {copy.width, copy.height, 1};
    return region;
}

// Records the input copies into the slot's buffers. The resources are in
// COPY_SOURCE state already.
static bool record_vram_inputs(Feature& feature, ID3D12GraphicsCommandList* list, InputSlot& slot,
                               const VramCopy* copies, int count, uint32_t frame)
{
    VkCommandBuffer cmd = VK_NULL_HANDLE;
    if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
        return false;
    g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
                 nullptr, 0, nullptr);
    for (int index = 0; index < count; ++index)
    {
        const VramCopy& copy = copies[index];
        VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
        g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_SOURCE, &layout);
        VkBufferImageCopy region = full_region(copy);
        static const Plane kPlanes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
        if (feature.linearInputs)
            region.bufferRowLength = static_cast<uint32_t>(slot.host[index].rowBytes / canonical_texel_bytes(kPlanes[index]));
        if (!copy.convert)
        {
            g_vk.copyImageToBuffer(cmd, copy.image, layout, slot.vram[index].buffer, 1, &region);
            continue;
        }
        // Convert colour to RGBA16F or exposure's first component to R32F,
        // then copy from the canonical image into the shared buffer.
        VkImage conversion = kPlanes[index] == Plane::Exposure ? feature.exposureConversion.image
                                                               : feature.colorConversion.image;
        VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
        toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
        toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
        toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
        toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
        toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        toDestination.image = conversion;
        toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
        g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
                     &toDestination);
        VkImageBlit blit = {};
        blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
        blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
        blit.dstSubresource = blit.srcSubresource;
        blit.dstOffsets[1] = blit.srcOffsets[1];
        g_vk.blit(cmd, copy.image, layout, conversion, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit,
                  VK_FILTER_NEAREST);
        VkImageMemoryBarrier toSource = toDestination;
        toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
        toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
        toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
        toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
        g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
                     &toSource);
        g_vk.copyImageToBuffer(cmd, conversion, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, slot.vram[index].buffer, 1,
                               &region);
    }
    g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
                 nullptr, 0, nullptr);
    // After the barrier: the copies are complete and their writes available when the value lands.
    if (feature.gpuWait)
        g_vk.fill(cmd, feature.gpuMarker.buffer, 0, 4, frame);
    return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
}

// Records the copy of a finished result into the output texture, which is in
// COPY_DEST state already.
static bool record_vram_output(ID3D12GraphicsCommandList* list, const VramBuffer& buffer, const VramCopy& copy,
                               const VramImage& conversion)
{
    VkCommandBuffer cmd = VK_NULL_HANDLE;
    if (FAILED(g_vk.interop->BeginVkCommandBufferInterop(list, &cmd)))
        return false;
    VkImageLayout layout = VK_IMAGE_LAYOUT_GENERAL;
    g_vk.interop->GetVulkanImageLayout(copy.resource, D3D12_RESOURCE_STATE_COPY_DEST, &layout);
    const VkBufferImageCopy region = full_region(copy);
    g_vk.barrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &kBeforeTransfer, 0,
                 nullptr, 0, nullptr);
    if (!copy.convert)
        g_vk.copyBufferToImage(cmd, buffer.buffer, copy.image, layout, 1, &region);
    else
    {
        VkImageMemoryBarrier toDestination = {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER};
        toDestination.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
        toDestination.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
        toDestination.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
        toDestination.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
        toDestination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        toDestination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
        toDestination.image = conversion.image;
        toDestination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
        g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
                     &toDestination);
        g_vk.copyBufferToImage(cmd, buffer.buffer, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &region);
        VkImageMemoryBarrier toSource = toDestination;
        toSource.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT;
        toSource.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
        toSource.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
        toSource.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
        g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1,
                     &toSource);
        VkImageBlit blit = {};
        blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1};
        blit.srcOffsets[1] = {static_cast<int32_t>(copy.width), static_cast<int32_t>(copy.height), 1};
        blit.dstSubresource = blit.srcSubresource;
        blit.dstOffsets[1] = blit.srcOffsets[1];
        g_vk.blit(cmd, conversion.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, copy.image, layout, 1, &blit,
                  VK_FILTER_NEAREST);
    }
    g_vk.barrier(cmd, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &kAfterTransfer, 0,
                 nullptr, 0, nullptr);
    return SUCCEEDED(g_vk.interop->EndVkCommandBufferInterop(list));
}

// --- Split frames -------------------------------------------------------------------
//
// To present frame N's own result, everything the game records after its DLSS
// call must wait for DLSS. With the d4r vkd3d-proton patch, evaluate() splits
// the game's command list right after the input copies and the frame marker;
// vkd3d-proton submits the second half separately, waiting (at queue submission
// level, so no GPU ring is ever blocked) for splitSemaphore to reach N. The
// worker writes N's result into output slot N % kOutputSlots and signals N when
// the frame retires; a dropped frame signals too, and a watchdog signals any
// frame the GPU has reached but nobody released within the timeout.

static void signal_split(Feature* feature, uint64_t value, const char* reason)
{
    std::lock_guard<std::mutex> lock(feature->splitMutex);
    if (value <= feature->splitSignalled)
        return;
    VkSemaphoreSignalInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO};
    info.semaphore = feature->splitSemaphore;
    info.value = value;
    const VkResult result = g_vk.signalSemaphore(g_vk.device, &info);
    feature->splitSignalled = value;
    if (result != VK_SUCCESS || reason != nullptr)
        logf("split frame %llu released%s%s (VkResult %d)", static_cast<unsigned long long>(value),
             reason != nullptr ? " by " : "", reason != nullptr ? reason : "", result);
}

static void release_split_frame(Feature* feature, uint32_t frame)
{
    signal_split(feature, frame, nullptr);
}

static void split_watchdog()
{
    const auto timeout = std::chrono::milliseconds(env_uint("D4R_SHIM_SPLIT_TIMEOUT_MS", 200));
    struct Seen
    {
        uint32_t marker = 0;
        std::chrono::steady_clock::time_point at;
    };
    std::vector<std::pair<Feature*, Seen>> seen;
    for (;;)
    {
        std::this_thread::sleep_for(std::chrono::milliseconds(5));
        std::lock_guard<std::mutex> lock(g_featuresMutex);
        const auto now = std::chrono::steady_clock::now();
        for (Feature* feature : g_features)
        {
            if (!feature->split || feature->markerValue == nullptr)
                continue;
            auto it = std::find_if(seen.begin(), seen.end(), [feature](const auto& entry) { return entry.first == feature; });
            if (it == seen.end())
                it = seen.insert(seen.end(), {feature, Seen{}});
            const uint32_t marker = *feature->markerValue;
            uint64_t signalled;
            {
                std::lock_guard<std::mutex> splitLock(feature->splitMutex);
                signalled = feature->splitSignalled;
            }
            // GPU-side input wait for a frame the game's queue has not reached for over a second
            const uint32_t waiting = feature->gpuWaitFrame.load();
            if (feature->gpuWait && waiting != 0 && static_cast<int32_t>(marker - waiting) < 0 &&
                now.time_since_epoch().count() - feature->gpuWaitSince.load() >
                    std::chrono::duration_cast<std::chrono::steady_clock::duration>(std::chrono::seconds(1)).count() &&
                [&] { uint32_t expected = waiting; return feature->gpuWaitFrame.compare_exchange_strong(expected, 0u); }())
            {
                logf("frame %u: its inputs never reached the GPU (marker %u); releasing the GPU-side wait", waiting, marker);
                g.cu.writeValue32(feature->gpuMarker.device, waiting);
            }
            if (marker <= signalled)
                continue; // nothing on the GPU is waiting on us
            if (it->second.marker != marker)
                it->second = Seen{marker, now};
            else if (now - it->second.at > timeout)
                signal_split(feature, marker, "the watchdog");
        }
        // Forget released features.
        seen.erase(std::remove_if(seen.begin(), seen.end(),
                                  [](const auto& entry) {
                                      return std::find(g_features.begin(), g_features.end(), entry.first) ==
                                             g_features.end();
                                  }),
                   seen.end());
    }
}

static bool create_split_semaphore(Feature& feature)
{
    VkSemaphoreTypeCreateInfo type = {VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
    type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
    type.initialValue = 0;
    const VkSemaphoreCreateInfo info = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type};
    if (g_vk.createSemaphore(g_vk.device, &info, nullptr, &feature.splitSemaphore) != VK_SUCCESS)
    {
        logf("split frames: vkCreateSemaphore failed");
        return false;
    }
    static std::once_flag watchdog;
    std::call_once(watchdog, [] { std::thread(split_watchdog).detach(); });
    return true;
}

// Game thread: grows a buffer; an outgrown one is kept until the feature is
// released since queued command lists may still reference it.
static bool ensure_vram_buffer(Feature& feature, VramBuffer& buffer, size_t bytes)
{
    if (buffer.buffer != VK_NULL_HANDLE && buffer.bytes >= bytes)
        return true;
    if (buffer.buffer != VK_NULL_HANDLE)
        feature.retiredBuffers.push_back(buffer);
    buffer = VramBuffer{};
    return create_vram_buffer(buffer, bytes);
}

static bool ensure_conversion_image(Feature& feature, VramImage& image, UINT width, UINT height, VkFormat format)
{
    if (image.image != VK_NULL_HANDLE && image.width == width && image.height == height && image.format == format)
        return true;
    if (image.image != VK_NULL_HANDLE)
        feature.retiredImages.push_back(image);
    image = VramImage{};
    return create_vram_image(image, width, height, format);
}

static void release_vram(Feature& feature)
{
    if (feature.splitSemaphore != VK_NULL_HANDLE)
        g_vk.destroySemaphore(g_vk.device, feature.splitSemaphore, nullptr);
    for (InputSlot& slot : feature.inputs)
        for (VramBuffer& buffer : slot.vram)
            destroy_vram_buffer(buffer);
    for (OutputSlot& slot : feature.outputs)
        destroy_vram_buffer(slot.vram);
    destroy_vram_buffer(feature.gpuMarker);
    for (VramBuffer& buffer : feature.retiredBuffers)
        destroy_vram_buffer(buffer);
    feature.retiredBuffers.clear();
    destroy_vram_image(feature.colorConversion);
    destroy_vram_image(feature.exposureConversion);
    destroy_vram_image(feature.outputConversion);
    for (VramImage& image : feature.retiredImages)
        destroy_vram_image(image);
    feature.retiredImages.clear();
}

// Prep stage: converts one readback staging buffer into the canonical layout.
static void stage_plane(Plane plane, const Staging& staging, HostPlane& host, FrameTiming* timing, int timingIndex)
{
    const size_t texelBytes = (plane_format(plane) == CUDA_FORMAT_HALF ? 2 : 4) * plane_channels(plane);
    reserve_host(host, texelBytes * staging.width * staging.height);
    host.width = staging.width;
    host.height = staging.height;
    host.rowBytes = texelBytes * staging.width;
    uint8_t* rows = host.bytes;
    const size_t rowBytes = host.rowBytes;
    const uint8_t* source = staging.mapped + staging.layout.Offset;
    const size_t sourcePitch = staging.layout.Footprint.RowPitch;
    const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
    if (canonical_input(plane, staging.format))
        copy_rows(g_inputRows, rows, rowBytes, source, sourcePitch, rowBytes, staging.height);
    else
        g_inputRows.run(staging.height, rowBytes * staging.height, [&](UINT begin, UINT end) {
            for (UINT y = begin; y < end; ++y)
                if (plane == Plane::Color && staging.format == DXGI_FORMAT_R11G11B10_FLOAT)
                    convert_r11g11b10_row(source + y * sourcePitch,
                                          reinterpret_cast<uint16_t*>(rows + rowBytes * y), staging.width);
                else
                    convert_row_in(plane, staging.format, source + y * sourcePitch, rows + rowBytes * y,
                                   staging.width);
        });
    if (timing != nullptr)
        timing->convert[timingIndex] = profile_ms(conversionStart, ProfileClock::now());
}

// Worker stage: uploads one staged plane into its CUDA array.
// D4R_SHIM_MV_STATS=N logs, every N frames, what the game's motion vectors
// look like (staged RG16F plane): how much of the texture is populated, the
// populated bounding box (a render-size sub-rectangle means render-resolution
// vectors in a display-size texture), magnitudes in texture units, and the
// global image shift between the previous and current colour frames measured
// in render pixels, so the vectors' unit can be read off their ratio.
static std::vector<float> downsampled_luma(const HostPlane& color, UINT& width, UINT& height)
{
    // RGBA16F colour, 2x2 box-downsampled luma.
    width = color.width / 2, height = color.height / 2;
    std::vector<float> luma(static_cast<size_t>(width) * height);
    const uint8_t* bytes = color.bytes != nullptr ? color.bytes : color.fallback.data();
    for (UINT y = 0; y < height; ++y)
        for (UINT x = 0; x < width; ++x)
        {
            float sum = 0.0f;
            for (UINT dy = 0; dy < 2; ++dy)
            {
                const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + (2 * y + dy) * color.rowBytes);
                for (UINT dx = 0; dx < 2; ++dx)
                    for (UINT c = 0; c < 3; ++c)
                        sum += static_cast<float>(std::bit_cast<_Float16>(row[(2 * x + dx) * 4 + c]));
            }
            luma[static_cast<size_t>(y) * width + x] = sum / (sum + 12.0f); // tone-mapped (HDR input)
        }
    return luma;
}

static void log_motion_stats(const HostPlane& host, const HostPlane& color, uint32_t frame)
{
    static const uint32_t every = env_uint("D4R_SHIM_MV_STATS", 0);
    static std::vector<float> previousLuma;
    static uint32_t previousFrame = 0;
    if (every == 0 || host.width == 0 || color.width == 0)
        return;
    const bool report = frame % every == 0;
    if (!report && (frame + 1) % every != 0)
        return;
    UINT lumaWidth = 0, lumaHeight = 0;
    std::vector<float> luma = downsampled_luma(color, lumaWidth, lumaHeight);
    if (!report)
    {
        previousLuma = std::move(luma);
        previousFrame = frame;
        return;
    }
    const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
    size_t nonzero = 0;
    double sumMagnitude = 0.0;
    float maxMagnitude = 0.0f;
    UINT minX = host.width, minY = host.height, maxX = 0, maxY = 0;
    std::vector<float> xs, ys;
    for (UINT y = 0; y < host.height; ++y)
    {
        const uint16_t* row = reinterpret_cast<const uint16_t*>(bytes + y * host.rowBytes);
        for (UINT x = 0; x < host.width; ++x)
        {
            const float mx = static_cast<float>(std::bit_cast<_Float16>(row[x * 2]));
            const float my = static_cast<float>(std::bit_cast<_Float16>(row[x * 2 + 1]));
            if ((x % 8) == 0 && (y % 8) == 0)
            {
                xs.push_back(mx);
                ys.push_back(my);
            }
            if (mx == 0.0f && my == 0.0f)
                continue;
            ++nonzero;
            const float magnitude = std::sqrt(mx * mx + my * my);
            sumMagnitude += magnitude;
            maxMagnitude = std::max(maxMagnitude, magnitude);
            minX = std::min(minX, x), maxX = std::max(maxX, x);
            minY = std::min(minY, y), maxY = std::max(maxY, y);
        }
    }
    auto median = [](std::vector<float>& values) {
        if (values.empty())
            return 0.0f;
        std::nth_element(values.begin(), values.begin() + values.size() / 2, values.end());
        return values[values.size() / 2];
    };
    const float medianX = median(xs), medianY = median(ys);
    // Global shift d (render pixels, current -> previous, the vectors'
    // convention) minimising |cur(p) - prev(p + d)| over the central region.
    int bestX = 0, bestY = 0;
    double bestCost = -1.0, zeroCost = 0.0;
    if (previousFrame + 1 == frame && previousLuma.size() == luma.size())
    {
        const int range = 80, rangeY = 40; // in downsampled pixels
        const int x0 = range, x1 = static_cast<int>(lumaWidth) - range;
        const int y0 = rangeY, y1 = static_cast<int>(lumaHeight) - rangeY;
        for (int dy = -rangeY; dy <= rangeY; ++dy)
            for (int dx = -range; dx <= range; ++dx)
            {
                double cost = 0.0;
                for (int y = y0; y < y1; y += 2)
                    for (int x = x0; x < x1; x += 2)
                        cost += std::fabs(luma[static_cast<size_t>(y) * lumaWidth + x] -
                                          previousLuma[static_cast<size_t>(y + dy) * lumaWidth + (x + dx)]);
                if (dx == 0 && dy == 0)
                    zeroCost = cost;
                if (bestCost < 0.0 || cost < bestCost)
                    bestCost = cost, bestX = dx, bestY = dy;
            }
    }
    const double pixels = static_cast<double>(host.width) * host.height;
    logf("MV_STATS frame=%u size=%ux%u nonzero=%.4f bbox=%u,%u-%u,%u median=(%.3f,%.3f) mean_abs=%.3f max=%.3f "
         "image_shift_render_px=(%d,%d) match=%.3f",
         frame, host.width, host.height, nonzero / pixels, nonzero ? minX : 0, nonzero ? minY : 0, maxX, maxY,
         medianX, medianY, nonzero ? sumMagnitude / nonzero : 0.0, maxMagnitude, 2 * bestX, 2 * bestY,
         zeroCost > 0.0 ? bestCost / zeroCost : 1.0);
}

static bool upload_plane(Plane plane, const HostPlane& host, CudaImage& image, uint32_t frame, FrameTiming* timing,
                         int timingIndex)
{
    if (!ensure_cuda_image(image, host.width, host.height, plane_format(plane), plane_channels(plane), false,
                           plane == Plane::Color))
        return false;
    if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
    {
        uint64_t hash = 1469598103934665603ull;
        for (size_t index = 0; index < host.size(); ++index)
            hash = (hash ^ host.bytes[index]) * 1099511628211ull;
        logf("frame %u upload plane=%u array=%p object=0x%llx bytes=%zu hash=%016llx", frame,
             static_cast<unsigned int>(plane), image.array,
             static_cast<unsigned long long>(image.object), host.size(),
             static_cast<unsigned long long>(hash));
    }
    CudaMemcpy2D copy = {};
    copy.srcMemoryType = CUDA_MEMORY_HOST;
    copy.srcHost = host.bytes;
    copy.srcPitch = host.rowBytes;
    copy.dstMemoryType = CUDA_MEMORY_ARRAY;
    copy.dstArray = image.array;
    copy.WidthInBytes = host.rowBytes;
    copy.Height = host.height;
    const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
    const int result = g.cu.memcpy2D(&copy);
    if (timing != nullptr)
        timing->upload[timingIndex] = profile_ms(copyStart, ProfileClock::now());
    if (result != 0)
        logf("cuMemcpy2D upload failed: %d", result);
    if (result != 0)
        return false;
    if (GetEnvironmentVariableA("D4R_SHIM_SYNC_UPLOAD", nullptr, 0) != 0)
    {
        const int syncResult = g.cu.ctxSynchronize();
        if (syncResult != 0)
            logf("cuCtxSynchronize after upload failed: %d", syncResult);
        return syncResult == 0;
    }
    return true;
}

static void log_output_hash(const HostPlane& host, uint32_t frame)
{
    if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) == 0)
        return;
    uint64_t hash = 1469598103934665603ull;
    for (size_t index = 0; index < host.size(); ++index)
        hash = (hash ^ host.bytes[index]) * 1099511628211ull;
    logf("frame %u output hash=%016llx", frame, static_cast<unsigned long long>(hash));
}

static std::atomic<uint32_t> g_captureFirst{0}; // first frame of a triggered capture

static bool dump_frame_selected(uint32_t frame, const char* startName, const char* countName, const char* everyName)
{
    char trigger[MAX_PATH];
    const DWORD triggerLength = GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH);
    if (triggerLength > 0 && triggerLength < MAX_PATH)
    {
        uint32_t captured = g_captureFirst.load();
        if (captured == 0 && GetFileAttributesA(trigger) != INVALID_FILE_ATTRIBUTES)
        {
            uint32_t expected = 0;
            if (g_captureFirst.compare_exchange_strong(expected, frame))
                logf("raw input/output capture triggered on frame %u", frame);
            captured = g_captureFirst.load();
        }
        // D4R_SHIM_CAPTURE_COUNT=N keeps capturing N consecutive frames (replay sequences).
        static const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
        return captured != 0 && frame >= captured && frame - captured < count;
    }
    const uint32_t start = env_uint(startName, 1);
    const uint32_t count = env_uint(countName, 1);
    const uint32_t every = std::max(1u, env_uint(everyName, 1));
    return frame >= start && count != 0 && (frame - start) / every < count && (frame - start) % every == 0;
}

static bool output_dump_path(uint32_t frame, char (&path)[MAX_PATH])
{
    char directory[MAX_PATH];
    const DWORD length = GetEnvironmentVariableA("D4R_SHIM_OUTPUT_DUMP_DIR", directory, MAX_PATH);
    if (length == 0 || length >= MAX_PATH)
        return false;
    if (!dump_frame_selected(frame, "D4R_SHIM_OUTPUT_DUMP_START", "D4R_SHIM_OUTPUT_DUMP_COUNT",
                             "D4R_SHIM_OUTPUT_DUMP_EVERY"))
        return false;
    const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u.rgba16f", directory, frame);
    return written > 0 && written < MAX_PATH;
}

// Capture files. With D4R_SHIM_CAPTURE_MEMORY=1 they are kept in memory and
// written by a background thread once the triggered capture is complete, so a
// sequence can be recorded at the game's normal frame rate.
static std::mutex g_captureMutex;
static std::vector<std::pair<std::string, std::vector<uint8_t>>> g_captureFiles;

// Pinned staging for memory captures (D4R_SHIM_CAPTURE_POOL_MB, default 3072):
// device readbacks land in it directly, so a capture costs a few ms per frame.
struct CapturePool
{
    uint8_t* base = nullptr;
    size_t size = 0, used = 0;
    bool tried = false;
};
static CapturePool g_capturePool;
static std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> g_capturePinned;

// Worker thread (a CUDA context is current). Returns nullptr when memory
// captures are off or the pool is exhausted; the caller then uses ordinary memory.
static uint8_t* capture_pinned(const char* path, size_t bytes)
{
    if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 || g.cu.memAllocHost == nullptr)
        return nullptr;
    std::lock_guard<std::mutex> lock(g_captureMutex);
    CapturePool& pool = g_capturePool;
    if (!pool.tried)
    {
        pool.tried = true;
        for (size_t mb = env_uint("D4R_SHIM_CAPTURE_POOL_MB", 3072); mb >= 256 && pool.base == nullptr; mb /= 2)
        {
            void* memory = nullptr;
            if (g.cu.memAllocHost(&memory, mb << 20) == 0)
                pool.base = static_cast<uint8_t*>(memory), pool.size = mb << 20;
        }
        logf("capture pool: %zu MB pinned", pool.size >> 20);
    }
    const size_t aligned = (bytes + 4095) & ~size_t(4095);
    if (pool.base == nullptr || pool.used + aligned > pool.size)
        return nullptr;
    uint8_t* block = pool.base + pool.used;
    pool.used += aligned;
    g_capturePinned.emplace_back(path, std::make_pair(block, bytes));
    return block;
}

static void write_capture_file(uint32_t frame, const char* what, const char* path, const void* data, size_t bytes)
{
    static const bool memory = env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) != 0;
    if (memory)
    {
        if (uint8_t* pinned = capture_pinned(path, bytes))
        {
            std::memcpy(pinned, data, bytes);
            return;
        }
        const auto* first = static_cast<const uint8_t*>(data);
        std::lock_guard<std::mutex> lock(g_captureMutex);
        g_captureFiles.emplace_back(path, std::vector<uint8_t>(first, first + bytes));
        return;
    }
    FILE* file = std::fopen(path, "wb");
    if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
        logf("frame %u %s dump failed: %s", frame, what, path);
    else
        logf("frame %u %s saved: %s", frame, what, path);
    if (file != nullptr)
        std::fclose(file);
}

// Called after each frame's output dump: once the last triggered frame is in,
// the buffered files go to disk.
static void flush_capture_if_complete(uint32_t frame)
{
    char trigger[MAX_PATH];
    if (env_uint("D4R_SHIM_CAPTURE_MEMORY", 0) == 0 ||
        GetEnvironmentVariableA("D4R_SHIM_CAPTURE_TRIGGER", trigger, MAX_PATH) == 0)
        return;
    const uint32_t count = std::max(1u, env_uint("D4R_SHIM_CAPTURE_COUNT", 1));
    const uint32_t first = g_captureFirst.load();
    if (first == 0 || frame != first + count - 1)
        return;
    std::vector<std::pair<std::string, std::vector<uint8_t>>> files;
    std::vector<std::pair<std::string, std::pair<const uint8_t*, size_t>>> pinned;
    {
        std::lock_guard<std::mutex> lock(g_captureMutex);
        files.swap(g_captureFiles);
        pinned.swap(g_capturePinned);
    }
    logf("capture of frames %u-%u complete in memory; writing %zu files", first, frame, files.size() + pinned.size());
    // The pinned pool is not reused, so its blocks stay valid for the writer.
    std::thread([files = std::move(files), pinned = std::move(pinned), first, frame] {
        size_t failed = 0;
        auto write = [&failed](const std::string& path, const uint8_t* data, size_t bytes) {
            FILE* file = std::fopen(path.c_str(), "wb");
            if (file == nullptr || std::fwrite(data, 1, bytes, file) != bytes)
                ++failed;
            if (file != nullptr)
                std::fclose(file);
        };
        for (const auto& [path, block] : pinned)
            write(path, block.first, block.second);
        for (const auto& [path, bytes] : files)
            write(path, bytes.data(), bytes.size());
        logf("capture of frames %u-%u written (%zu files, %zu failed)", first, frame, files.size() + pinned.size(),
             failed);
    }).detach();
}

static void dump_output(const HostPlane& host, uint32_t frame, const char* path)
{
    write_capture_file(frame, "raw DLSS output", path, host.bytes, host.size());
    flush_capture_if_complete(frame);
}

static bool input_dump_directory(uint32_t frame, char (&directory)[MAX_PATH])
{
    const DWORD length = GetEnvironmentVariableA("D4R_SHIM_INPUT_DUMP_DIR", directory, MAX_PATH);
    return length != 0 && length < MAX_PATH &&
           dump_frame_selected(frame, "D4R_SHIM_INPUT_DUMP_START", "D4R_SHIM_INPUT_DUMP_COUNT",
                               "D4R_SHIM_INPUT_DUMP_EVERY");
}

static void dump_input_planes(const HostPlane (&planes)[4], uint32_t frame, const FrameParams& p,
                              const char* directory)
{
    constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
    for (int index = 0; index < (planes[3].width != 0 ? 4 : 3); ++index)
    {
        const HostPlane& host = planes[index];
        const uint8_t* bytes = host.bytes != nullptr ? host.bytes : host.fallback.data();
        if (bytes == nullptr || host.width == 0 || host.height == 0)
            continue;
        char path[MAX_PATH];
        const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
        if (written > 0 && written < MAX_PATH)
            write_capture_file(frame, names[index], path, bytes, host.size());
    }
    // The evaluation parameters, so the harness can replay the sequence.
    char path[MAX_PATH];
    const int written = std::snprintf(path, MAX_PATH, "%s\\frame-%06u-params.txt", directory, frame);
    if (written <= 0 || written >= MAX_PATH)
        return;
    char text[1024];
    const int length = std::snprintf(
        text, sizeof(text),
        "jitter %.9g %.9g\nmv_scale %.9g %.9g\nsharpness %.9g\npre_exposure %.9g\n"
        "exposure_scale %.9g\nframe_time %.9g\nreset %d\ninvert %d %d\nrender %u %u\n"
        "color_base %u %u\ndepth_base %u %u\nmv_base %u %u\noutput_base %u %u\nhas_exposure %d\n",
        p.jitterX, p.jitterY, p.mvScaleX, p.mvScaleY, p.sharpness, p.preExposure, p.exposureScale, p.frameTime,
        p.reset, p.invertX, p.invertY, p.renderWidth, p.renderHeight, p.colorBaseX, p.colorBaseY, p.depthBaseX,
        p.depthBaseY, p.mvBaseX, p.mvBaseY, p.outputBaseX, p.outputBaseY, p.hasExposure ? 1 : 0);
    if (length > 0)
        write_capture_file(frame, "params", path, text, static_cast<size_t>(length));
}

// VRAM interop without D4R_SHIM_VRAM_VERIFY: reads the selected frame's input
// planes straight from the imported buffers DLSS is about to consume.
static void dump_vram_inputs(const InputSlot& slot, uint32_t frame, const FrameParams& p, int count)
{
    char directory[MAX_PATH];
    if (!input_dump_directory(frame, directory))
        return;
    HostPlane planes[4];
    constexpr const char* names[4] = {"color.rgba16f", "depth.r32f", "motion.rg16f", "exposure.r32f"};
    bool allPinned = true;
    for (int index = 0; index < count; ++index)
    {
        const HostPlane& geometry = slot.host[index];
        HostPlane& plane = planes[index];
        plane.width = geometry.width, plane.height = geometry.height, plane.rowBytes = geometry.rowBytes;
        char path[MAX_PATH];
        std::snprintf(path, MAX_PATH, "%s\\frame-%06u-%s", directory, frame, names[index]);
        plane.bytes = capture_pinned(path, geometry.size());
        if (plane.bytes == nullptr)
        {
            allPinned = false;
            plane.fallback.resize(geometry.size());
        }
        CudaMemcpy2D readback = {};
        readback.srcMemoryType = CUDA_MEMORY_DEVICE;
        readback.srcDevice = slot.vram[index].device;
        readback.srcPitch = geometry.rowBytes;
        readback.dstMemoryType = CUDA_MEMORY_HOST;
        readback.dstHost = plane.bytes != nullptr ? plane.bytes : plane.fallback.data();
        readback.dstPitch = geometry.rowBytes;
        readback.WidthInBytes = geometry.rowBytes;
        readback.Height = geometry.height;
        if (g.cu.memcpy2D(&readback) != 0)
        {
            logf("frame %u: VRAM input readback for the dump failed", frame);
            return;
        }
    }
    // Pinned planes are already registered for writing; only the rest (and the
    // parameters) go through dump_input_planes.
    for (HostPlane& plane : planes)
        if (plane.bytes != nullptr)
            plane.width = 0;
    (void)allPinned;
    dump_input_planes(planes, frame, p, directory);
}

// Worker stage: downloads the DLSS output into host memory.
static bool download_output(Feature& feature, HostPlane& host, uint32_t frame, FrameTiming* timing)
{
    reserve_host(host, static_cast<size_t>(feature.output.width) * 8 * feature.output.height);
    host.width = feature.output.width;
    host.height = feature.output.height;
    host.rowBytes = static_cast<size_t>(feature.output.width) * 8;
    CudaMemcpy2D copy = {};
    copy.srcMemoryType = CUDA_MEMORY_ARRAY;
    copy.srcArray = feature.output.array;
    copy.dstMemoryType = CUDA_MEMORY_HOST;
    copy.dstHost = host.bytes;
    copy.dstPitch = host.rowBytes;
    copy.WidthInBytes = host.rowBytes;
    copy.Height = host.height;
    const auto copyStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
    const int result = g.cu.memcpy2D(&copy);
    if (timing != nullptr)
        timing->download = profile_ms(copyStart, ProfileClock::now());
    if (result != 0)
    {
        logf("cuMemcpy2D download failed: %d", result);
        return false;
    }
    log_output_hash(host, frame);
    char dumpPath[MAX_PATH];
    if (output_dump_path(frame, dumpPath))
        dump_output(host, frame, dumpPath);
    return true;
}

// Finish stage: converts a downloaded result into an upload staging buffer.
static void write_output(Staging& staging, const HostPlane& host, FrameTiming* timing)
{
    const auto conversionStart = timing != nullptr ? ProfileClock::now() : ProfileClock::time_point{};
    uint8_t* destination = staging.mapped + staging.layout.Offset;
    const size_t destinationPitch = staging.layout.Footprint.RowPitch;
    const UINT width = staging.width < host.width ? staging.width : host.width;
    const UINT height = staging.height < host.height ? staging.height : host.height;
    const uint8_t* rows = host.bytes;
    if (staging.format == DXGI_FORMAT_R16G16B16A16_FLOAT || staging.format == DXGI_FORMAT_R16G16B16A16_TYPELESS)
        copy_rows(g_outputRows, destination, destinationPitch, rows, host.rowBytes, static_cast<size_t>(width) * 8,
                  height);
    else
        g_outputRows.run(height, host.rowBytes * height, [&](UINT begin, UINT end) {
            for (UINT y = begin; y < end; ++y)
                convert_row_out(staging.format, reinterpret_cast<const uint16_t*>(rows + host.rowBytes * y),
                                destination + y * destinationPitch, width);
        });
    if (timing != nullptr)
        timing->outputConvert = profile_ms(conversionStart, ProfileClock::now());
}

static void set_evaluation_parameters(Feature& feature, const FrameParams& p)
{
    void* params = feature.cudaParams;
    feature.colorHandle = feature.color.object;
    feature.depthHandle = feature.depth.object;
    feature.motionHandle = feature.motion.object;
    feature.exposureHandle = feature.exposure.object;
    feature.outputHandle = feature.output.object;
    // CUDA resource parameters point to variables holding the object handles.
    d4r_ngx_set_void(params, "Color", &feature.colorHandle);
    d4r_ngx_set_void(params, "Depth", &feature.depthHandle);
    d4r_ngx_set_void(params, "MotionVectors", &feature.motionHandle);
    d4r_ngx_set_void(params, "Output", &feature.outputHandle);
    d4r_ngx_set_void(params, "ExposureTexture", p.hasExposure ? &feature.exposureHandle : nullptr);
    d4r_ngx_set_void(params, "TransparencyMask", nullptr);
    d4r_ngx_set_void(params, "DLSS.Input.Bias.Current.Color.Mask", nullptr);
    d4r_ngx_set_float(params, "Jitter.Offset.X", p.jitterX);
    d4r_ngx_set_float(params, "Jitter.Offset.Y", p.jitterY);
    d4r_ngx_set_float(params, "MV.Scale.X", p.mvScaleX);
    d4r_ngx_set_float(params, "MV.Scale.Y", p.mvScaleY);
    d4r_ngx_set_float(params, "Sharpness", p.sharpness);
    // The DLSS library stamps "DLSS SDK - DO NOT DISTRIBUTE" over the output in this setup; the SDK's
    // Disable.Watermark parameter turns that off. D4R_SHIM_WATERMARK=1 keeps it.
    static const bool watermark = env_uint("D4R_SHIM_WATERMARK", 0) != 0;
    d4r_ngx_set_int(params, "Disable.Watermark", watermark ? 0 : 1);
    d4r_ngx_set_int(params, "Reset", p.reset);
    d4r_ngx_set_float(params, "DLSS.Pre.Exposure", p.preExposure);
    d4r_ngx_set_float(params, "DLSS.Exposure.Scale", p.exposureScale);
    d4r_ngx_set_float(params, "FrameTimeDeltaInMsec", p.frameTime);
    d4r_ngx_set_int(params, "DLSS.Indicator.Invert.X.Axis", p.invertX);
    d4r_ngx_set_int(params, "DLSS.Indicator.Invert.Y.Axis", p.invertY);
    d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Width", p.renderWidth);
    d4r_ngx_set_uint(params, "DLSS.Render.Subrect.Dimensions.Height", p.renderHeight);
    d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.X", p.colorBaseX);
    d4r_ngx_set_uint(params, "DLSS.Input.Color.Subrect.Base.Y", p.colorBaseY);
    d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.X", p.depthBaseX);
    d4r_ngx_set_uint(params, "DLSS.Input.Depth.Subrect.Base.Y", p.depthBaseY);
    d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.X", p.mvBaseX);
    d4r_ngx_set_uint(params, "DLSS.Input.MV.Subrect.Base.Y", p.mvBaseY);
    d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.X", p.outputBaseX);
    d4r_ngx_set_uint(params, "DLSS.Output.Subrect.Base.Y", p.outputBaseY);
    if (feature.scratch != 0)
    {
        d4r_ngx_set_void(params, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature.scratch)));
        d4r_ngx_set_ull(params, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
    }
}

// DLSS evaluations run as a three-stage pipeline so the CPU copies of one
// frame overlap the GPU work of another:
//   prepare_inputs (prep thread)  waits for the game's input copies and
//                                 converts the readback staging into host planes;
//   run_evaluation (CUDA worker)  uploads them, evaluates, downloads the output;
//   publish_output (finish thread) writes it into an upload staging buffer and
//                                 makes it the result evaluate() presents.
// Every NGX-core and CUDA call stays on the worker. A stage only reuses a
// buffer once the next stage has released it (hostBusy / outputHostBusy).
static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing);
static int claim_output_slot(Feature* feature);
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
                           const FrameTiming& timing, ProfileClock::time_point start);

// Spins until `flag` clears; false after two seconds.
static bool wait_for_release(const std::atomic<bool>& flag)
{
    const auto start = std::chrono::steady_clock::now();
    while (flag.load())
    {
        if (std::chrono::steady_clock::now() - start > std::chrono::seconds(2))
            return false;
        std::this_thread::sleep_for(std::chrono::microseconds(100));
    }
    return true;
}

static void release_split_frame(Feature* feature, uint32_t frame);

// Every queued frame ends in exactly one call of this (published or dropped).
static void frame_retired(Feature* feature, uint32_t frame)
{
    if (feature->split)
        release_split_frame(feature, frame);
    feature->inFlight.fetch_sub(1);
}

// Overlapped staging, worker thread only. Uploads the staged planes with async
// copies into linear buffers on the "up" stream, then moves them into the CUDA
// arrays device-to-device (on the null stream, ahead of NGX's kernels).
static bool upload_inputs_async(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
                                FrameTiming* timing)
{
    CopyStreams* streams = copy_streams();
    const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
    CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
    const int count = hasExposure ? 4 : 3;
    const auto start = ProfileClock::now();
    for (int index = 0; index < count; ++index)
    {
        const HostPlane& host = slot.host[index];
        if (!ensure_cuda_image(*images[index], host.width, host.height, plane_format(planes[index]),
                               plane_channels(planes[index]), false, planes[index] == Plane::Color) ||
            !ensure_linear(feature.planeLinear[index], feature.planeLinearBytes[index], host.size()))
            return false;
        if ((frame <= 3 || env_uint("D4R_SHIM_INPUT_HASH_ALL", 0) != 0) && GetEnvironmentVariableA("D4R_SHIM_INPUT_HASH", nullptr, 0) != 0)
        {
            uint64_t hash = 1469598103934665603ull;
            for (size_t byte = 0; byte < host.size(); ++byte)
                hash = (hash ^ host.bytes[byte]) * 1099511628211ull;
            logf("frame %u upload plane=%d bytes=%zu hash=%016llx (async)", frame, index, host.size(),
                 static_cast<unsigned long long>(hash));
        }
        const auto issue = ProfileClock::now();
        const int result = g.cu.memcpyHtoDAsync(feature.planeLinear[index], host.bytes, host.size(), streams->up);
        if (timing != nullptr)
            timing->upload[index] = profile_ms(issue, ProfileClock::now());
        if (result != 0)
        {
            logf("cuMemcpyHtoDAsync failed: %d", result);
            return false;
        }
    }
    int result = g.cu.streamSynchronize(streams->up);
    for (int index = 0; index < count && result == 0; ++index)
    {
        const HostPlane& host = slot.host[index];
        CudaMemcpy2D copy = {};
        copy.srcMemoryType = CUDA_MEMORY_DEVICE;
        copy.srcDevice = feature.planeLinear[index];
        copy.srcPitch = host.rowBytes;
        copy.dstMemoryType = CUDA_MEMORY_ARRAY;
        copy.dstArray = images[index]->array;
        copy.WidthInBytes = host.rowBytes;
        copy.Height = host.height;
        result = copy_2d(copy);
    }
    if (timing != nullptr)
        timing->h2dTotal = profile_ms(start, ProfileClock::now());
    if (result != 0)
        logf("staged upload failed: %d", result);
    return result == 0;
}

// Starts the output download: array -> linear buffer, then an async copy on
// the "down" stream that the next frame's uploads overlap.
static bool start_download(Feature& feature, HostPlane& host, FrameTiming* timing)
{
    const size_t rowBytes = static_cast<size_t>(feature.output.width) * 8;
    const size_t bytes = rowBytes * feature.output.height;
    reserve_host(host, bytes);
    host.width = feature.output.width;
    host.height = feature.output.height;
    host.rowBytes = rowBytes;
    if (!ensure_linear(feature.outputLinear, feature.outputLinearBytes, bytes))
        return false;
    const auto start = ProfileClock::now();
    CudaMemcpy2D copy = {};
    copy.srcMemoryType = CUDA_MEMORY_ARRAY;
    copy.srcArray = feature.output.array;
    copy.dstMemoryType = CUDA_MEMORY_DEVICE;
    copy.dstDevice = feature.outputLinear;
    copy.dstPitch = rowBytes;
    copy.WidthInBytes = rowBytes;
    copy.Height = host.height;
    int result = feature.outputRedirected ? 0 : copy_2d(copy);
    // The null-stream copy must land before the non-blocking stream reads it.
    if (result == 0)
        result = g.cu.ctxSynchronize();
    if (result == 0)
        result = g.cu.memcpyDtoHAsync(host.bytes, feature.outputLinear, bytes, copy_streams()->down);
    if (timing != nullptr)
        timing->d2hIssue = profile_ms(start, ProfileClock::now());
    if (result != 0)
        logf("staged download failed: %d", result);
    return result == 0;
}

// A result whose download is still in flight so the next frame's uploads can
// overlap it. Worker thread only.
struct PendingDownload
{
    Feature* feature = nullptr;
    int hostIndex = 0;
    uint32_t frame = 0;
    FrameParams params;
    FrameTiming timing;
};
static PendingDownload g_pendingDownload;

static void post_publish(Feature* feature, int hostIndex, uint32_t frame, const FrameParams& params,
                         FrameTiming timing)
{
    if (timing.enabled)
        timing.finishQueued = ProfileClock::now();
    g.finish.post([feature, hostIndex, frame, params, timing] {
        publish_output(feature, hostIndex, frame, params, timing);
    });
}

// Waits for the in-flight download, if any, and hands it to publish_output.
// `inline_completion` marks the case without a next frame to overlap: the wait
// then belongs to this frame's worker time.
static void complete_pending_download(bool inline_completion = false)
{
    PendingDownload pending = g_pendingDownload;
    if (pending.feature == nullptr)
        return;
    g_pendingDownload = {};
    const auto waitStart = ProfileClock::now();
    const int result = g.cu.streamSynchronize(copy_streams()->down);
    if (pending.timing.enabled)
    {
        pending.timing.d2hWait = profile_ms(waitStart, ProfileClock::now());
        pending.timing.download = pending.timing.d2hIssue + pending.timing.d2hWait;
        if (inline_completion)
            pending.timing.worker += pending.timing.d2hWait;
    }
    if (result != 0)
    {
        logf("frame %u: output download failed: %d", pending.frame, result);
        pending.feature->outputHostBusy[pending.hostIndex] = false;
        frame_retired(pending.feature, pending.frame);
        return;
    }
    log_output_hash(pending.feature->outputHost[pending.hostIndex], pending.frame);
    char dumpPath[MAX_PATH];
    if (output_dump_path(pending.frame, dumpPath))
        dump_output(pending.feature->outputHost[pending.hostIndex], pending.frame, dumpPath);
    post_publish(pending.feature, pending.hostIndex, pending.frame, pending.params, pending.timing);
}

// Linear inputs, worker thread: pitch-linear texture objects on the slot's interop buffers (recreated
// when a buffer or its geometry changes), reported to NGX as arrays by the bridge.
static bool ensure_linear_inputs(InputSlot& slot, bool hasExposure)
{
    const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
    const int count = hasExposure ? 4 : 3;
    for (int index = 0; index < count; ++index)
    {
        const HostPlane& geometry = slot.host[index];
        const VramBuffer& buffer = slot.vram[index];
        if (slot.linearTexture[index] != 0 && slot.linearPointer[index] == buffer.device &&
            slot.linearWidth[index] == geometry.width && slot.linearHeight[index] == geometry.height &&
            slot.linearPitch[index] == geometry.rowBytes)
            continue;
        if (slot.linearTexture[index] != 0)
            g.cu.texObjectDestroy(slot.linearTexture[index]);
        slot.linearTexture[index] = 0;
        struct Pitch2D
        {
            CudaDevicePtr pointer;
            uint32_t format, channels;
            size_t width, height, pitch;
        } pitch2D = {buffer.device, plane_format(planes[index]), plane_channels(planes[index]), geometry.width,
                     geometry.height, geometry.rowBytes};
        static_assert(sizeof(Pitch2D) == 40, "CUDA_RESOURCE_DESC pitch2D");
        CudaResourceDesc resource = {};
        resource.resType = 3; // CU_RESOURCE_TYPE_PITCH2D
        memcpy(&resource.res, &pitch2D, sizeof(pitch2D));
        const CudaTextureDesc sampler = input_sampler(geometry.width, pitch2D.channels, planes[index] == Plane::Color);
        CudaObject object = 0;
        int result = g.cu.texObjectCreate(&object, &resource, &sampler, nullptr);
        if (result == 0)
            result = g.cu.registerLinearTexture(object, geometry.width, geometry.height, pitch2D.format, pitch2D.channels);
        if (result != 0)
        {
            logf("linear input texture %d (%ux%u, pitch %zu) failed: %d", index, geometry.width, geometry.height,
                 geometry.rowBytes, result);
            if (object != 0)
                g.cu.texObjectDestroy(object);
            return false;
        }
        slot.linearTexture[index] = object;
        slot.linearPointer[index] = buffer.device;
        slot.linearWidth[index] = geometry.width;
        slot.linearHeight[index] = geometry.height;
        slot.linearPitch[index] = geometry.rowBytes;
    }
    return true;
}

// VRAM interop, worker thread: moves the planes from the imported buffers
// into the CUDA arrays on the GPU. With D4R_SHIM_VRAM_VERIFY also compares
// them with the host-staged planes of the same frame.
static bool upload_inputs_vram(Feature& feature, InputSlot& slot, bool hasExposure, uint32_t frame,
                               FrameTiming* timing)
{
    const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
    CudaImage* images[4] = {&feature.color, &feature.depth, &feature.motion, &feature.exposure};
    const int count = hasExposure ? 4 : 3;
    const bool verify = env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
    const auto start = ProfileClock::now();
    int result = 0;
    for (int index = 0; index < count && result == 0; ++index)
    {
        const HostPlane& geometry = slot.host[index];
        if (!ensure_cuda_image(*images[index], geometry.width, geometry.height, plane_format(planes[index]),
                               plane_channels(planes[index]), false, planes[index] == Plane::Color))
            return false;
        if (verify)
        {
            std::vector<uint8_t> vram(geometry.size());
            CudaMemcpy2D readback = {};
            readback.srcMemoryType = CUDA_MEMORY_DEVICE;
            readback.srcDevice = slot.vram[index].device;
            readback.srcPitch = geometry.rowBytes;
            readback.dstMemoryType = CUDA_MEMORY_HOST;
            readback.dstHost = vram.data();
            readback.dstPitch = geometry.rowBytes;
            readback.WidthInBytes = geometry.rowBytes;
            readback.Height = geometry.height;
            size_t differing = 0;
            if (g.cu.memcpy2D(&readback) == 0)
                for (size_t byte = 0; byte < vram.size(); ++byte)
                    differing += vram[byte] != geometry.bytes[byte];
            if (differing != 0 || frame <= 3 || frame % 120 == 0)
                logf("frame %u VRAM verify plane=%d: %zu of %zu bytes differ from host staging", frame, index,
                     differing, vram.size());
        }
        CudaMemcpy2D copy = {};
        copy.srcMemoryType = CUDA_MEMORY_DEVICE;
        copy.srcDevice = slot.vram[index].device;
        copy.srcPitch = geometry.rowBytes;
        copy.dstMemoryType = CUDA_MEMORY_ARRAY;
        copy.dstArray = images[index]->array;
        copy.WidthInBytes = geometry.rowBytes;
        copy.Height = geometry.height;
        result = copy_2d(copy);
        // D4R_SHIM_VRAM_ARRAY_VERIFY=1: read the CUDA array DLSS samples back
        // and compare it with the imported buffer it was filled from.
        static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
        if (arrayVerify && result == 0 && g.cu.ctxSynchronize() == 0)
        {
            std::vector<uint8_t> fromBuffer(geometry.size()), fromArray(geometry.size());
            CudaMemcpy2D readBuffer = {};
            readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
            readBuffer.srcDevice = slot.vram[index].device;
            readBuffer.srcPitch = geometry.rowBytes;
            readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
            readBuffer.dstHost = fromBuffer.data();
            readBuffer.dstPitch = geometry.rowBytes;
            readBuffer.WidthInBytes = geometry.rowBytes;
            readBuffer.Height = geometry.height;
            CudaMemcpy2D readArray = {};
            readArray.srcMemoryType = CUDA_MEMORY_ARRAY;
            readArray.srcArray = images[index]->array;
            readArray.dstMemoryType = CUDA_MEMORY_HOST;
            readArray.dstHost = fromArray.data();
            readArray.dstPitch = geometry.rowBytes;
            readArray.WidthInBytes = geometry.rowBytes;
            readArray.Height = geometry.height;
            if (g.cu.memcpy2D(&readBuffer) == 0 && g.cu.memcpy2D(&readArray) == 0)
            {
                size_t differing = 0, firstRow = geometry.height;
                for (size_t byte = 0; byte < fromArray.size(); ++byte)
                    if (fromArray[byte] != fromBuffer[byte])
                    {
                        if (differing++ == 0)
                            firstRow = byte / geometry.rowBytes;
                    }
                if (differing != 0 || frame <= 3 || frame % 120 == 0)
                    logf("frame %u VRAM array verify plane=%d: %zu of %zu bytes differ (first row %zu)", frame, index,
                         differing, fromArray.size(), firstRow);
            }
        }
    }
    if (timing != nullptr)
        timing->h2dTotal = profile_ms(start, ProfileClock::now());
    if (result != 0)
        logf("VRAM input copy failed: %d", result);
    return result == 0;
}

// VRAM interop, worker thread: copies the finished result into a free output
// slot's imported buffer and publishes it; evaluate() copies it on to the game.
static void publish_vram(Feature* feature, uint32_t frame, const FrameParams& params, FrameTiming& timing,
                         ProfileClock::time_point workerStart)
{
    const auto start = ProfileClock::now();
    // A split frame's second half reads slot frame % kOutputSlots; its previous
    // reader, frame - kOutputSlots, ran before this frame's marker.
    const int target = params.split ? static_cast<int>(frame % kOutputSlots) : claim_output_slot(feature);
    if (target < 0)
    {
        logf("frame %u: no free output slot; dropping result", frame);
        frame_retired(feature, frame);
        return;
    }
    const VramBuffer& buffer = feature->outputs[target].vram;
    const size_t rowBytes = static_cast<size_t>(feature->output.width) * 8;
    CudaMemcpy2D copy = {};
    copy.srcMemoryType = CUDA_MEMORY_ARRAY;
    copy.srcArray = feature->output.array;
    copy.dstMemoryType = CUDA_MEMORY_DEVICE;
    copy.dstDevice = buffer.device;
    copy.dstPitch = rowBytes;
    copy.WidthInBytes = rowBytes;
    copy.Height = feature->output.height;
    int result = rowBytes * feature->output.height > buffer.bytes ? -1 : feature->outputRedirected ? 0 : copy_2d(copy);
    if (result == 0)
        result = g.cu.ctxSynchronize();
    if (result != 0)
    {
        logf("frame %u: VRAM output copy failed: %d", frame, result);
        std::lock_guard<std::mutex> lock(feature->outputMutex);
        feature->outputs[target].lastReadFrame = 0;
        frame_retired(feature, frame);
        return;
    }
    static const bool arrayVerify = env_uint("D4R_SHIM_VRAM_ARRAY_VERIFY", 0) != 0;
    if (arrayVerify)
    {
        // The result as DLSS left it in its array vs the buffer the game copies from.
        std::vector<uint8_t> fromArray(rowBytes * feature->output.height), fromBuffer(fromArray.size());
        CudaMemcpy2D readArray = copy;
        readArray.dstMemoryType = CUDA_MEMORY_HOST;
        readArray.dstHost = fromArray.data();
        CudaMemcpy2D readBuffer = {};
        readBuffer.srcMemoryType = CUDA_MEMORY_DEVICE;
        readBuffer.srcDevice = buffer.device;
        readBuffer.srcPitch = rowBytes;
        readBuffer.dstMemoryType = CUDA_MEMORY_HOST;
        readBuffer.dstHost = fromBuffer.data();
        readBuffer.dstPitch = rowBytes;
        readBuffer.WidthInBytes = rowBytes;
        readBuffer.Height = feature->output.height;
        if (g.cu.memcpy2D(&readArray) == 0 && g.cu.memcpy2D(&readBuffer) == 0)
        {
            size_t differing = 0;
            for (size_t byte = 0; byte < fromArray.size(); ++byte)
                differing += fromArray[byte] != fromBuffer[byte];
            if (differing != 0 || frame <= 3 || frame % 120 == 0)
                logf("frame %u VRAM output verify: %zu of %zu bytes differ between result array and buffer", frame,
                     differing, fromArray.size());
        }
    }
    char dumpPath[MAX_PATH];
    const bool saveRaw = output_dump_path(frame, dumpPath);
    if (GetEnvironmentVariableA("D4R_SHIM_OUTPUT_HASH", nullptr, 0) != 0 || saveRaw)
    {
        HostPlane host;
        uint8_t* pinned = saveRaw ? capture_pinned(dumpPath, rowBytes * feature->output.height) : nullptr;
        if (pinned == nullptr)
            host.fallback.resize(rowBytes * feature->output.height);
        host.bytes = pinned != nullptr ? pinned : host.fallback.data();
        host.width = feature->output.width;
        host.height = feature->output.height;
        host.rowBytes = rowBytes;
        copy.dstMemoryType = CUDA_MEMORY_HOST;
        copy.dstHost = host.bytes;
        if (g.cu.memcpy2D(&copy) == 0)
        {
            log_output_hash(host, frame);
            if (saveRaw && pinned != nullptr)
                flush_capture_if_complete(frame); // already registered for writing
            else if (saveRaw)
                dump_output(host, frame, dumpPath);
        }
        else if (saveRaw)
            logf("frame %u raw DLSS output readback failed", frame);
    }
    if (timing.enabled)
    {
        timing.d2hIssue = profile_ms(start, ProfileClock::now());
        timing.download = timing.d2hIssue;
        timing.worker = profile_ms(workerStart, ProfileClock::now());
    }
    finish_publish(feature, target, frame, params, timing, start);
}

static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
    InputSlot& slot = feature->inputs[slotIndex];
    const auto start = ProfileClock::now();
    if (timing.enabled)
        timing.workerWait = profile_ms(timing.workerQueued, start);
    FrameTiming* stages = timing.enabled ? &timing : nullptr;
    const bool overlapped = async_copies() && copy_streams() != nullptr;
    const bool gpuWait = params.vram && feature->gpuWait;
    if (params.vram && !gpuWait && env_uint("D4R_SHIM_VRAM_VERIFY", 0) == 0)
        dump_vram_inputs(slot, frame, params, params.hasExposure ? 4 : 3);
    if (gpuWait)
    {
        feature->gpuWaitSince = std::chrono::steady_clock::now().time_since_epoch().count();
        feature->gpuWaitFrame = frame;
    }
    if (gpuWait && g.cu.streamWaitValue32(feature->gpuMarker.device, frame) != 0)
    {
        logf("frame %u: queueing the GPU-side input wait failed", frame);
        slot.busy = false;
        frame_retired(feature, frame);
        return;
    }
    const bool linearInputs = gpuWait && feature->linearInputs;
    bool ok = linearInputs ? ensure_linear_inputs(slot, params.hasExposure)
              : params.vram ? upload_inputs_vram(*feature, slot, params.hasExposure, frame, stages)
              : overlapped
                  ? upload_inputs_async(*feature, slot, params.hasExposure, frame, stages)
                  : upload_plane(Plane::Color, slot.host[0], feature->color, frame, stages, 0) &&
                        upload_plane(Plane::Depth, slot.host[1], feature->depth, frame, stages, 1) &&
                        upload_plane(Plane::Motion, slot.host[2], feature->motion, frame, stages, 2) &&
                        (!params.hasExposure ||
                         upload_plane(Plane::Exposure, slot.host[3], feature->exposure, frame, stages, 3));
    ok = ok && ensure_cuda_image(feature->output, feature->outputWidth, feature->outputHeight, CUDA_FORMAT_HALF, 4,
                                 true, false);
    // Device-to-array copies (VRAM interop, staged uploads) return before they
    // finish, and NGX does not evaluate on the stream they were issued on, so
    // without this DLSS can sample a mix of this frame's and the previous
    // frame's colour/depth/motion: invisible when the camera is still, beaded
    // thin geometry in motion. D4R_SHIM_INPUT_SYNC=0 restores the old behaviour.
    static const bool inputSync = env_uint("D4R_SHIM_INPUT_SYNC", 1) != 0;
    // With gpuWait everything up to the output copy is queued in order on the null stream.
    if (ok && inputSync && !gpuWait)
    {
        const int syncResult = g.cu.ctxSynchronize();
        if (syncResult != 0)
        {
            logf("frame %u: input copy synchronize failed: %d", frame, syncResult);
            ok = false;
        }
    }
    slot.hostBusy = false; // the host planes are no longer needed
    // The previous frame's download ran alongside these uploads.
    complete_pending_download();
    if (!ok)
    {
        slot.busy = false;
        frame_retired(feature, frame);
        return;
    }

    // The first frame NGX evaluates for a feature starts its history. Decided
    // here, in evaluation order, so it does not depend on thread timing.
    if (feature->evaluatedFrames++ == 0)
        params.reset = 1;
    set_evaluation_parameters(*feature, params);
    if (linearInputs)
    {
        // the parameters point at these variables
        feature->colorHandle = slot.linearTexture[0];
        feature->depthHandle = slot.linearTexture[1];
        feature->motionHandle = slot.linearTexture[2];
        feature->exposureHandle = params.hasExposure ? slot.linearTexture[3] : feature->exposureHandle;
    }
    static const bool outputDirect = env_uint("D4R_SHIM_OUTPUT_DIRECT", 0) != 0;
    // The redirect is carried out by the native output kernel's stores (ZLUDA's compile of NVIDIA's kernel
    // writes the array), so it is set while the previous frame's output kernel was the native one and
    // undone below when this frame's was not.
    const bool nativeOutput = g.cu.outputKernelNative == nullptr || g.cu.outputKernelNative() == 1;
    if (outputDirect && feature->outputDirectAllowed && g.cu.setArrayRedirect != nullptr && feature->output.array != nullptr &&
        (nativeOutput || feature->outputRedirected))
    {
        // split frames write their own slot; the staged path downloads from outputLinear
        const size_t rowBytes = static_cast<size_t>(feature->output.width) * 8, bytes = rowBytes * feature->output.height;
        CudaDevicePtr target = 0;
        if (params.vram && params.split && feature->outputs[frame % kOutputSlots].vram.bytes >= bytes)
            target = feature->outputs[frame % kOutputSlots].vram.device;
        else if (!params.vram && overlapped && ensure_linear(feature->outputLinear, feature->outputLinearBytes, bytes))
            target = feature->outputLinear;
        const bool redirect = nativeOutput && target != 0 &&
                              g.cu.setArrayRedirect(feature->output.array, target, static_cast<uint32_t>(rowBytes)) == 0;
        if (!redirect && feature->outputRedirected)
            g.cu.setArrayRedirect(feature->output.array, 0, 0);
        feature->outputRedirected = redirect;
    }
    const int eventStartResult = timing.enabled && feature->profileEventsReady
                                     ? g.cu.eventRecord(feature->profileStart, nullptr) : -1;
    const auto evalStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
    const NgxResult result = g.ngx.evaluateFeature(feature->cudaHandle, feature->cudaParams, nullptr);
    if (feature->outputRedirected && g.cu.outputKernelNative != nullptr && g.cu.outputKernelNative() != 1)
    {
        // this frame's output kernel ignored the redirect: publish from the array as usual
        g.cu.setArrayRedirect(feature->output.array, 0, 0);
        feature->outputRedirected = false;
    }
    const auto evalReturned = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
    const int eventEndResult = eventStartResult == 0 ? g.cu.eventRecord(feature->profileEnd, nullptr) : -1;
    const auto syncStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
    // D4R_SHIM_EVAL_SYNC=0 (VRAM interop): no wait between NGX's kernels and the output copy queued
    // behind them on the same stream; publish_vram synchronises before the result is released.
    static const bool evalSync = env_uint("D4R_SHIM_EVAL_SYNC", 1) != 0;
    const int syncResult = params.vram && !evalSync ? 0 : g.cu.ctxSynchronize();
    const auto evaluated = ProfileClock::now();
    float gpuEvalMs = -1.0f;
    if (eventStartResult == 0 && eventEndResult == 0 && syncResult == 0 &&
        g.cu.eventElapsedTime(&gpuEvalMs, feature->profileStart, feature->profileEnd) != 0)
        gpuEvalMs = -1.0f;
    if (params.vram && !gpuWait)
        slot.busy = false; // the input copies ran before the evaluation
    if (result != NGX_SUCCESS || syncResult != 0)
    {
        logf("frame %u: CUDA_EvaluateFeature -> 0x%08x, sync %d", frame, result, syncResult);
        if (gpuWait)
        {
            g.cu.ctxSynchronize();
            feature->gpuWaitFrame = 0;
            slot.busy = false;
        }
        frame_retired(feature, frame);
        return;
    }
    if (timing.enabled)
    {
        timing.ngxHost = profile_ms(evalStart, evalReturned);
        timing.ctxSync = profile_ms(syncStart, evaluated);
        timing.gpuEval = gpuEvalMs;
    }
    if (params.vram)
    {
        publish_vram(feature, frame, params, timing, start);
        if (gpuWait)
        {
            feature->gpuWaitFrame = 0;
            slot.busy = false; // publish_vram waited for the whole frame, input copies included
        }
        return;
    }

    const int hostIndex = static_cast<int>(feature->nextOutputHost++ % 2);
    if (!wait_for_release(feature->outputHostBusy[hostIndex]))
    {
        logf("frame %u: output host buffer still busy; dropping result", frame);
        frame_retired(feature, frame);
        return;
    }
    if (overlapped)
    {
        if (!start_download(*feature, feature->outputHost[hostIndex], stages))
        {
            frame_retired(feature, frame);
            return;
        }
        feature->outputHostBusy[hostIndex] = true;
        if (timing.enabled)
            timing.worker = profile_ms(start, ProfileClock::now());
        g_pendingDownload = {feature, hostIndex, frame, params, timing};
        // Without a queued frame to overlap, finish it now rather than add latency.
        if (g.worker.pending() == 0)
            complete_pending_download(true);
        return;
    }
    if (!download_output(*feature, feature->outputHost[hostIndex], frame, stages))
    {
        frame_retired(feature, frame);
        return;
    }
    feature->outputHostBusy[hostIndex] = true;
    if (timing.enabled)
        timing.worker = profile_ms(start, ProfileClock::now());
    post_publish(feature, hostIndex, frame, params, timing);
}

static void prepare_inputs(Feature* feature, int slotIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
    InputSlot& slot = feature->inputs[slotIndex];
    const auto start = ProfileClock::now();
    // Wait until the game's command list with this frame's input copies ran (with gpuWait the GPU waits).
    while (!(params.vram && feature->gpuWait) && static_cast<int32_t>(*feature->markerValue - frame) < 0)
    {
        std::atomic_thread_fence(std::memory_order_acquire);
        if (std::chrono::steady_clock::now() - start > std::chrono::seconds(5))
        {
            logf("frame %u: timed out waiting for GPU marker (at %u); dropping", frame, *feature->markerValue);
            slot.busy = false;
            frame_retired(feature, frame);
            return;
        }
        // D4R_SHIM_MARKER_POLL_US: poll interval (0 = yield); detection latency is on the critical path
        static const unsigned markerPollUs = env_uint("D4R_SHIM_MARKER_POLL_US", 200);
        if (markerPollUs == 0)
            std::this_thread::yield();
        else
            std::this_thread::sleep_for(std::chrono::microseconds(markerPollUs));
    }
    const auto ready = ProfileClock::now();
    // With VRAM interop the inputs are already on the GPU; host staging runs
    // only to cross-check them (D4R_SHIM_VRAM_VERIFY).
    if (!params.vram || env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0)
    {
        if (!wait_for_release(slot.hostBusy))
        {
            logf("frame %u: host planes of slot %d still busy; dropping", frame, slotIndex);
            slot.busy = false;
            frame_retired(feature, frame);
            return;
        }
        FrameTiming* stages = timing.enabled ? &timing : nullptr;
        stage_plane(Plane::Color, slot.color, slot.host[0], stages, 0);
        stage_plane(Plane::Depth, slot.depth, slot.host[1], stages, 1);
        stage_plane(Plane::Motion, slot.motion, slot.host[2], stages, 2);
        log_motion_stats(slot.host[2], slot.host[0], frame);
        if (params.hasExposure)
            stage_plane(Plane::Exposure, slot.exposure, slot.host[3], stages, 3);
        char dumpDirectory[MAX_PATH];
        if (input_dump_directory(frame, dumpDirectory))
            dump_input_planes(slot.host, frame, params, dumpDirectory);
        slot.hostBusy = true;
    }
    if (!params.vram)
        slot.busy = false; // the readback buffers are no longer needed
    if (timing.enabled)
    {
        timing.prepStart = start;
        timing.markerWait = profile_ms(start, ready);
        timing.prep = profile_ms(start, ProfileClock::now());
        timing.workerQueued = ProfileClock::now();
    }
    g.worker.post([feature, slotIndex, frame, params, timing] {
        run_evaluation(feature, slotIndex, frame, params, timing);
    });
}

// Picks a result slot that no queued command list still copies from, and
// marks it so evaluate() does not present it while it is rewritten. A reader
// frame's output copy is recorded after its marker, so the GPU is past it
// only once a later frame's marker has landed. Waits briefly for that.
static int claim_output_slot(Feature* feature)
{
    const auto start = std::chrono::steady_clock::now();
    for (;;)
    {
        {
            std::lock_guard<std::mutex> lock(feature->outputMutex);
            const uint32_t gpuFrame = *feature->markerValue;
            for (int index = 0; index < kOutputSlots; ++index)
            {
                if (index == feature->latestOutput.load())
                    continue;
                if (static_cast<int32_t>(gpuFrame - feature->outputs[index].lastReadFrame.load()) > 0)
                {
                    feature->outputs[index].lastReadFrame = 0x7fffffffu + gpuFrame;
                    return index;
                }
            }
        }
        if (std::chrono::steady_clock::now() - start > std::chrono::milliseconds(50))
            return -1;
        std::this_thread::sleep_for(std::chrono::microseconds(200));
    }
}

// Makes `target` the result evaluate() presents and retires the frame.
static void finish_publish(Feature* feature, int target, uint32_t frame, const FrameParams& params,
                           const FrameTiming& timing, ProfileClock::time_point start)
{
    {
        std::lock_guard<std::mutex> lock(feature->outputMutex);
        feature->outputs[target].lastReadFrame = 0;
        feature->outputs[target].producedFrame = frame;
        feature->latestOutput = target;
    }
    const uint32_t completed = ++feature->completedFrames;
    frame_retired(feature, frame);
    const auto finished = ProfileClock::now();
    if (timing.enabled)
    {
        static thread_local ProfileClock::time_point lastFinished;
        const double interval = lastFinished.time_since_epoch().count() != 0
                                    ? profile_ms(lastFinished, finished) : -1.0;
        lastFinished = finished;
        const InputSlot& slot = feature->inputs[frame % kSlots];
        logf("D4R_PROFILE frame=%u render=%ux%u output=%ux%u color=%ux%u depth=%ux%u "
             "motion=%ux%u exposure=%d quality=%d flags=0x%x preset=%u "
             "jitter=%.6f,%.6f mv_scale=%.3f,%.3f reset=%d pre_exposure=%.6f exposure_scale=%.6f "
             "frame_dt=%.3f game_interval=%.3f "
             "queue_depth=%zu queue_wait=%.3f marker_wait=%.3f slot_wait=%.3f "
             "d3d_input_record=%.3f d3d_output_record=%.3f game_call=%.3f "
             "color_convert=%.3f depth_convert=%.3f motion_convert=%.3f exposure_convert=%.3f "
             "color_h2d=%.3f depth_h2d=%.3f motion_h2d=%.3f exposure_h2d=%.3f "
             "ngx_host=%.3f ctx_sync=%.3f gpu_eval=%.3f d2h=%.3f output_convert=%.3f "
             "h2d_total=%.3f d2h_issue=%.3f d2h_wait=%.3f "
             "prep=%.3f worker_wait=%.3f worker=%.3f finish_wait=%.3f finish=%.3f pipeline=%.3f "
             "throughput_interval=%.3f presented=%u age=%d",
             frame, params.renderWidth, params.renderHeight, feature->outWidth, feature->outHeight,
             slot.host[0].width, slot.host[0].height, slot.host[1].width, slot.host[1].height,
             slot.host[2].width, slot.host[2].height, params.hasExposure, feature->quality, feature->flags,
             feature->preset, params.jitterX, params.jitterY, params.mvScaleX, params.mvScaleY,
             params.reset, params.preExposure, params.exposureScale, params.frameTime, timing.gameInterval,
             timing.queueDepth, profile_ms(timing.queuedAt, timing.prepStart), timing.markerWait, timing.slotWait,
             timing.inputRecord, timing.outputRecord, timing.gameCall,
             timing.convert[0], timing.convert[1], timing.convert[2], timing.convert[3],
             timing.upload[0], timing.upload[1], timing.upload[2], timing.upload[3],
             timing.ngxHost, timing.ctxSync, timing.gpuEval, timing.download, timing.outputConvert,
             timing.h2dTotal, timing.d2hIssue, timing.d2hWait,
             timing.prep, timing.workerWait, timing.worker, timing.finishWait, profile_ms(start, finished),
             profile_ms(timing.prepStart, finished), interval,
             timing.presentedFrame, timing.presentedFrame != 0 ? static_cast<int>(frame - timing.presentedFrame) : -1);
    }
    if (completed <= 5 || completed % 120 == 0)
        logf("frame %u done (slot %d)", frame, target);
}

static void publish_output(Feature* feature, int hostIndex, uint32_t frame, FrameParams params, FrameTiming timing)
{
    const auto start = ProfileClock::now();
    if (timing.enabled)
        timing.finishWait = profile_ms(timing.finishQueued, start);
    const int target = claim_output_slot(feature);
    if (target < 0)
    {
        feature->outputHostBusy[hostIndex] = false;
        logf("frame %u: no free output slot; dropping result", frame);
        frame_retired(feature, frame);
        return;
    }
    write_output(feature->outputs[target].staging, feature->outputHost[hostIndex], timing.enabled ? &timing : nullptr);
    feature->outputHostBusy[hostIndex] = false;
    finish_publish(feature, target, frame, params, timing, start);
}

// Waits until every frame queued so far has passed through all three stages.
static void drain_pipeline()
{
    g.prep.call([] { return 0; });
    g.worker.call([] {
        complete_pending_download();
        return 0;
    });
    g.finish.call([] { return 0; });
}

// --- exported NGX D3D12 API ------------------------------------------------------

#define D4R_EXPORT extern "C" __declspec(dllexport)

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init(unsigned long long applicationId, const wchar_t* dataPath,
                                          ID3D12Device* device, const NgxFeatureCommonInfo* featureInfo,
                                          unsigned int sdkVersion)
{
    logf("NVSDK_NGX_D3D12_Init(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
    return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_Ext(unsigned long long applicationId, const wchar_t* dataPath,
                                              ID3D12Device* device, unsigned int sdkVersion,
                                              const NgxFeatureCommonInfo* featureInfo)
{
    logf("NVSDK_NGX_D3D12_Init_Ext(app=%llu, device=%p, sdk=0x%x)", applicationId, device, sdkVersion);
    return initialize(applicationId, dataPath, device, featureInfo, sdkVersion);
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Init_ProjectID(const char* projectId, int engineType, const char* engineVersion,
                                                    const wchar_t* dataPath, ID3D12Device* device,
                                                    unsigned int sdkVersion, const NgxFeatureCommonInfo* featureInfo)
{
    logf("NVSDK_NGX_D3D12_Init_ProjectID(project=%s, engine=%d %s, device=%p, sdk=0x%x)",
         projectId != nullptr ? projectId : "", engineType, engineVersion != nullptr ? engineVersion : "", device,
         sdkVersion);
    ProjectIdentity project;
    project.id = projectId != nullptr ? projectId : "";
    project.engineType = engineType;
    project.engineVersion = engineVersion != nullptr ? engineVersion : "";
    return initialize(0, dataPath, device, featureInfo, sdkVersion, &project);
}

// Parameter objects handed to callers are the shim's own implementation: the
// CUDA-path objects of the official core cannot hold D3D12 resources.
static std::mutex g_parametersMutex;
static std::vector<void*> g_ownedParameters;

static void* new_parameters()
{
    void* parameters = d4r_ngx_parameters_create();
    std::lock_guard<std::mutex> lock(g_parametersMutex);
    g_ownedParameters.push_back(parameters);
    return parameters;
}

// Copies the official capability values (availability, driver requirements,
// and the optimal-settings/stats callbacks, which work on any parameter
// implementation) into one of our parameter objects.
static NgxResult fill_capabilities(void* parameters)
{
    return g.worker.call([&]() -> NgxResult {
        void* official = nullptr;
        const NgxResult result = g.ngx.getCapabilityParameters(&official);
        if (result != NGX_SUCCESS || official == nullptr)
            return result;
        const char* integers[] = {"SuperSampling.Available", "SuperSampling.NeedsUpdatedDriver",
                                  "SuperSampling.FeatureInitResult"};
        for (const char* name : integers)
        {
            int value = 0;
            if (d4r_ngx_get_int(official, name, &value) == NGX_SUCCESS)
                d4r_ngx_set_int(parameters, name, value);
        }
        const char* unsignedValues[] = {"SuperSampling.MinDriverVersionMajor", "SuperSampling.MinDriverVersionMinor"};
        for (const char* name : unsignedValues)
        {
            unsigned int value = 0;
            if (d4r_ngx_get_uint(official, name, &value) == NGX_SUCCESS)
                d4r_ngx_set_uint(parameters, name, value);
        }
        const char* pointers[] = {"DLSSOptimalSettingsCallback", "DLSSGetStatsCallback"};
        for (const char* name : pointers)
        {
            void* value = nullptr;
            if (d4r_ngx_get_void(official, name, &value) == NGX_SUCCESS && value != nullptr)
                d4r_ngx_set_void(parameters, name, value);
        }
        g.ngx.destroyParameters(official);
        return NGX_SUCCESS;
    });
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetParameters(void** parameters)
{
    if (!g.ngxInitialized)
        return NGX_FAIL_NOT_INITIALIZED;
    *parameters = new_parameters();
    return fill_capabilities(*parameters);
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_AllocateParameters(void** parameters)
{
    if (!g.ngxInitialized)
        return NGX_FAIL_NOT_INITIALIZED;
    *parameters = new_parameters();
    return NGX_SUCCESS;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetCapabilityParameters(void** parameters)
{
    if (!g.ngxInitialized)
        return NGX_FAIL_NOT_INITIALIZED;
    *parameters = new_parameters();
    const NgxResult result = fill_capabilities(*parameters);
    int available = -1, initResult = -1, needsDriver = -1;
    d4r_ngx_get_int(*parameters, "SuperSampling.Available", &available);
    d4r_ngx_get_int(*parameters, "SuperSampling.FeatureInitResult", &initResult);
    d4r_ngx_get_int(*parameters, "SuperSampling.NeedsUpdatedDriver", &needsDriver);
    logf("NVSDK_NGX_D3D12_GetCapabilityParameters -> 0x%08x, SuperSampling.Available=%d FeatureInitResult=0x%08x "
         "NeedsUpdatedDriver=%d",
         result, available, static_cast<unsigned int>(initResult), needsDriver);
    return result;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_DestroyParameters(void* parameters)
{
    {
        std::lock_guard<std::mutex> lock(g_parametersMutex);
        for (auto it = g_ownedParameters.begin(); it != g_ownedParameters.end(); ++it)
            if (*it == parameters)
            {
                g_ownedParameters.erase(it);
                d4r_ngx_parameters_destroy(parameters);
                return NGX_SUCCESS;
            }
    }
    return NGX_FAIL_INVALID_PARAMETER;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetScratchBufferSize(unsigned int, const void*, size_t* size)
{
    // Scratch memory for the CUDA path is allocated internally.
    if (size != nullptr)
        *size = 0;
    return NGX_SUCCESS;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_GetFeatureRequirements(IUnknown*, const void*, NgxFeatureRequirement* requirement)
{
    if (requirement != nullptr)
    {
        std::memset(requirement, 0, sizeof(*requirement));
        requirement->FeatureSupported = 0; // supported
        requirement->MinHWArchitecture = 0x160;
    }
    return NGX_SUCCESS;
}

static unsigned int get_uint_or(void* parameters, const char* name, unsigned int fallback)
{
    unsigned int value = fallback;
    if (d4r_ngx_get_uint(parameters, name, &value) != NGX_SUCCESS)
    {
        int signedValue = 0;
        if (d4r_ngx_get_int(parameters, name, &signedValue) == NGX_SUCCESS)
            return static_cast<unsigned int>(signedValue);
        return fallback;
    }
    return value;
}

static int get_int_or(void* parameters, const char* name, int fallback)
{
    int value = fallback;
    if (d4r_ngx_get_int(parameters, name, &value) != NGX_SUCCESS)
    {
        unsigned int unsignedValue = 0;
        if (d4r_ngx_get_uint(parameters, name, &unsignedValue) == NGX_SUCCESS)
            return static_cast<int>(unsignedValue);
        return fallback;
    }
    return value;
}

static float get_float_or(void* parameters, const char* name, float fallback)
{
    float value = fallback;
    return d4r_ngx_get_float(parameters, name, &value) == NGX_SUCCESS ? value : fallback;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned int featureId, void* parameters,
                                                   NgxHandle** handle)
{
    if (!g.ngxInitialized)
        return NGX_FAIL_NOT_INITIALIZED;
    if (featureId != NGX_FEATURE_SUPER_SAMPLING || parameters == nullptr || handle == nullptr)
    {
        logf("NVSDK_NGX_D3D12_CreateFeature(feature=%u) unsupported", featureId);
        return NGX_FAIL_FEATURE_NOT_SUPPORTED;
    }
    auto* feature = new Feature();
    feature->width = get_uint_or(parameters, "Width", 0);
    feature->height = get_uint_or(parameters, "Height", 0);
    feature->outWidth = get_uint_or(parameters, "OutWidth", 0);
    feature->outHeight = get_uint_or(parameters, "OutHeight", 0);
    const int quality = get_int_or(parameters, "PerfQualityValue", 0);
    const int flags = get_int_or(parameters, "DLSS.Feature.Create.Flags", 0);
    feature->quality = quality;
    feature->flags = flags;
    const int subrects = get_int_or(parameters, "DLSS.Enable.Output.Subrects", 0);
    const char* presetNames[] = {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality",
                                 "DLSS.Hint.Render.Preset.Balanced", "DLSS.Hint.Render.Preset.Performance",
                                 "DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"};
    unsigned int presets[6];
    for (int index = 0; index < 6; ++index)
        presets[index] = get_uint_or(parameters, presetNames[index], 0);
    // D4R_DLSS_PRESET forces one render preset (NVSDK_NGX_DLSS_Hint_Render_Preset
    // value, e.g. 5 = E) for every quality mode.
    const std::string forced = env_string("D4R_DLSS_PRESET");
    if (!forced.empty())
    {
        const unsigned int value = static_cast<unsigned int>(strtoul(forced.c_str(), nullptr, 0));
        for (unsigned int& preset : presets)
            preset = value;
    }
    feature->preset = presets[1];
    // Direct output needs the native output kernel of the preset in use (verified: K = 11; M's result is
    // not written through the redirected stores). Only a forced preset qualifies.
    if (!forced.empty())
    {
        char allowed[128] = "11";
        if (const std::string list = env_string("D4R_SHIM_OUTPUT_DIRECT_PRESETS"); !list.empty())
            snprintf(allowed, sizeof(allowed), "%s", list.c_str());
        for (char* token = strtok(allowed, ","); token != nullptr; token = strtok(nullptr, ","))
            feature->outputDirectAllowed |= strtoul(token, nullptr, 0) == presets[1];
    }
    logf("NVSDK_NGX_D3D12_CreateFeature: %ux%u -> %ux%u quality=%d flags=0x%x subrects=%d preset=%u", feature->width,
         feature->height, feature->outWidth, feature->outHeight, quality, flags, subrects, presets[1]);

    const NgxResult result = g.worker.call([&]() -> NgxResult {
        NgxResult allocation = g.ngx.allocateParameters(&feature->cudaParams);
        if (allocation != NGX_SUCCESS)
            return allocation;
        void* p = feature->cudaParams;
        d4r_ngx_set_uint(p, "Width", feature->width);
        d4r_ngx_set_uint(p, "Height", feature->height);
        d4r_ngx_set_uint(p, "OutWidth", feature->outWidth);
        d4r_ngx_set_uint(p, "OutHeight", feature->outHeight);
        d4r_ngx_set_int(p, "PerfQualityValue", quality);
        d4r_ngx_set_int(p, "DLSS.Feature.Create.Flags", flags);
        d4r_ngx_set_int(p, "DLSS.Enable.Output.Subrects", subrects);
        d4r_ngx_set_uint(p, "CreationNodeMask", 1);
        d4r_ngx_set_uint(p, "VisibilityNodeMask", 1);
        for (int index = 0; index < 6; ++index)
            d4r_ngx_set_uint(p, presetNames[index], presets[index]);
        if (g.cu.memAlloc(&feature->scratch, 64ull * 1024 * 1024) == 0)
        {
            d4r_ngx_set_void(p, "Scratch", reinterpret_cast<void*>(static_cast<uintptr_t>(feature->scratch)));
            d4r_ngx_set_ull(p, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
        }
        else
            feature->scratch = 0;
        const NgxResult created = g.ngx.createFeature(NGX_FEATURE_SUPER_SAMPLING, p, &feature->cudaHandle);
        if (created == NGX_SUCCESS && profile_enabled() && g.cu.eventCreate != nullptr &&
            g.cu.eventRecord != nullptr && g.cu.eventElapsedTime != nullptr && g.cu.eventDestroy != nullptr)
        {
            const int startResult = g.cu.eventCreate(&feature->profileStart, 0);
            const int endResult = startResult == 0 ? g.cu.eventCreate(&feature->profileEnd, 0) : -1;
            feature->profileEventsReady = startResult == 0 && endResult == 0;
            if (!feature->profileEventsReady)
            {
                logf("D4R_PROFILE: CUDA event creation failed (%d, %d)", startResult, endResult);
                if (feature->profileStart != nullptr)
                    g.cu.eventDestroy(feature->profileStart);
                if (feature->profileEnd != nullptr)
                    g.cu.eventDestroy(feature->profileEnd);
                feature->profileStart = feature->profileEnd = nullptr;
            }
        }
        return created;
    });
    logf("NVSDK_NGX_CUDA_CreateFeature -> 0x%08x", result);
    if (result != NGX_SUCCESS)
    {
        delete feature;
        return result;
    }
    if (!create_buffer(D3D12_HEAP_TYPE_READBACK, 256, &feature->marker,
                       reinterpret_cast<uint8_t**>(const_cast<uint32_t**>(&feature->markerValue))))
    {
        delete feature;
        return NGX_FAIL_PLATFORM_ERROR;
    }
    *feature->markerValue = 0;
    feature->handle.Id = g.nextHandleId++;
    {
        std::lock_guard<std::mutex> lock(g_featuresMutex);
        g_features.push_back(feature);
    }
    *handle = &feature->handle;
    return NGX_SUCCESS;
}

static Feature* find_feature(const NgxHandle* handle)
{
    std::lock_guard<std::mutex> lock(g_featuresMutex);
    for (Feature* feature : g_features)
        if (&feature->handle == handle)
            return feature;
    return nullptr;
}

static ID3D12Resource* get_resource(void* parameters, const char* name)
{
    ID3D12Resource* resource = nullptr;
    if (d4r_ngx_get_d3d12_resource(parameters, name, &resource) == NGX_SUCCESS && resource != nullptr)
        return resource;
    void* raw = nullptr;
    if (d4r_ngx_get_void(parameters, name, &raw) == NGX_SUCCESS)
        return static_cast<ID3D12Resource*>(raw);
    return nullptr;
}

static NgxResult evaluate(ID3D12GraphicsCommandList* list, const NgxHandle* handle, void* parameters)
{
    FrameTiming timing;
    timing.enabled = profile_enabled();
    if (timing.enabled)
    {
        timing.gameStart = ProfileClock::now();
        static thread_local ProfileClock::time_point lastGameStart;
        if (lastGameStart.time_since_epoch().count() != 0)
            timing.gameInterval = profile_ms(lastGameStart, timing.gameStart);
        lastGameStart = timing.gameStart;
    }
    Feature* feature = find_feature(handle);
    if (feature == nullptr || list == nullptr || parameters == nullptr)
        return NGX_FAIL_INVALID_PARAMETER;
    ID3D12Resource* color = get_resource(parameters, "Color");
    ID3D12Resource* depth = get_resource(parameters, "Depth");
    ID3D12Resource* motion = get_resource(parameters, "MotionVectors");
    ID3D12Resource* output = get_resource(parameters, "Output");
    ID3D12Resource* exposure = get_resource(parameters, "ExposureTexture");
    const uint32_t frame = ++feature->frame;
    if (color == nullptr || depth == nullptr || motion == nullptr || output == nullptr)
    {
        if (frame <= 3)
            logf("evaluate frame %u: missing resource color=%p depth=%p mv=%p output=%p", frame, color, depth, motion,
                 output);
        return NGX_FAIL_INVALID_PARAMETER;
    }

    D3D12_RESOURCE_DESC colorDesc, depthDesc, motionDesc, outputDesc;
    color->GetDesc(&colorDesc);
    depth->GetDesc(&depthDesc);
    motion->GetDesc(&motionDesc);
    output->GetDesc(&outputDesc);
    if (frame == 1)
        logf("evaluate: color %llux%u fmt=%d, depth %llux%u fmt=%d, mv %llux%u fmt=%d, output %llux%u fmt=%d, "
             "exposure=%p",
             colorDesc.Width, colorDesc.Height, colorDesc.Format, depthDesc.Width, depthDesc.Height, depthDesc.Format,
             motionDesc.Width, motionDesc.Height, motionDesc.Format, outputDesc.Width, outputDesc.Height,
             outputDesc.Format, exposure);
    if (!supported_input(Plane::Color, colorDesc.Format) || !supported_input(Plane::Depth, depthDesc.Format) ||
        !supported_input(Plane::Motion, motionDesc.Format) || !supported_output(outputDesc.Format))
    {
        if (frame <= 3)
            logf("evaluate frame %u: unsupported format", frame);
        return NGX_FAIL_UNSUPPORTED_FORMAT;
    }
    if (exposure != nullptr)
    {
        D3D12_RESOURCE_DESC exposureDesc;
        exposure->GetDesc(&exposureDesc);
        if (!supported_input(Plane::Exposure, exposureDesc.Format))
            exposure = nullptr;
    }

    FrameParams p;
    p.jitterX = get_float_or(parameters, "Jitter.Offset.X", 0.0f);
    p.jitterY = get_float_or(parameters, "Jitter.Offset.Y", 0.0f);
    p.mvScaleX = get_float_or(parameters, "MV.Scale.X", 1.0f);
    p.mvScaleY = get_float_or(parameters, "MV.Scale.Y", 1.0f);
    p.sharpness = get_float_or(parameters, "Sharpness", 0.0f);
    p.preExposure = get_float_or(parameters, "DLSS.Pre.Exposure", 1.0f);
    p.exposureScale = get_float_or(parameters, "DLSS.Exposure.Scale", 1.0f);
    p.frameTime = get_float_or(parameters, "FrameTimeDeltaInMsec", 16.6f);
    p.reset = get_int_or(parameters, "Reset", 0);
    p.invertX = get_int_or(parameters, "DLSS.Indicator.Invert.X.Axis", 0);
    p.invertY = get_int_or(parameters, "DLSS.Indicator.Invert.Y.Axis", 0);
    p.renderWidth = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Width", 0);
    p.renderHeight = get_uint_or(parameters, "DLSS.Render.Subrect.Dimensions.Height", 0);
    if (p.renderWidth == 0 || p.renderHeight == 0)
    {
        p.renderWidth = feature->width;
        p.renderHeight = feature->height;
    }
    p.colorBaseX = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.X", 0);
    p.colorBaseY = get_uint_or(parameters, "DLSS.Input.Color.Subrect.Base.Y", 0);
    p.depthBaseX = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.X", 0);
    p.depthBaseY = get_uint_or(parameters, "DLSS.Input.Depth.Subrect.Base.Y", 0);
    p.mvBaseX = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.X", 0);
    p.mvBaseY = get_uint_or(parameters, "DLSS.Input.MV.Subrect.Base.Y", 0);
    p.outputBaseX = get_uint_or(parameters, "DLSS.Output.Subrect.Base.X", 0);
    p.outputBaseY = get_uint_or(parameters, "DLSS.Output.Subrect.Base.Y", 0);
    p.hasExposure = exposure != nullptr;

    const int slotIndex = static_cast<int>(frame % kSlots);
    InputSlot& slot = feature->inputs[slotIndex];
    // Throttle the game to DLSS speed when the pipeline still owns this slot
    // or already holds the maximum number of frames.
    static const int maxInFlight = static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) > 0
                                       ? static_cast<int>(env_uint("D4R_SHIM_MAX_IN_FLIGHT", 3)) : 3;
    // Split frames reuse frame N's result slot for frame N + kOutputSlots.
    const int cap = feature->split && maxInFlight > kOutputSlots - 1 ? kOutputSlots - 1 : maxInFlight;
    const auto waitStart = std::chrono::steady_clock::now();
    while (slot.busy.load() || feature->inFlight.load() >= cap)
    {
        if (std::chrono::steady_clock::now() - waitStart > std::chrono::seconds(2))
        {
            logf("frame %u: input slot %d busy or %d frames in flight; skipping evaluation", frame, slotIndex, feature->inFlight.load());
            return NGX_SUCCESS;
        }
        std::this_thread::sleep_for(std::chrono::microseconds(200));
    }
    if (timing.enabled)
        timing.slotWait = profile_ms(waitStart, ProfileClock::now());

    // VRAM interop when every resource qualifies (decided once per feature).
    ID3D12Resource* const inputs[4] = {color, depth, motion, exposure};
    const Plane planes[4] = {Plane::Color, Plane::Depth, Plane::Motion, Plane::Exposure};
    const int inputCount = exposure != nullptr ? 4 : 3;
    VramCopy vramInputs[4], vramOutput;
    if ((!feature->vramDecided || feature->vram) && vram_interop_available())
    {
        p.vram = describe_vram_copy(output, Plane::Color, vramOutput, true);
        for (int index = 0; index < inputCount && p.vram; ++index)
            p.vram = describe_vram_copy(inputs[index], planes[index], vramInputs[index]);
    }
    if (!feature->vramDecided)
    {
        feature->vramDecided = true;
        feature->vram = p.vram;
        feature->split = p.vram && env_uint("D4R_SHIM_SPLIT_FRAME", 0) != 0 && g_vk.split != nullptr &&
                         create_split_semaphore(*feature);
        if (feature->split && env_uint("D4R_SHIM_GPU_WAIT", 0) != 0 && g.cu.streamWaitValue32 != nullptr &&
            g.cu.writeValue32 != nullptr && g_vk.fill != nullptr)
            feature->gpuWait = ensure_vram_buffer(*feature, feature->gpuMarker, 256) &&
                               g.worker.call([feature] { return g.cu.writeValue32(feature->gpuMarker.device, 0); }) == 0;
        feature->linearInputs = feature->gpuWait && env_uint("D4R_SHIM_LINEAR_INPUTS", 0) != 0 &&
                                g.cu.registerLinearTexture != nullptr;
        if (feature->linearInputs)
            logf("linear inputs: NGX samples the interop buffers directly");
        logf("VRAM interop %s for this feature%s%s", p.vram ? "on" : "off",
             feature->split ? ", presenting each frame's own result (split frames)" : "",
             feature->gpuWait ? ", DLSS queued behind a GPU-side wait for the inputs" : "");
    }
    else if (feature->vram && !p.vram)
    {
        logf("frame %u: resources no longer qualify for VRAM interop; staging through host memory from now on", frame);
        drain_pipeline();
        {
            std::lock_guard<std::mutex> lock(feature->splitMutex);
            feature->split = false;
        }
        feature->vram = false;
        feature->gpuWait = false;
        feature->linearInputs = false;
        feature->latestOutput = -1;
        feature->outputFormat = DXGI_FORMAT_UNKNOWN; // recreates the output staging below
    }
    const bool verify = p.vram && env_uint("D4R_SHIM_VRAM_VERIFY", 0) != 0;
    if (p.vram)
    {
        for (int index = 0; index < inputCount; ++index)
        {
            HostPlane& geometry = slot.host[index]; // geometry only; the bytes stay in VRAM
            geometry.width = vramInputs[index].width;
            geometry.height = vramInputs[index].height;
            geometry.rowBytes = canonical_texel_bytes(planes[index]) * geometry.width;
            if (feature->linearInputs)
                geometry.rowBytes = (geometry.rowBytes + 255) & ~static_cast<size_t>(255); // texture pitch alignment
            if (!ensure_vram_buffer(*feature, slot.vram[index], geometry.size()))
                return NGX_FAIL_PLATFORM_ERROR;
        }
        if (vramInputs[0].convert && !ensure_conversion_image(*feature, feature->colorConversion,
                                                               vramInputs[0].width, vramInputs[0].height,
                                                               VK_FORMAT_R16G16B16A16_SFLOAT))
            return NGX_FAIL_PLATFORM_ERROR;
        if (inputCount == 4 && vramInputs[3].convert &&
            !ensure_conversion_image(*feature, feature->exposureConversion,
                                     vramInputs[3].width, vramInputs[3].height, VK_FORMAT_R32_SFLOAT))
            return NGX_FAIL_PLATFORM_ERROR;
        if (vramOutput.convert && !ensure_conversion_image(*feature, feature->outputConversion,
                                                             vramOutput.width, vramOutput.height,
                                                             VK_FORMAT_R16G16B16A16_SFLOAT))
            return NGX_FAIL_PLATFORM_ERROR;
    }
    if ((!p.vram || verify) &&
        (!ensure_staging(slot.color, color, D3D12_HEAP_TYPE_READBACK) ||
         !ensure_staging(slot.depth, depth, D3D12_HEAP_TYPE_READBACK) ||
         !ensure_staging(slot.motion, motion, D3D12_HEAP_TYPE_READBACK) ||
         (exposure != nullptr && !ensure_staging(slot.exposure, exposure, D3D12_HEAP_TYPE_READBACK))))
        return NGX_FAIL_PLATFORM_ERROR;
    if (outputDesc.Format != feature->outputFormat || outputDesc.Width != feature->outputWidth ||
        outputDesc.Height != feature->outputHeight)
    {
        // A new output size invalidates earlier results; let queued work finish
        // before the output buffers go away.
        drain_pipeline();
        feature->latestOutput = -1;
        feature->outputFormat = outputDesc.Format;
        feature->outputWidth = static_cast<UINT>(outputDesc.Width);
        feature->outputHeight = outputDesc.Height;
        for (OutputSlot& outputSlot : feature->outputs)
        {
            outputSlot.staging.release();
            if (p.vram ? !ensure_vram_buffer(*feature, outputSlot.vram, static_cast<size_t>(outputDesc.Width) * 8 *
                                                                            outputDesc.Height)
                       : !ensure_staging(outputSlot.staging, output, D3D12_HEAP_TYPE_UPLOAD))
                return NGX_FAIL_PLATFORM_ERROR;
        }
    }

    const auto inputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_INPUT_STATE", 0x40));
    const auto depthState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_DEPTH_STATE", 0x40));
    const auto outputState = static_cast<D3D12_RESOURCE_STATES>(env_uint("D4R_SHIM_OUTPUT_STATE", 0x8));
    const auto inputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
    if (p.vram)
    {
        const D3D12_RESOURCE_STATES states[4] = {inputState, depthState, inputState, inputState};
        for (int index = 0; index < inputCount; ++index)
            transition(list, inputs[index], states[index], D3D12_RESOURCE_STATE_COPY_SOURCE);
        const bool recorded = record_vram_inputs(*feature, list, slot, vramInputs, inputCount, frame);
        for (int index = 0; index < inputCount; ++index)
            transition(list, inputs[index], D3D12_RESOURCE_STATE_COPY_SOURCE, states[index]);
        if (!recorded)
        {
            logf("frame %u: BeginVkCommandBufferInterop failed", frame);
            return NGX_FAIL_PLATFORM_ERROR;
        }
    }
    if (!p.vram || verify)
    {
        copy_to_staging(list, color, slot.color, inputState);
        copy_to_staging(list, depth, slot.depth, depthState);
        copy_to_staging(list, motion, slot.motion, inputState);
        if (exposure != nullptr)
            copy_to_staging(list, exposure, slot.exposure, inputState);
    }
    if (timing.enabled)
        timing.inputRecord = profile_ms(inputRecordStart, ProfileClock::now());

    // Frame marker, written once the copies above have completed.
    ID3D12GraphicsCommandList2* list2 = nullptr;
    if (FAILED(list->QueryInterface(__uuidof(ID3D12GraphicsCommandList2), reinterpret_cast<void**>(&list2))))
    {
        logf("ID3D12GraphicsCommandList2 unavailable; cannot place frame marker");
        return NGX_FAIL_PLATFORM_ERROR;
    }
    D3D12_WRITEBUFFERIMMEDIATE_PARAMETER marker = {feature->marker->GetGPUVirtualAddress(), frame};
    D3D12_WRITEBUFFERIMMEDIATE_MODE mode = D3D12_WRITEBUFFERIMMEDIATE_MODE_MARKER_OUT;
    list2->WriteBufferImmediate(1, &marker, &mode);
    list2->Release();

    // Present the most recent finished DLSS result.
    int latest;
    {
        std::lock_guard<std::mutex> lock(feature->outputMutex);
        latest = feature->latestOutput.load();
        if (latest >= 0)
        {
            feature->outputs[latest].lastReadFrame = frame;
            if (timing.enabled)
                timing.presentedFrame = feature->outputs[latest].producedFrame.load();
        }
    }
    const auto outputRecordStart = timing.enabled ? ProfileClock::now() : ProfileClock::time_point{};
    if (p.vram && feature->split)
        p.split = SUCCEEDED(g_vk.split->SplitCommandListForExternalWait(
            list, reinterpret_cast<UINT64>(feature->splitSemaphore), frame));
    if (p.split)
    {
        // Runs in the second half of the list, once frame's result is in place.
        const int target = static_cast<int>(frame % kOutputSlots);
        transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
        if (!record_vram_output(list, feature->outputs[target].vram, vramOutput, feature->outputConversion))
            logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
        transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
        if (timing.enabled)
            timing.presentedFrame = frame;
    }
    else if (latest >= 0 && p.vram)
    {
        transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
        if (!record_vram_output(list, feature->outputs[latest].vram, vramOutput, feature->outputConversion))
            logf("frame %u: BeginVkCommandBufferInterop failed for the output", frame);
        transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
    }
    else if (latest >= 0)
    {
        OutputSlot& outputSlot = feature->outputs[latest];
        transition(list, output, outputState, D3D12_RESOURCE_STATE_COPY_DEST);
        D3D12_TEXTURE_COPY_LOCATION destination = {};
        destination.pResource = output;
        destination.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
        destination.SubresourceIndex = 0;
        D3D12_TEXTURE_COPY_LOCATION source = {};
        source.pResource = outputSlot.staging.buffer;
        source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
        source.PlacedFootprint = outputSlot.staging.layout;
        list->CopyTextureRegion(&destination, 0, 0, 0, &source, nullptr);
        transition(list, output, D3D12_RESOURCE_STATE_COPY_DEST, outputState);
    }
    if (timing.enabled)
    {
        timing.outputRecord = profile_ms(outputRecordStart, ProfileClock::now());
        timing.queueDepth = g.prep.pending() + g.worker.pending() + g.finish.pending() + 1;
        timing.queuedAt = ProfileClock::now();
        timing.gameCall = profile_ms(timing.gameStart, timing.queuedAt);
    }

    slot.busy = true;
    feature->inFlight.fetch_add(1);
    g.prep.post([feature, slotIndex, frame, p, timing] { prepare_inputs(feature, slotIndex, frame, p, timing); });
    if (frame <= 3 || frame % 600 == 0 || GetEnvironmentVariableA("D4R_SHIM_TRACE_FRAMES", nullptr, 0) != 0)
        logf("evaluate frame %u queued (slot %d, showing result %d, jitter %.6f,%.6f, reset %d, "
             "mvScale %.3f,%.3f, preExposure %.3f, exposureScale %.3f, frameTime %.3f, sharpness %.3f)",
             frame, slotIndex, latest, p.jitterX, p.jitterY, p.reset, p.mvScaleX, p.mvScaleY,
             p.preExposure, p.exposureScale, p.frameTime, p.sharpness);
    return NGX_SUCCESS;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
                                                     void* parameters, void*)
{
    return evaluate(list, handle, parameters);
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_EvaluateFeature_C(ID3D12GraphicsCommandList* list, const NgxHandle* handle,
                                                       void* parameters, void*)
{
    return evaluate(list, handle, parameters);
}

static void release_feature(Feature* feature)
{
    // Wait for outstanding evaluations, then free CUDA resources on the worker.
    drain_pipeline();
    g.worker.call([feature] {
        if (feature->cudaHandle != nullptr)
            g.ngx.releaseFeature(feature->cudaHandle);
        if (feature->cudaParams != nullptr)
            g.ngx.destroyParameters(feature->cudaParams);
        for (InputSlot& slot : feature->inputs)
            for (CudaObject& texture : slot.linearTexture)
                if (texture != 0)
                    g.cu.texObjectDestroy(texture), texture = 0;
        for (CudaImage* image : {&feature->color, &feature->depth, &feature->motion, &feature->exposure})
        {
            if (image->object != 0)
                g.cu.texObjectDestroy(image->object);
            if (image->array != nullptr)
                g.cu.arrayDestroy(image->array);
        }
        if (feature->output.object != 0)
            g.cu.surfObjectDestroy(feature->output.object);
        if (feature->output.array != nullptr)
            g.cu.arrayDestroy(feature->output.array);
        if (feature->profileStart != nullptr)
            g.cu.eventDestroy(feature->profileStart);
        if (feature->profileEnd != nullptr)
            g.cu.eventDestroy(feature->profileEnd);
        if (feature->scratch != 0)
            g.cu.memFree(feature->scratch);
        for (CudaDevicePtr buffer : feature->planeLinear)
            if (buffer != 0)
                g.cu.memFree(buffer);
        if (feature->outputLinear != 0)
            g.cu.memFree(feature->outputLinear);
        for (InputSlot& slot : feature->inputs)
            for (HostPlane& host : slot.host)
                release_host(host);
        for (HostPlane& host : feature->outputHost)
            release_host(host);
        return 0;
    });
    for (InputSlot& slot : feature->inputs)
        for (Staging* staging : {&slot.color, &slot.depth, &slot.motion, &slot.exposure})
            staging->release();
    for (OutputSlot& slot : feature->outputs)
        slot.staging.release();
    if (g_vk.ready && feature->vramDecided)
    {
        // Queued command lists may still copy from or wait on these; the last
        // marker lands just before the last output copy.
        const auto start = std::chrono::steady_clock::now();
        while (feature->markerValue != nullptr &&
               static_cast<int32_t>(*feature->markerValue - feature->frame) < 0 &&
               std::chrono::steady_clock::now() - start < std::chrono::seconds(2))
            std::this_thread::sleep_for(std::chrono::milliseconds(1));
        std::this_thread::sleep_for(std::chrono::milliseconds(50));
        release_vram(*feature);
    }
    if (feature->marker != nullptr)
        feature->marker->Release();
    delete feature;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_ReleaseFeature(NgxHandle* handle)
{
    Feature* feature = nullptr;
    {
        std::lock_guard<std::mutex> lock(g_featuresMutex);
        for (auto it = g_features.begin(); it != g_features.end(); ++it)
            if (&(*it)->handle == handle)
            {
                feature = *it;
                g_features.erase(it);
                break;
            }
    }
    if (feature == nullptr)
        return NGX_FAIL_INVALID_PARAMETER;
    logf("NVSDK_NGX_D3D12_ReleaseFeature(id=%u) after %u completed frames", feature->handle.Id,
         feature->completedFrames.load());
    release_feature(feature);
    return NGX_SUCCESS;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown()
{
    logf("NVSDK_NGX_D3D12_Shutdown");
    std::vector<Feature*> features;
    {
        std::lock_guard<std::mutex> lock(g_featuresMutex);
        features.swap(g_features);
    }
    for (Feature* feature : features)
        release_feature(feature);
    std::lock_guard<std::mutex> lock(g.mutex);
    if (g.ngxInitialized)
    {
        drain_pipeline();
        g.worker.call([] { return g.ngx.shutdown(); });
    }
    g.ngxInitialized = false;
    if (g.device != nullptr)
    {
        g.device->Release();
        g.device = nullptr;
    }
    return NGX_SUCCESS;
}

D4R_EXPORT NgxResult NVSDK_NGX_D3D12_Shutdown1(ID3D12Device*)
{
    return NVSDK_NGX_D3D12_Shutdown();
}

D4R_EXPORT NgxResult NVSDK_NGX_UpdateFeature(const void*, unsigned int)
{
    return NGX_SUCCESS;
}

BOOL WINAPI DllMain(HINSTANCE instance, DWORD reason, LPVOID)
{
    if (reason == DLL_PROCESS_ATTACH)
    {
        g_selfModule = instance;
        DisableThreadLibraryCalls(instance);
    }
    return TRUE;
}