Measure Windows submission costs and validate faster private K output stores

tools/windows/build_native_texture.py

@@ -0,0 +1,90 @@
+"""Build a private gfx1201 texture override from the user's local DLSS DLL.
+
+The resulting PTX/code object contains NVIDIA code and must remain local.
+No Wine, shell script, GPU, CUDA SDK or system installation is required.
+"""
+import argparse
+import hashlib
+import json
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
+    parser.add_argument('--hip-root', type=pathlib.Path, required=True)
+    parser.add_argument('--zluda-root', type=pathlib.Path, required=True)
+    parser.add_argument('--kernel', required=True)
+    parser.add_argument('--output-directory', type=pathlib.Path, required=True)
+    args = parser.parse_args()
+    if not re.fullmatch(r'hiluma_engine_output_depth(?:inv|reg)_mv(?:hi|lo)_(?:hdr|ldr)(?:_max)?_v2_rel', args.kernel):
+        parser.error('this initial validated build recipe supports K v2 output variants only')
+    repo = pathlib.Path(__file__).resolve().parents[2]
+    output = args.output_directory.resolve()
+    output.mkdir(parents=True, exist_ok=True)
+    work = output / ('work-' + args.kernel)
+    work.mkdir(exist_ok=True)
+    hip = args.hip_root.resolve()
+    zluda = args.zluda_root.resolve()
+    tools = {name: hip / 'bin' / (name + '.exe') for name in ('clang++', 'llvm-dis', 'llvm-as')}
+    emitter = zluda / 'd4r_emit.exe'
+    for path in [args.dlss_dll, emitter, *tools.values()]:
+        if not path.is_file():
+            parser.error(f'missing dependency: {path}')
+    env = os.environ.copy()
+    env.update(D4R_PREFER_ACCURACY='1', D4R_ZLUDA_WMMA='1', D4R_ZLUDA_WMMA_FP8='1',
+               D4R_ZLUDA_WMMA_FP8_NATIVE='0', D4R_ZLUDA_WMMA_F16_REFERENCE='1',
+               D4R_ZLUDA_IMPLICIT_MAX_BLOCK='256')
+    for name in ('D4R_ZLUDA_IGNORE_DENORMAL', 'D4R_ZLUDA_FAST_MATH', 'D4R_ZLUDA_WMMA_F32ACC',
+                 'D4R_ZLUDA_WAVE64', 'D4R_ZLUDA_PROFILE'):
+        env.pop(name, None)
+    env['PATH'] = str(zluda) + os.pathsep + str(hip / 'bin') + os.pathsep + env['PATH']
+
+    def run(label, command):
+        with (work / (label + '.log')).open('wb') as log:
+            result = subprocess.run([str(arg) for arg in command], env=env, stdout=log, stderr=subprocess.STDOUT)
+        if result.returncode:
+            print((work / (label + '.log')).read_text(errors='replace')[-8000:], file=sys.stderr)
+            raise RuntimeError(f'{label} failed ({result.returncode}); see {work}')
+
+    ptx = work / 'ptx'
+    run('extract', [sys.executable, repo / 'kernels/tools/extract_dlss_ptx.py', args.dlss_dll, ptx])
+    env['D4R_DLSS_PTX_DIR'] = str(ptx)
+    edited = work / (args.kernel + '.ptx')
+    run('make-ptx', [sys.executable, repo / 'kernels/tex/make_ptx.py', args.kernel, edited])
+    raw, ir, extra = work / 'raw.bc', work / 'raw.ll', work / 'extra.bc'
+    run('compile-helper', [tools['clang++'], '-x', 'hip', '-std=c++20', '-nogpuinc', '-nogpulib', '-O3',
+                          '-mno-wavefrontsize64', '--offload-device-only', '--offload-arch=gfx1201',
+                          '-fgpu-rdc', '-emit-llvm', '-c', '-Xclang', '-fdenormal-fp-math=dynamic',
+                          '-DD4R_ACCURACY', '-DD4R_KERNEL_NAME=' + args.kernel,
+                          '-o', raw, repo / 'kernels/tex/sust_only.hip'])
+    run('disassemble', [tools['llvm-dis'], raw, '-o', ir])
+    lines = []
+    for line in ir.read_text().splitlines():
+        if re.search(r'@llvm.used|wchar_size|llvm.module.flags|__hip_cuid', line):
+            continue
+        line = line.replace('optnone', '').replace('"target-cpu"="gfx1201"', '')
+        lines.append(re.sub(r'"target-features"="[^"]+"', '', line))
+    ir.write_text('\n'.join(lines) + '\n')
+    run('assemble', [tools['llvm-as'], ir, '-o', extra])
+    env['D4R_ZLUDA_EXTRA_BC'] = str(extra)
+    run('emit', [emitter, edited, work / 'emitted', 'gfx1201'])
+    shutil.copyfile(work / 'emitted/module.hsaco', output / (args.kernel + '.hsaco'))
+    run('manifest', [sys.executable, repo / 'kernels/tools/kernel_manifest.py', output, args.dlss_dll])
+    metadata = dict(architecture='gfx1201', accuracy=True, nativeFP8=False,
+                    privateNvidiaDerivedCode=True, validation='not yet validated; do not install before comparison',
+                    dlssSha256=hashlib.sha256(args.dlss_dll.read_bytes()).hexdigest(),
+                    zludaBuild=json.loads((zluda / 'build-info.json').read_text(encoding='utf-8-sig')),
+                    compiler=subprocess.check_output([str(tools['clang++']), '--version'], env=env).decode(errors='replace'),
+                    objects={p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in output.glob('*.hsaco')})
+    (output / 'build-info.json').write_text(json.dumps(metadata, indent=2) + '\n')
+    print(f'Built PRIVATE gfx1201 output override: {output}; validation required before use')
+
+
+if __name__ == '__main__':
+    main()

tools/windows/d3d12_command_hooks.cpp

@@ -6,6 +6,7 @@
 #include <mutex>
 #include <unordered_map>
 #include <vector>
+#include <chrono>
 
 namespace d4r::win::commands {
 namespace {
@@ -13,6 +14,9 @@ thread_local bool internal = false;
 std::mutex map_mutex, installation_mutex, error_mutex;
 std::string last_error;
 bool installed = false;
+using Clock = std::chrono::steady_clock;
+bool profile_commands() { static const bool enabled = std::getenv("D4R_PROFILE_STAGES") != nullptr; return enabled; }
+double milliseconds(Clock::time_point start) { return std::chrono::duration<double, std::milli>(Clock::now() - start).count(); }
 struct Event { UINT metadata; std::vector<uint8_t> data; };
 using State = std::function<void(ID3D12GraphicsCommandList*)>;
 struct TrackedResource {
@@ -385,6 +389,8 @@ void complete(ID3D12CommandQueue* queue) {
 }
 void STDMETHODCALLTYPE hook_execute(ID3D12CommandQueue* queue, UINT count, ID3D12CommandList* const* lists) {
     if (internal) { original_execute(queue, count, lists); return; }
+    const auto submitStart = Clock::now();
+    double publishMs = 0, boundaryMs = 0, drainMs = 0;
     std::vector<std::shared_ptr<Recording>> owners;
     std::vector<ID3D12CommandList*> pending;
     std::vector<ResourceStates> pending_states;
@@ -394,7 +400,9 @@ void STDMETHODCALLTYPE hook_execute(ID3D12CommandQueue* queue, UINT count, ID3D1
             original_execute(queue, UINT(pending.size()), pending.data());
             // Publish in the actual submitted order, never while recording.
             // The callback's external fence waits for these producers on GPU.
+            const auto publishStart = Clock::now();
             for (const auto& states : pending_states) publish_states(states);
+            publishMs += milliseconds(publishStart);
             pending.clear(); pending_states.clear();
         }
     };
@@ -416,13 +424,19 @@ void STDMETHODCALLTYPE hook_execute(ID3D12CommandQueue* queue, UINT count, ID3D1
             owners.push_back(recording); boundaries = true;
             for (const auto& segment : recording->segments) {
                 pending.push_back(segment.list); pending_states.push_back(segment.resource_states); flush();
+                const auto boundaryStart = Clock::now();
                 segment.boundary(queue);
+                boundaryMs += milliseconds(boundaryStart);
             }
             pending.push_back(recording->current.Get());
             pending_states.push_back(recording->resource_states);
         }
         flush();
-        if (boundaries) complete(queue);
+        if (boundaries) {
+            const auto drainStart = Clock::now(); complete(queue); drainMs = milliseconds(drainStart);
+            if (profile_commands()) std::printf("D4R_COMMAND_SUBMIT total_ms=%.3f boundary_ms=%.3f publish_ms=%.3f suffix_drain_ms=%.3f\n",
+                milliseconds(submitStart), boundaryMs, publishMs, drainMs);
+        }
     } catch (const std::exception& failure) {
         { std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
         std::fprintf(stderr, "D4R_D3D12_SUBMIT_FAILURE %s\n", failure.what());
@@ -510,6 +524,7 @@ void install(ID3D12Device* device) {
     }
 }
 void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback) {
+    const auto recordStart = Clock::now();
     if (!installed || !list || !callback) throw std::runtime_error("D3D12 command backend is not initialized");
     if (!error().empty()) throw std::runtime_error(error());
     if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) throw std::runtime_error("DLSS requires a direct command list");
@@ -527,7 +542,9 @@ void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback) {
     ComPtr<ID3D12CommandAllocator> nextAllocator; ComPtr<ID3D12GraphicsCommandList> next;
     dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(nextAllocator.GetAddressOf())), "Suffix allocator");
     dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, nextAllocator.Get(), nullptr, IID_PPV_ARGS(next.GetAddressOf())), "Suffix command list");
+    const auto replayStart = Clock::now();
     for (auto& state : recording.state) state(next.Get());
+    const auto replayMs = milliseconds(replayStart);
     recording.segments.reserve(recording.segments.size() + 1);
     for (const auto& event : recording.events) original_BeginEvent(next.Get(), event.metadata, event.data.data(), UINT(event.data.size()));
     for (size_t n = 0; n < recording.events.size(); ++n) original_EndEvent(segment.list);
@@ -537,5 +554,14 @@ void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback) {
     recording.current_allocator = std::move(nextAllocator);
     recording.current = std::move(next);
     std::printf("D4R_D3D12_RECORD boundary=%zu state_calls=%zu frame_age=0\n", recording.segments.size(), recording.state.size());
+    if (profile_commands()) {
+        // This is the interval between NGX recording calls on this thread,
+        // not a Present/FPS measurement or an extra queued frame.
+        thread_local Clock::time_point previous{};
+        const double interval = previous == Clock::time_point{} ? 0 : std::chrono::duration<double, std::milli>(recordStart - previous).count();
+        previous = recordStart;
+        std::printf("D4R_COMMAND_RECORD total_ms=%.3f replay_ms=%.3f state_calls=%zu interval_ms=%.3f\n",
+            milliseconds(recordStart), replayMs, recording.state.size(), interval);
+    }
 }
 }

tools/windows/d3d12_dlss_probe.cpp

@@ -73,7 +73,9 @@ int main(int argc, char** argv) {
         ComPtr<ID3D12GraphicsCommandList> list;
         dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator");
         dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "CreateCommandList");
-        constexpr unsigned width = 256, height = 144, outWidth = 512, outHeight = 288;
+        const unsigned outWidth = args.output_width, outHeight = args.output_height;
+        const unsigned width = outWidth / 2, height = outHeight / 2;
+        std::printf("D3D12_NGX_DIMENSIONS input=%ux%u output=%ux%u\n", width, height, outWidth, outHeight);
         struct Texture { ComPtr<ID3D12Resource> image, upload; D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint{}; UINT64 bytes; } textures[5];
         for (unsigned i = 0; i < 5; ++i) {
             auto& texture = textures[i];
@@ -230,7 +232,8 @@ int main(int argc, char** argv) {
         for (const auto& pair : {std::make_pair("Width", width), std::make_pair("Height", height),
              std::make_pair("OutWidth", outWidth), std::make_pair("OutHeight", outHeight),
              std::make_pair("CreationNodeMask", 1u), std::make_pair("VisibilityNodeMask", 1u)}) d4r_ngx_set_uint(parameters, pair.first, pair.second);
-        d4r_ngx_set_int(parameters, "PerfQualityValue", 2); d4r_ngx_set_int(parameters, "DLSS.Feature.Create.Flags", 0);
+        d4r_ngx_set_int(parameters, "PerfQualityValue", 2); d4r_ngx_set_int(parameters, "DLSS.Feature.Create.Flags", args.ngx_create_flags);
+        std::printf("D3D12_NGX_CREATE_FLAGS value=%u\n", args.ngx_create_flags);
         for (const char* name : {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
              "DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"}) d4r_ngx_set_uint(parameters, name, args.preset);
         void* handle = nullptr;

tools/windows/debug_launcher.cpp

@@ -2,6 +2,7 @@
 #include "diagnostic.h"
 #include <map>
 #include <fstream>
+#include <tlhelp32.h>
 
 namespace {
 std::wstring quote(const std::wstring& value) {
@@ -31,14 +32,16 @@ uint32_t module_size(HANDLE process, uintptr_t base) {
 }
 int main(int argc, char** argv) {
     try {
-        if (argc < 5 || std::strcmp(argv[1], "--output-directory") || std::strcmp(argv[3], "--"))
-            throw std::runtime_error("Usage: d4r_debug_launcher --output-directory PATH -- EXE [arguments]");
+        const bool debug = argc < 4 || std::strcmp(argv[3], "--no-debug");
+        const int separator = debug ? 3 : 4;
+        if (argc <= separator + 1 || std::strcmp(argv[1], "--output-directory") || std::strcmp(argv[separator], "--"))
+            throw std::runtime_error("Usage: d4r_debug_launcher --output-directory PATH [--no-debug] -- EXE [arguments]");
         const std::filesystem::path directory(d4r::diag::wide(argv[2]));
         std::filesystem::create_directories(directory);
         std::ofstream log(directory / L"debugger.log", std::ios::app);
-        const auto executable = d4r::diag::wide(argv[4]);
+        const auto executable = d4r::diag::wide(argv[separator + 1]);
         std::wstring command;
-        for (int i = 4; i < argc; ++i) { if (i > 4) command += L' '; command += quote(d4r::diag::wide(argv[i])); }
+        for (int i = separator + 1; i < argc; ++i) { if (i > separator + 1) command += L' '; command += quote(d4r::diag::wide(argv[i])); }
         STARTUPINFOW startup{sizeof(startup)}; PROCESS_INFORMATION child{};
         startup.dwFlags = STARTF_USESTDHANDLES;
         startup.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
@@ -50,12 +53,39 @@ int main(int argc, char** argv) {
         if (!job || !SetInformationJobObject(job, JobObjectExtendedLimitInformation, &limits, sizeof(limits)))
             throw std::runtime_error("Cannot create diagnostic process lifetime job");
         if (!CreateProcessW(executable.c_str(), command.data(), nullptr, nullptr, TRUE,
-            DEBUG_ONLY_THIS_PROCESS | CREATE_NO_WINDOW, nullptr, nullptr, &startup, &child))
+            (debug ? DEBUG_ONLY_THIS_PROCESS : 0) | CREATE_SUSPENDED | CREATE_NO_WINDOW, nullptr, nullptr, &startup, &child))
             throw std::runtime_error("CreateProcess debugger error=" + std::to_string(GetLastError()));
         if (!AssignProcessToJobObject(job, child.hProcess)) {
             TerminateProcess(child.hProcess, 1); CloseHandle(job);
             throw std::runtime_error("AssignProcessToJobObject debugger failed");
         }
+        log << "LAUNCH pid=" << child.dwProcessId << " attached_debugger=" << debug << std::endl;
+        if (ResumeThread(child.hThread) == DWORD(-1)) {
+            CloseHandle(job); throw std::runtime_error("Cannot resume diagnostic child");
+        }
+        if (!debug) {
+            // Retain bounded process ownership and raw inherited output without
+            // debugger stops on DLL/thread/exception/debug-string events.
+            std::map<uintptr_t, Module> inventory;
+            while (WaitForSingleObject(child.hProcess, 1000) == WAIT_TIMEOUT) {
+                HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, child.dwProcessId);
+                if (snapshot == INVALID_HANDLE_VALUE) continue;
+                MODULEENTRY32W module{sizeof(module)};
+                if (Module32FirstW(snapshot, &module)) do {
+                    const auto base = reinterpret_cast<uintptr_t>(module.modBaseAddr);
+                    if (inventory.count(base)) continue;
+                    inventory[base] = {d4r::diag::utf8(module.szExePath), module.modBaseSize};
+                    log << "LOAD base=0x" << std::hex << base << " size=0x" << module.modBaseSize
+                        << " path=" << inventory[base].path << '\n';
+                } while (Module32NextW(snapshot, &module));
+                CloseHandle(snapshot); log.flush();
+            }
+            DWORD result = 1;
+            GetExitCodeProcess(child.hProcess, &result);
+            log << "EXIT code=0x" << std::hex << result << std::endl;
+            CloseHandle(child.hThread); CloseHandle(child.hProcess); CloseHandle(job);
+            return static_cast<int>(result);
+        }
         std::map<uintptr_t, Module> modules;
         bool initialBreakpoint = true, dumped = false; DWORD exitCode = 1;
         auto add_module = [&](void* base, HANDLE file) {
@@ -73,7 +103,17 @@ int main(int argc, char** argv) {
             }
         };
         DEBUG_EVENT event{};
+        uint64_t debugEvents = 0, debugStrings = 0, exceptions = 0;
+        ULONGLONG lastSample = GetTickCount64();
         while (WaitForDebugEvent(&event, INFINITE)) {
+            ++debugEvents;
+            if (event.dwDebugEventCode == OUTPUT_DEBUG_STRING_EVENT) ++debugStrings;
+            if (event.dwDebugEventCode == EXCEPTION_DEBUG_EVENT) ++exceptions;
+            const auto now = GetTickCount64();
+            if (now - lastSample >= 5000) {
+                log << "DEBUG_EVENTS events=" << std::dec << debugEvents << " debug_strings=" << debugStrings << " exceptions=" << exceptions << std::endl;
+                lastSample = now;
+            }
             DWORD disposition = DBG_CONTINUE;
             if (event.dwDebugEventCode == CREATE_PROCESS_DEBUG_EVENT) {
                 add_module(event.u.CreateProcessInfo.lpBaseOfImage, event.u.CreateProcessInfo.hFile);

tools/windows/diagnostic.h

@@ -125,6 +125,8 @@ struct Args {
     std::string ngx_frontend = "d4r";
     std::string ngx_mode = "init";
     unsigned preset = 11;
+    unsigned ngx_create_flags = 0;
+    unsigned output_width = 512, output_height = 288;
     std::string fixture_dir;
     std::string output_dir;
     std::string kernel_name = "enc1";
@@ -159,6 +161,22 @@ struct Args {
                 ngx_frontend = value;
             }
             else if (key == "--ngx-mode") ngx_mode = value;
+            else if (key == "--ngx-create-flags") {
+                size_t end = 0;
+                ngx_create_flags = static_cast<unsigned>(std::stoul(value, &end));
+                if (end != value.size() || (ngx_create_flags != 0 && ngx_create_flags != 11))
+                    throw std::runtime_error("ngx-create-flags must be 0 or 11 (HDR, low-resolution MV, inverted depth)");
+            }
+            else if (key == "--ngx-output-resolution") {
+                const auto separator = value.find('x');
+                if (separator == std::string::npos) throw std::runtime_error("Output resolution must be WIDTHxHEIGHT");
+                size_t wend = 0, hend = 0;
+                output_width = static_cast<unsigned>(std::stoul(value.substr(0, separator), &wend));
+                output_height = static_cast<unsigned>(std::stoul(value.substr(separator + 1), &hend));
+                if (wend != separator || hend != value.size() - separator - 1 || output_width < 128 || output_width > 4096 ||
+                    output_height < 128 || output_height > 2160 || output_width % 2 || output_height % 2)
+                    throw std::runtime_error("Output resolution must be even and within 128x128..4096x2160");
+            }
             else if (key == "--preset") {
                 size_t end = 0;
                 preset = static_cast<unsigned>(std::stoul(value, &end));

tools/windows/profile_report.py

@@ -39,19 +39,27 @@ def metadata(directory):
 def report(directory, metadata_directory=None):
     kernel_metadata = metadata(metadata_directory)
     stages = collections.defaultdict(list)
+    commands = collections.defaultdict(list)
     kernels = collections.defaultdict(list)
     for path in sorted(directory.glob('*.log')):
         text = path.read_text(encoding='utf-8-sig', errors='replace')
         for name, value in re.findall(r'D4R_STAGE name=(\S+) cpu_ms=([0-9.]+)', text):
             stages[name].append(float(value))
+        for kind, fields in re.findall(r'(?m)^D4R_COMMAND_(RECORD|SUBMIT) ([^\r\n]+)', text):
+            for name, value in re.findall(r'(\w+_ms)=([0-9.]+)', fields):
+                number = float(value)
+                if name != 'interval_ms' or number > 0:
+                    commands[kind.lower() + '.' + name].append(number)
         for line in text.splitlines():
             if not line.startswith('D4R_KERNEL_PROFILE '):
                 continue
             fields = dict(re.findall(r'(\w+)=("[^"]+"|\S+)', line))
             kernels[(fields['kernel'].strip('"'), fields['backend'], fields['phase'])].append(fields)
     result = dict(cpuStages={name: stats(values) for name, values in stages.items()},
+                  commandStages={name: stats(values) for name, values in commands.items()},
                   kernels=[], notes=[
                       'CPU stage times include waits and host work; they are not isolated GPU timings.',
+                      'Command record intervals are between NGX recording calls on one thread, not Present/FPS.',
                       'HIP-event profiling synchronizes every sampled launch and changes scheduling.',
                       'Occupancy is an API prediction, not a measured hardware counter.',
                       'Bandwidth and WMMA utilization require additional supported hardware tooling.'])
@@ -83,6 +91,8 @@ def main():
         args.output.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
     for name, row in result['cpuStages'].items():
         print(f"CPU {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
+    for name, row in result['commandStages'].items():
+        print(f"COMMAND {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
     for row in result['kernels']:
         times = row['gpuMs']
         print(f"GPU {row['kernel']} {row['backend']} {row['phase']}: n={times['samples']} mean={times['mean']:.6f} ms total={times['total']:.3f} ms")