wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clear windows branch; preserve previous contents in win-old

Browse Source
This commit is contained in:
ayo committed 2026-10-16 05:20:08 -04:00
1 parent f0c881985b
commit 73994d4634
311 files changed
-57018
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/windows/benchmark_native_replay.py
-104
View File
@@ -1,104 +0,0 @@
"""Compare resident native modules with exact replay and paired batch timings.
Fixtures can contain private weights; this utility never packages them.
"""
import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import sys
from profile_report import report
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def run(args):
args.output_dir.mkdir(parents=True, exist_ok=False)
summary = dict(passed=False, numericalExact=False, fixture=str(args.fixture_dir),
module=str(args.module), candidate=str(args.candidate), probe=str(args.probe),
pairs=args.pairs, batches=args.batches,
timingSources=args.timings, executions=[])
environment = os.environ.copy()
environment['D4R_DIAG_DIR'] = str(args.output_dir)
environment['PATH'] = str(args.hip_root / 'bin') + os.pathsep + environment.get('PATH', '')
try:
summary.update(moduleSha256=sha(args.module), candidateSha256=sha(args.candidate),
probeSha256=sha(args.probe))
def execute(name, module, options=()):
command = [str(args.probe), '--hip-root', str(args.hip_root), '--module', str(module),
'--fixture-dir', str(args.fixture_dir), '--output-dir', str(args.output_dir / name),
'--iterations', '1', *options]
with (args.output_dir / (name + '.stdout.log')).open('wb') as out, \
(args.output_dir / (name + '.stderr.log')).open('wb') as err:
try:
completed = subprocess.run(command, stdout=out, stderr=err, env=environment, timeout=180)
except subprocess.TimeoutExpired:
summary['executions'].append(dict(name=name, command=command, timedOut=True))
raise
summary['executions'].append(dict(name=name, command=command, exitCode=completed.returncode))
if completed.returncode:
raise RuntimeError(f'{name} failed ({completed.returncode}); inspect preserved logs/dump')
return args.output_dir / name
control = execute('control-reference', args.module)
candidate = execute('candidate-reference', args.candidate)
reference = {p.name: sha(p) for p in control.glob('alloc-*.bin')}
if not reference:
raise RuntimeError('Replay produced no allocations')
def check(folder):
actual = {p.name: sha(p) for p in folder.glob('alloc-*.bin')}
if actual != reference:
changed = sorted(set(actual) ^ set(reference) |
{name for name in reference if actual.get(name) != reference[name]})
raise RuntimeError(f'{folder.name}: allocations differ from fresh control: {changed}')
# Stop before timing a candidate that changes the numerical result.
check(candidate)
summary['numericalExact'] = True
summary['allocationSha256'] = reference
for timing in args.timings:
for batch in args.batches:
folder = execute(f'{timing}-{batch}', args.module,
['--benchmark-module', str(args.candidate), '--benchmark-timing', timing,
'--benchmark-batch', str(batch), '--iterations', str(args.pairs)])
check(folder)
measured = report(args.output_dir)
summary['profile'] = measured
expected = len(args.timings) * len(args.batches)
if len(measured['replayPairs']) != expected or any(
row['controlGpuMs']['samples'] != args.pairs for row in measured['replayPairs']):
raise RuntimeError('Incomplete/invalid timing pairs; this run is not accepted for performance comparison')
summary['passed'] = True
for row in measured['replayPairs']:
print(f"PAIRED {row['kernel']} {row['timingSource']} batch={row['launchesPerSample']} "
f"control={row['controlGpuMs']['median']:.6f} candidate={row['candidateGpuMs']['median']:.6f} ms "
f"ratio={row['candidateToControl']['median']:.6f}")
print('PASS NATIVE_BENCHMARK exact_allocations=1 timings_valid=1 '
'isolated_replay=1 game_speedup=unmeasured')
except Exception as error:
summary['error'] = str(error)
print(f'FAIL NATIVE_BENCHMARK {error}', file=sys.stderr)
finally:
(args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
return 0 if summary['passed'] else 5
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
for name in ('probe', 'hip-root', 'module', 'candidate', 'fixture-dir', 'output-dir'):
parser.add_argument('--' + name, required=True, type=lambda s: pathlib.Path(s).resolve())
parser.add_argument('--pairs', type=int, default=32)
parser.add_argument('--batches', type=int, nargs='+', default=[1, 64])
parser.add_argument('--timings', choices=['events', 'dispatch'], nargs='+', default=['events', 'dispatch'])
args = parser.parse_args()
if not 1 <= args.pairs <= 10000 or any(not 1 <= batch <= 256 for batch in args.batches):
parser.error('pairs must be 1..10000 and batches 1..256')
if len(set(args.batches)) != len(args.batches) or len(set(args.timings)) != len(args.timings):
parser.error('duplicate batches/timing sources are not allowed')
raise SystemExit(run(args))
tools/windows/binary16.h
-51
View File
@@ -1,51 +0,0 @@
#pragma once
// Adapted from realdody/d4r ae530b0 (MIT), realdody <[email protected]>.
#include <cstdint>
#include <cstring>
#include <cmath>
#include <limits>
namespace d4r::diag::binary16 {
// MSVC has no _Float16 extension. Convert double<->IEEE binary16 explicitly with
// round-to-nearest-even, matching a native _Float16 cast. This probe compares the
// f16-accumulator MMA against a double sum rounded once to half, so the rounding
// must match IEEE 754 binary16 exactly.
inline uint16_t half_bits(double value) {
uint64_t bits;
std::memcpy(&bits, &value, sizeof(bits));
const uint16_t sign = static_cast<uint16_t>((bits >> 63) << 15);
int32_t exponent = static_cast<int32_t>((bits >> 52) & 0x7ff);
uint64_t mantissa = bits & 0x000fffffffffffffULL;
if (exponent == 0x7ff) return static_cast<uint16_t>(sign | (mantissa ? 0x7e00 : 0x7c00));
if (exponent == 0) return sign; // double subnormal rounds to half zero
int32_t unbiased = exponent - 1023;
if (unbiased > 15) return static_cast<uint16_t>(sign | 0x7c00);
mantissa |= 0x0010000000000000ULL; // implicit leading one (53-bit significand)
const int shift = unbiased >= -14 ? 42 : 28 - unbiased;
uint64_t rounded = shift >= 64 ? 0 : (mantissa >> shift);
if (shift < 64) {
const uint64_t remainder = mantissa & ((1ULL << shift) - 1);
const uint64_t halfway = 1ULL << (shift - 1);
if (remainder > halfway || (remainder == halfway && (rounded & 1))) ++rounded;
}
if (unbiased >= -14) {
if (rounded >= 0x800) { // rounding overflowed into the exponent
rounded = 0x400;
if (++unbiased > 15) return static_cast<uint16_t>(sign | 0x7c00);
}
return static_cast<uint16_t>(sign | ((unbiased + 15) << 10) | static_cast<uint16_t>(rounded & 0x3ff));
}
if (rounded >= 0x400) return static_cast<uint16_t>(sign | 0x0400); // smallest normal half
return static_cast<uint16_t>(sign | static_cast<uint16_t>(rounded));
}
inline double half_value(uint16_t bits) {
const bool negative = (bits & 0x8000u) != 0;
const uint32_t exponent = (bits >> 10) & 0x1fu;
const uint32_t mantissa = bits & 0x3ffu;
double magnitude;
if (exponent == 0) magnitude = std::ldexp(static_cast<double>(mantissa), -24);
else if (exponent == 0x1fu) magnitude = mantissa ? std::numeric_limits<double>::quiet_NaN() : std::numeric_limits<double>::infinity();
else magnitude = std::ldexp(static_cast<double>(mantissa) / 1024.0 + 1.0, static_cast<int>(exponent) - 15);
return negative ? -magnitude : magnitude;
}
}
tools/windows/binary16_probe.cpp
-45
View File
@@ -1,45 +0,0 @@
#include "binary16.h"
#include <cstdio>
#include <initializer_list>
#include <stdexcept>
static void check(double value, uint16_t expected) {
const auto actual = d4r::diag::binary16::half_bits(value);
if (actual != expected) throw std::runtime_error("binary16 rounding mismatch");
}
int main() {
using namespace d4r::diag::binary16;
try {
unsigned encodings = 0, boundaries = 0;
for (unsigned bits = 0; bits < 65536; ++bits) {
if ((bits & 0x7c00) == 0x7c00 && (bits & 0x3ff)) continue;
check(half_value(uint16_t(bits)), uint16_t(bits));
++encodings;
}
for (unsigned bits = 0; bits < 0x7bff; ++bits) {
const double low = half_value(uint16_t(bits)), high = half_value(uint16_t(bits + 1));
const double midpoint = (low + high) * 0.5;
const uint16_t tie = uint16_t(bits + (bits & 1));
check(midpoint, tie); check(-midpoint, uint16_t(tie | 0x8000));
check(std::nextafter(midpoint, low), uint16_t(bits));
check(std::nextafter(midpoint, high), uint16_t(bits + 1));
#if !defined(_MSC_VER)
for (double value : {midpoint, -midpoint, std::nextafter(midpoint, low), std::nextafter(midpoint, high)}) {
const _Float16 native = static_cast<_Float16>(value);
uint16_t expected; std::memcpy(&expected, &native, sizeof(expected));
check(value, expected);
}
#endif
++boundaries;
}
check(65519.0, 0x7bff); check(65520.0, 0x7c00);
check(-65520.0, 0xfc00);
check(std::numeric_limits<double>::denorm_min(), 0);
check(-std::numeric_limits<double>::denorm_min(), 0x8000);
check(std::numeric_limits<double>::quiet_NaN(), 0x7e00);
std::printf("PASS BINARY16 encodings=%u rounding_boundaries=%u\n", encodings, boundaries);
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL BINARY16 %s\n", e.what()); return 5;
}
}
tools/windows/build_native_texture.py
-123
View File
@@ -1,123 +0,0 @@
"""Build a private target-specific texture override from the user's local DLSS DLL.
The resulting PTX/code object contains NVIDIA code and must remain local.
No Wine, shell script, GPU, CUDA SDK or system installation is required.
"""
import argparse
import ast
import hashlib
import json
import os
import pathlib
import re
import shutil
import subprocess
import sys
from gpu_targets import TARGETS
def require_code_object_target(path, target):
with path.open('rb') as file:
header = file.read(64)
if (len(header) != 64 or header[:6] != b'\x7fELF\x02\x01' or
header[18:20] != b'\xe0\x00' or header[48] != TARGETS[target]['elfMachine']):
raise RuntimeError(f'Code object does not target {target}: {path}')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
parser.add_argument('--hip-root', type=pathlib.Path, required=True)
parser.add_argument('--zluda-root', type=pathlib.Path, required=True)
parser.add_argument('--kernel', required=True)
parser.add_argument('--output-directory', type=pathlib.Path, required=True)
parser.add_argument('--gpu-arch', choices=tuple(TARGETS), default='gfx1201')
parser.add_argument('--shader-mode', choices=['cu', 'wgp'], default='cu')
args = parser.parse_args()
if not re.fullmatch(r'hiluma_engine_(?:output_depth(?:inv|reg)_mv(?:hi|lo)_(?:hdr|ldr)(?:_max)?|'
r'input_depth(?:inv|reg)_mv(?:hi|lo)_(?:hdr|ldr))_v2_rel', args.kernel):
parser.error('this private experimental build recipe supports K v2 input/output variants only')
repo = pathlib.Path(__file__).resolve().parents[2]
output = args.output_directory.resolve()
output.mkdir(parents=True, exist_ok=True)
for existing in output.glob('*.hsaco'):
require_code_object_target(existing, args.gpu_arch)
work = output / ('work-' + args.kernel)
work.mkdir(exist_ok=True)
hip = args.hip_root.resolve()
zluda = args.zluda_root.resolve()
tools = {name: hip / 'bin' / (name + '.exe') for name in ('clang++', 'llvm-dis', 'llvm-as')}
emitter = zluda / 'd4r_emit.exe'
for path in [args.dlss_dll, emitter, *tools.values()]:
if not path.is_file():
parser.error(f'missing dependency: {path}')
env = os.environ.copy()
env.update(D4R_PREFER_ACCURACY='1', D4R_ZLUDA_WMMA='1', D4R_ZLUDA_WMMA_FP8='1',
D4R_ZLUDA_WMMA_FP8_NATIVE='0', D4R_ZLUDA_WMMA_F16_REFERENCE='1',
D4R_ZLUDA_WGP='1' if args.shader_mode == 'wgp' else '0',
D4R_ZLUDA_IMPLICIT_MAX_BLOCK='256')
for name in ('D4R_ZLUDA_IGNORE_DENORMAL', 'D4R_ZLUDA_FAST_MATH', 'D4R_ZLUDA_WMMA_F32ACC',
'D4R_ZLUDA_WAVE64', 'D4R_ZLUDA_PROFILE'):
env.pop(name, None)
env['PATH'] = str(zluda) + os.pathsep + str(hip / 'bin') + os.pathsep + env['PATH']
def run(label, command):
with (work / (label + '.log')).open('wb') as log:
result = subprocess.run([str(arg) for arg in command], env=env, stdout=log, stderr=subprocess.STDOUT)
if result.returncode:
print((work / (label + '.log')).read_text(errors='replace')[-8000:], file=sys.stderr)
raise RuntimeError(f'{label} failed ({result.returncode}); see {work}')
ptx = work / 'ptx'
run('extract', [sys.executable, repo / 'kernels/tools/extract_dlss_ptx.py', args.dlss_dll, ptx])
env['D4R_DLSS_PTX_DIR'] = str(ptx)
edited = work / (args.kernel + '.ptx')
run('make-ptx', [sys.executable, repo / 'kernels/tex/make_ptx.py', args.kernel, edited])
rewrite_match = re.search(r'surface stores replaced: (\{[^\n]+\})',
(work / 'make-ptx.log').read_text(errors='replace'))
if not rewrite_match:
raise RuntimeError('PTX recipe did not report its changes')
rewrite_counts = ast.literal_eval(rewrite_match[1])
if not any(rewrite_counts.values()):
raise RuntimeError('PTX recipe made no changes; refuse a no-op override')
raw, ir, extra = work / 'raw.bc', work / 'raw.ll', work / 'extra.bc'
run('compile-helper', [tools['clang++'], '-x', 'hip', '-std=c++20', '-nogpuinc', '-nogpulib', '-O3',
'-mno-wavefrontsize64', '--offload-device-only', '--offload-arch=' + args.gpu_arch,
'-fgpu-rdc', '-emit-llvm', '-c', '-Xclang', '-fdenormal-fp-math=dynamic',
'-DD4R_ACCURACY', '-DD4R_KERNEL_NAME=' + args.kernel,
'-o', raw, repo / 'kernels/tex/sust_only.hip'])
run('disassemble', [tools['llvm-dis'], raw, '-o', ir])
lines = []
for line in ir.read_text().splitlines():
if re.search(r'@llvm.used|wchar_size|llvm.module.flags|__hip_cuid', line):
continue
line = line.replace('optnone', '').replace(f'"target-cpu"="{args.gpu_arch}"', '')
lines.append(re.sub(r'"target-features"="[^"]+"', '', line))
ir.write_text('\n'.join(lines) + '\n')
run('assemble', [tools['llvm-as'], ir, '-o', extra])
env['D4R_ZLUDA_EXTRA_BC'] = str(extra)
run('emit', [emitter, edited, work / 'emitted', args.gpu_arch])
optimized = (work / 'emitted/opt.ll').read_text()
definition = re.search(r'^define amdgpu_kernel .*@' + re.escape(args.kernel) + r'\(.*#(\d+)\s*\{', optimized, re.M)
if not definition:
raise RuntimeError('Cannot verify shader mode: kernel attributes are missing')
attributes = re.search(r'^attributes #' + definition[1] + r' = \{([^\n]+)\}', optimized, re.M)
expected_mode = '-cumode' if args.shader_mode == 'wgp' else '+cumode'
if not attributes or expected_mode not in attributes[1]:
raise RuntimeError(f'Emitter did not honor {args.shader_mode} mode; build the corresponding ZLUDA patch first')
require_code_object_target(work / 'emitted/module.hsaco', args.gpu_arch)
shutil.copyfile(work / 'emitted/module.hsaco', output / (args.kernel + '.hsaco'))
run('manifest', [sys.executable, repo / 'kernels/tools/kernel_manifest.py', output, args.dlss_dll])
metadata = dict(architecture=args.gpu_arch, accuracy=True, nativeFP8=False, shaderMode=args.shader_mode,
lastBuiltKernel=args.kernel, lastRecipeRewrites=rewrite_counts,
privateNvidiaDerivedCode=True, validation='not yet validated; do not install before comparison',
dlssSha256=hashlib.sha256(args.dlss_dll.read_bytes()).hexdigest(),
zludaBuild=json.loads((zluda / 'build-info.json').read_text(encoding='utf-8-sig')),
compiler=subprocess.check_output([str(tools['clang++']), '--version'], env=env).decode(errors='replace'),
objects={p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in output.glob('*.hsaco')})
(output / 'build-info.json').write_text(json.dumps(metadata, indent=2) + '\n')
print(f'Built PRIVATE {args.gpu_arch} texture override: {output}; validation required before use')
if __name__ == '__main__':
main()
tools/windows/cuda_api.h
-48
View File
@@ -1,48 +0,0 @@
#pragma once
#include "diagnostic.h"
// CUDA Driver API ABI subset. No CUDA SDK, CUDA runtime or NVIDIA binary is needed.
using CUdevice = int;
using CUdeviceptr = uint64_t;
using CUcontext = void*;
using CUmodule = void*;
using CUfunction = void*;
struct CudaApi {
bool verbose = true;
d4r::diag::SearchDirectory search;
d4r::diag::Library library;
#define D4R_CU(name, ret, ...) using name##_fn = ret(WINAPI*)(__VA_ARGS__); name##_fn name = library.symbol<name##_fn>(#name)
D4R_CU(cuInit, int, unsigned);
D4R_CU(cuDriverGetVersion, int, int*);
D4R_CU(cuDeviceGetCount, int, int*);
D4R_CU(cuDeviceGet, int, CUdevice*, int);
D4R_CU(cuDeviceGetName, int, char*, int, CUdevice);
D4R_CU(cuDeviceGetAttribute, int, int*, int, CUdevice);
D4R_CU(cuDevicePrimaryCtxRetain, int, CUcontext*, CUdevice);
D4R_CU(cuDevicePrimaryCtxRelease_v2, int, CUdevice);
D4R_CU(cuCtxSetCurrent, int, CUcontext);
D4R_CU(cuCtxCreate_v2, int, CUcontext*, unsigned, CUdevice);
D4R_CU(cuCtxDestroy_v2, int, CUcontext);
D4R_CU(cuCtxSynchronize, int);
D4R_CU(cuMemAlloc_v2, int, CUdeviceptr*, size_t);
D4R_CU(cuMemFree_v2, int, CUdeviceptr);
D4R_CU(cuMemcpyHtoD_v2, int, CUdeviceptr, const void*, size_t);
D4R_CU(cuMemcpyDtoH_v2, int, void*, CUdeviceptr, size_t);
D4R_CU(cuModuleLoadData, int, CUmodule*, const void*);
D4R_CU(cuModuleUnload, int, CUmodule);
D4R_CU(cuModuleGetFunction, int, CUfunction*, CUmodule, const char*);
D4R_CU(cuLaunchKernel, int, CUfunction, unsigned, unsigned, unsigned, unsigned, unsigned, unsigned,
unsigned, void*, void**, void**);
D4R_CU(cuGetErrorName, int, int, const char**);
D4R_CU(cuGetErrorString, int, int, const char**);
#undef D4R_CU
explicit CudaApi(const std::string& path) :
search(std::filesystem::path(d4r::diag::wide(path)).parent_path()),
library(d4r::diag::wide(path)) {}
void check(int result, const char* call) {
const char* name = "unknown", *message = "unknown";
if (result) { cuGetErrorName(result, &name); cuGetErrorString(result, &message); }
else name = "CUDA_SUCCESS";
if (verbose || result) std::printf("CUDA %s -> %d (%s)\n", call, result, name ? name : "unknown");
if (result) throw std::runtime_error(std::string(call) + ": " + (message ? message : "unknown"));
}
};
tools/windows/cuda_driver_probe.cpp
-141
View File
@@ -1,141 +0,0 @@
#include "hip_api.h"
#include <algorithm>
#include "cuda_api.h"
static constexpr char ptx[] = R"PTX(
.version 7.0
.target sm_75
.address_size 64
.visible .entry d4r_ptx_pattern(
.param .u64 output, .param .u32 count, .param .u32 seed)
{
.reg .pred %p;
.reg .b32 %r<8>;
.reg .b64 %rd<4>;
ld.param.u64 %rd0, [output];
ld.param.u32 %r0, [count];
ld.param.u32 %r1, [seed];
mov.u32 %r2, %ctaid.x;
mov.u32 %r3, %ntid.x;
mov.u32 %r4, %tid.x;
mad.lo.u32 %r5, %r2, %r3, %r4;
setp.ge.u32 %p, %r5, %r0;
@%p bra done;
mad.lo.u32 %r6, %r5, 1664525, %r1;
xor.b32 %r7, %r6, 0xa5a55a5a;
mul.wide.u32 %rd1, %r5, 4;
add.u64 %rd2, %rd0, %rd1;
st.global.u32 [%rd2], %r7;
done:
ret;
}
)PTX";
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
uint64_t identity = 0xcbf29ce484222325ull;
for (const char* c = ptx; *c; ++c) identity = (identity ^ uint8_t(*c)) * 0x100000001b3ull;
std::printf("PTX_IDENTITY kernel=d4r_ptx_pattern fnv1a64=%016llx\n", static_cast<unsigned long long>(identity));
if (args.cuda_dll.empty()) throw std::runtime_error("--cuda-dll is required (absolute ZLUDA nvcuda.dll path)");
if (args.context != "primary" && args.context != "created") throw std::runtime_error("context must be primary or created");
// Require the real HIP architecture, then match the CUDA device by PCI identity.
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
CudaApi api(args.cuda_dll);
loaded_modules();
std::printf("STAGE cuInit\n");
api.check(api.cuInit(0), "cuInit");
int driver = 0, count = 0;
api.check(api.cuDriverGetVersion(&driver), "cuDriverGetVersion");
api.check(api.cuDeviceGetCount(&count), "cuDeviceGetCount");
std::printf("CUDA driver_api=%d devices=%d\n", driver, count);
CUdevice device = -1;
for (int ordinal = 0; ordinal < count; ++ordinal) {
CUdevice candidate;
api.check(api.cuDeviceGet(&candidate, ordinal), "cuDeviceGet");
char name[256]{};
int bus = -1, pci_device = -1, domain = -1;
api.check(api.cuDeviceGetName(name, sizeof(name), candidate), "cuDeviceGetName");
api.check(api.cuDeviceGetAttribute(&bus, 33, candidate), "cuDeviceGetAttribute(PCI_BUS_ID)");
api.check(api.cuDeviceGetAttribute(&pci_device, 34, candidate), "cuDeviceGetAttribute(PCI_DEVICE_ID)");
api.check(api.cuDeviceGetAttribute(&domain, 50, candidate), "cuDeviceGetAttribute(PCI_DOMAIN_ID)");
std::printf("CUDA GPU ordinal=%d name=%s pci=%04x:%02x:%02x\n", ordinal, name, domain, bus, pci_device);
if (bus == props.pciBusID && pci_device == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
}
if (device < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
CUcontext context = nullptr;
const bool primary = args.context == "primary";
if (primary) {
api.check(api.cuDevicePrimaryCtxRetain(&context, device), "cuDevicePrimaryCtxRetain");
} else api.check(api.cuCtxCreate_v2(&context, 0, device), "cuCtxCreate_v2");
struct ContextCleanup {
CudaApi& api; CUcontext context; CUdevice device; bool primary;
~ContextCleanup() { if (context) { api.cuCtxSetCurrent(nullptr);
if (primary) api.cuDevicePrimaryCtxRelease_v2(device); else api.cuCtxDestroy_v2(context); } }
} cleanup{api, context, device, primary};
api.check(api.cuCtxSetCurrent(context), "cuCtxSetCurrent");
void* stream = nullptr;
using CreateStream = int(WINAPI*)(void**, unsigned);
using DestroyStream = int(WINAPI*)(void*);
DestroyStream destroyStream = nullptr;
if (std::getenv("D4R_DIAG_NONBLOCKING_STREAM")) {
const auto createStream = api.library.symbol<CreateStream>("cuStreamCreate");
destroyStream = api.library.symbol<DestroyStream>("cuStreamDestroy_v2");
api.check(createStream(&stream, 1), "cuStreamCreate(NON_BLOCKING)");
}
struct StreamCleanup {
void*& stream; DestroyStream destroy;
~StreamCleanup() { if (stream && destroy) destroy(stream); }
} streamCleanup{stream, destroyStream};
const bool queued = std::getenv("D4R_DIAG_QUEUE_PTX") != nullptr;
if (queued && !stream) throw std::runtime_error("Queued PTX diagnostic requires D4R_DIAG_NONBLOCKING_STREAM=1");
CUmodule module = nullptr;
std::printf("STAGE cuModuleLoadData (PTX JIT)\n");
api.check(api.cuModuleLoadData(&module, ptx), "cuModuleLoadData");
struct ModuleCleanup { CudaApi& api; CUmodule m; ~ModuleCleanup() { if (m) api.cuModuleUnload(m); } } mc{api, module};
loaded_modules();
CUfunction function = nullptr;
api.check(api.cuModuleGetFunction(&function, module, "d4r_ptx_pattern"), "cuModuleGetFunction");
constexpr uint32_t nmax = 4099;
std::vector<uint32_t> result(nmax + 61, 0xdeadbeefu);
const size_t bytes = result.size() * sizeof(uint32_t);
CUdeviceptr pointer = 0;
api.check(api.cuMemAlloc_v2(&pointer, bytes), "cuMemAlloc_v2");
struct MemCleanup { CudaApi& api; CUdeviceptr p; ~MemCleanup() { if (p) api.cuMemFree_v2(p); } } ac{api, pointer};
if (queued) api.check(api.cuMemcpyHtoD_v2(pointer, result.data(), bytes), "cuMemcpyHtoD_v2(queued sentinel)");
for (unsigned i = 0; i < args.iterations; ++i) {
uint32_t n = queued ? nmax : nmax - i % 129u, seed = 0x12345678u + i * 31u;
if (!queued) {
std::fill(result.begin(), result.end(), 0xdeadbeefu);
api.check(api.cuMemcpyHtoD_v2(pointer, result.data(), bytes), "cuMemcpyHtoD_v2(sentinel)");
}
void* parameters[] = {&pointer, &n, &seed};
api.check(api.cuLaunchKernel(function, (n + 127u) / 128u, 1, 1, 128, 1, 1,
0, stream, parameters, nullptr), "cuLaunchKernel");
if (queued && i + 1 != args.iterations) continue;
api.check(api.cuCtxSynchronize(), "cuCtxSynchronize");
api.check(api.cuMemcpyDtoH_v2(result.data(), pointer, bytes), "cuMemcpyDtoH_v2");
if (!verify(result, n, seed)) return 5;
}
api.check(api.cuMemFree_v2(pointer), "cuMemFree_v2"); ac.p = 0;
api.check(api.cuModuleUnload(module), "cuModuleUnload"); mc.m = nullptr;
if (stream) { api.check(destroyStream(stream), "cuStreamDestroy_v2"); stream = nullptr; }
api.check(api.cuCtxSetCurrent(nullptr), "cuCtxSetCurrent(NULL)");
if (primary) api.check(api.cuDevicePrimaryCtxRelease_v2(device), "cuDevicePrimaryCtxRelease_v2");
else api.check(api.cuCtxDestroy_v2(context), "cuCtxDestroy_v2");
cleanup.context = nullptr;
std::printf("PASS CUDA architecture=%s context=%s iterations=%u guard_verified=1 queued=%u\n", d4r::diag::HipApi::target_arch(),
args.context.c_str(), args.iterations, unsigned(queued));
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL CUDA %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/cuda_image_api.h
-157
View File
@@ -1,157 +0,0 @@
#pragma once
#include "cuda_api.h"
namespace d4r::cuda {
// Public CUDA Driver API layouts, kept independent of NVIDIA SDK headers.
using Array = void*;
using Texture = uint64_t;
struct ArrayDescriptor { size_t width, height; uint32_t format, channels; };
struct Array3DDescriptor {
size_t width, height, depth;
uint32_t format, channels, flags;
};
struct Copy2D {
size_t srcX, srcY;
uint32_t srcType, srcAlignment;
const void* srcHost;
CUdeviceptr srcDevice;
Array srcArray;
size_t srcPitch, dstX, dstY;
uint32_t dstType, dstAlignment;
void* dstHost;
CUdeviceptr dstDevice;
Array dstArray;
size_t dstPitch, widthBytes, height;
};
struct ResourceDescriptor {
uint32_t type, alignment;
union { Array array; int reserved[32]; } resource;
uint32_t flags, reserved;
};
struct TextureDescriptor {
uint32_t addressMode[3], filterMode, flags, maxAnisotropy, mipmapFilterMode;
float mipmapBias, minMipmapClamp, maxMipmapClamp, borderColor[4];
int reserved[12];
};
static_assert(sizeof(Array3DDescriptor) == 40);
static_assert(sizeof(ArrayDescriptor) == 24);
static_assert(sizeof(Copy2D) == 128);
static_assert(sizeof(ResourceDescriptor) == 144);
static_assert(sizeof(TextureDescriptor) == 104);
struct ImageApi {
CudaApi& cuda;
#define D4R_IMAGE(name, ...) using name##_fn = int(WINAPI*)(__VA_ARGS__); name##_fn name = cuda.library.symbol<name##_fn>(#name)
D4R_IMAGE(cuArray3DCreate_v2, Array*, const Array3DDescriptor*);
D4R_IMAGE(cuArrayGetDescriptor_v2, ArrayDescriptor*, Array);
D4R_IMAGE(cuArray3DGetDescriptor_v2, Array3DDescriptor*, Array);
D4R_IMAGE(cuArrayDestroy, Array);
D4R_IMAGE(cuMemcpy2D_v2, const Copy2D*);
D4R_IMAGE(cuMemcpy2DAsync_v2, const Copy2D*, void*);
D4R_IMAGE(cuTexObjectCreate, Texture*, const ResourceDescriptor*, const TextureDescriptor*, const void*);
D4R_IMAGE(cuTexObjectDestroy, Texture);
D4R_IMAGE(cuTexObjectGetResourceDesc, ResourceDescriptor*, Texture);
D4R_IMAGE(cuSurfObjectCreate, Texture*, const ResourceDescriptor*);
D4R_IMAGE(cuSurfObjectDestroy, Texture);
D4R_IMAGE(cuSurfObjectGetResourceDesc, ResourceDescriptor*, Texture);
#undef D4R_IMAGE
explicit ImageApi(CudaApi& api) : cuda(api) {}
};
class Image {
ImageApi& api_;
void release() noexcept {
if (object) {
const int result = surface_ ? api_.cuSurfObjectDestroy(object) : api_.cuTexObjectDestroy(object);
if (result) std::fprintf(stderr, "IMAGE_DESTROY object result=%d\n", result);
object = 0;
}
if (array_) {
const int result = api_.cuArrayDestroy(array_);
if (result) std::fprintf(stderr, "IMAGE_DESTROY array result=%d\n", result);
array_ = nullptr;
}
}
Array array_ = nullptr;
bool surface_;
size_t rowBytes_, height_;
public:
Texture object = 0;
Image(ImageApi& api, unsigned width, unsigned height, uint32_t format,
unsigned channels, bool surface, unsigned filter = 0)
: api_(api), surface_(surface), rowBytes_(size_t(width) * channels * (format == 16 ? 2 : 4)), height_(height) {
try {
const Array3DDescriptor descriptor{width, height, 0, format, channels, surface ? 2u : 0u};
api_.cuda.check(api_.cuArray3DCreate_v2(&array_, &descriptor), "cuArray3DCreate_v2");
ResourceDescriptor resource{};
resource.resource.array = array_;
if (surface) api_.cuda.check(api_.cuSurfObjectCreate(&object, &resource), "cuSurfObjectCreate");
else {
TextureDescriptor texture{};
texture.addressMode[0] = texture.addressMode[1] = 1; // clamp
texture.filterMode = filter;
texture.flags = 2; // normalized coordinates
api_.cuda.check(api_.cuTexObjectCreate(&object, &resource, &texture, nullptr), "cuTexObjectCreate");
}
ResourceDescriptor roundtrip{};
api_.cuda.check(surface ? api_.cuSurfObjectGetResourceDesc(&roundtrip, object) :
api_.cuTexObjectGetResourceDesc(&roundtrip, object), "CUDA object resource descriptor roundtrip");
if (roundtrip.type != 0 || roundtrip.resource.array != array_ || roundtrip.flags != 0)
throw std::runtime_error("CUDA object descriptor does not match its array");
ArrayDescriptor twoD{};
Array3DDescriptor threeD{};
api_.cuda.check(api_.cuArrayGetDescriptor_v2(&twoD, array_), "cuArrayGetDescriptor_v2");
api_.cuda.check(api_.cuArray3DGetDescriptor_v2(&threeD, array_), "cuArray3DGetDescriptor_v2");
if (twoD.width != width || twoD.height != height || twoD.format != format || twoD.channels != channels ||
threeD.width != width || threeD.height != height || threeD.depth != 0 || threeD.format != format ||
threeD.channels != channels || threeD.flags != descriptor.flags)
throw std::runtime_error("CUDA array descriptor geometry or format mismatch");
} catch (...) { release(); throw; }
}
Image(const Image&) = delete;
Image& operator=(const Image&) = delete;
~Image() { release(); }
void verify_wrong_object_kind() {
ResourceDescriptor wrong{};
const int result = surface_ ? api_.cuTexObjectGetResourceDesc(&wrong, object) :
api_.cuSurfObjectGetResourceDesc(&wrong, object);
if (!result) throw std::runtime_error("Wrong CUDA object kind was accepted as a valid resource");
std::printf("CUDA wrong-kind descriptor safely rejected result=%d\n", result);
}
void upload(const void* source) {
Copy2D copy{};
copy.srcType = 1; copy.srcHost = source; copy.srcPitch = rowBytes_;
copy.dstType = 3; copy.dstArray = array_;
copy.widthBytes = rowBytes_; copy.height = height_;
api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(host to diagnostic array)");
}
void download(void* destination) {
Copy2D copy{};
copy.srcType = 3; copy.srcArray = array_;
copy.dstType = 1; copy.dstHost = destination; copy.dstPitch = rowBytes_;
copy.widthBytes = rowBytes_; copy.height = height_;
api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(diagnostic array to host)");
}
void upload_device(CUdeviceptr source, size_t pitch, bool deferred = false) {
if (pitch < rowBytes_) throw std::runtime_error("Device input pitch is smaller than its row");
Copy2D copy{};
copy.srcType = 2; copy.srcDevice = source; copy.srcPitch = pitch;
copy.dstType = 3; copy.dstArray = array_;
copy.widthBytes = rowBytes_; copy.height = height_;
if (deferred) {
// The Windows ZLUDA backend and HIP conversion use the same
// legacy default stream. Caller must complete the whole input
// batch before NGX may consume arrays on its own streams.
api_.cuda.check(api_.cuMemcpy2DAsync_v2(&copy, nullptr), "cuMemcpy2DAsync_v2(external VRAM to array)");
} else api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(external VRAM to array)");
}
void download_device(CUdeviceptr destination, size_t pitch) {
if (pitch < rowBytes_) throw std::runtime_error("Device output pitch is smaller than its row");
Copy2D copy{};
copy.srcType = 3; copy.srcArray = array_;
copy.dstType = 2; copy.dstDevice = destination; copy.dstPitch = pitch;
copy.widthBytes = rowBytes_; copy.height = height_;
api_.cuda.check(api_.cuMemcpy2D_v2(&copy), "cuMemcpy2D_v2(array to external VRAM)");
}
};
}
tools/windows/cuda_image_probe.cpp
-75
View File
@@ -1,75 +0,0 @@
#include "hip_api.h"
#include "cuda_image_api.h"
int main(int argc, char** argv) {
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
SearchDirectory search(std::filesystem::path(wide(args.cuda_dll)).parent_path());
CudaApi cuda(args.cuda_dll);
cuda.check(cuda.cuInit(0), "cuInit");
CUdevice device = -1;
int count = 0;
cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
for (int ordinal = 0; ordinal < count; ++ordinal) {
CUdevice candidate = -1;
int bus = -1, pciDevice = -1, domain = -1;
cuda.check(cuda.cuDeviceGet(&candidate, ordinal), "cuDeviceGet");
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "cuDeviceGetAttribute(PCI_BUS_ID)");
cuda.check(cuda.cuDeviceGetAttribute(&pciDevice, 34, candidate), "cuDeviceGetAttribute(PCI_DEVICE_ID)");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "cuDeviceGetAttribute(PCI_DOMAIN_ID)");
if (bus == props.pciBusID && pciDevice == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
}
if (device < 0) throw std::runtime_error("CUDA image probe HIP/CUDA PCI identity mismatch");
CUcontext context = nullptr;
cuda.check(cuda.cuCtxCreate_v2(&context, 0, device), "cuCtxCreate_v2");
struct Cleanup { CudaApi& api; CUcontext context; ~Cleanup() { (void)api.cuCtxDestroy_v2(context); } } cleanup{cuda, context};
d4r::cuda::ImageApi images(cuda);
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
for (unsigned format : {16u, 32u}) for (unsigned channels : {1u, 2u, 4u}) for (bool surface : {false, true}) {
const unsigned halvesPerPixel = channels * (format == 16 ? 1 : 2);
d4r::cuda::Image image(images, 17, 9, format, channels, surface);
image.verify_wrong_object_kind();
std::vector<uint16_t> input(17 * 9 * halvesPerPixel), output(input.size());
for (size_t i = 0; i < input.size(); ++i) input[i] = uint16_t((i + iteration) % 8 * 0x100);
image.upload(input.data());
image.download(output.data());
if (input != output) throw std::runtime_error("CUDA array storage roundtrip mismatch");
const size_t rowBytes = 17 * halvesPerPixel * sizeof(uint16_t), pitch = (rowBytes + 255) & ~size_t(255);
CUdeviceptr devicePixels = 0;
cuda.check(cuda.cuMemAlloc_v2(&devicePixels, pitch * 9), "cuMemAlloc_v2(pitched image regression)");
struct PixelsCleanup { CudaApi& cuda; CUdeviceptr pointer; ~PixelsCleanup() { (void)cuda.cuMemFree_v2(pointer); } } pixelsCleanup{cuda, devicePixels};
std::vector<uint8_t> padded(pitch * 9, 0xcd);
for (unsigned y = 0; y < 9; ++y) std::memcpy(padded.data() + y * pitch, input.data() + y * 17 * halvesPerPixel, rowBytes);
cuda.check(cuda.cuMemcpyHtoD_v2(devicePixels, padded.data(), padded.size()), "cuMemcpyHtoD_v2(pitched pattern)");
const bool deferred = std::getenv("D4R_DIAG_ASYNC_ARRAY_COPY") != nullptr;
image.upload_device(devicePixels, pitch, deferred);
if (deferred) cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize(input batch regression)");
image.download(output.data());
if (input != output) throw std::runtime_error("CUDA device-to-array pitched copy mismatch");
std::fill(padded.begin(), padded.end(), 0xee);
cuda.check(cuda.cuMemcpyHtoD_v2(devicePixels, padded.data(), padded.size()), "cuMemcpyHtoD_v2(output sentinel)");
image.download_device(devicePixels, pitch);
cuda.check(cuda.cuMemcpyDtoH_v2(padded.data(), devicePixels, padded.size()), "cuMemcpyDtoH_v2(device array output)");
for (unsigned y = 0; y < 9; ++y) {
if (std::memcmp(padded.data() + y * pitch, input.data() + y * 17 * halvesPerPixel, rowBytes))
throw std::runtime_error("CUDA array-to-device pitched copy mismatch");
if (!std::all_of(padded.begin() + y * pitch + rowBytes, padded.begin() + (y + 1) * pitch, [](uint8_t value) { return value == 0xee; }))
throw std::runtime_error("CUDA array-to-device copy overwrote pitch padding");
}
}
}
cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize");
std::printf("PASS CUDA_IMAGES architecture=%s iterations=%u descriptor_and_storage=1 pitched_device_copies=1 fp16_fp32=1 async_array_copy=%u\n", d4r::diag::HipApi::target_arch(),
args.iterations, unsigned(std::getenv("D4R_DIAG_ASYNC_ARRAY_COPY") != nullptr));
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL CUDA_IMAGES %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/cuda_mma_probe.cpp
-171
View File
@@ -1,171 +0,0 @@
#include "hip_api.h"
#include "cuda_api.h"
#include "binary16.h"
#include <array>
#include <cmath>
#include <cstring>
// Public synthetic matrix fragments, independent of NVIDIA DLLs and weights.
static constexpr char source[] = R"PTX(
.version 7.0
.target sm_75
.address_size 64
.visible .entry d4r_f16_mma_reference(
.param .u64 A, .param .u64 B, .param .u64 C, .param .u64 D)
{
.reg .b32 %r<24>;
.reg .b64 %rd<12>;
.reg .b16 %h<4>;
ld.param.u64 %rd0, [A];
ld.param.u64 %rd1, [B];
ld.param.u64 %rd2, [C];
ld.param.u64 %rd3, [D];
mov.u32 %r0, %ctaid.x;
mov.u32 %r1, %tid.x;
shr.u32 %r2, %r1, 2;
and.b32 %r3, %r1, 3;
shl.b32 %r3, %r3, 1;
mul.wide.u32 %rd4, %r0, 512;
add.u64 %rd0, %rd0, %rd4;
mul.wide.u32 %rd4, %r0, 256;
add.u64 %rd1, %rd1, %rd4;
add.u64 %rd2, %rd2, %rd4;
add.u64 %rd3, %rd3, %rd4;
mad.lo.u32 %r4, %r2, 16, %r3;
mul.wide.u32 %rd4, %r4, 2;
add.u64 %rd5, %rd0, %rd4;
ld.global.u32 %r8, [%rd5];
ld.global.u32 %r9, [%rd5+256];
ld.global.u32 %r10, [%rd5+16];
ld.global.u32 %r11, [%rd5+272];
mad.lo.u32 %r4, %r3, 8, %r2;
mul.wide.u32 %rd4, %r4, 2;
add.u64 %rd5, %rd1, %rd4;
ld.global.b16 %h0, [%rd5];
ld.global.b16 %h1, [%rd5+16];
mov.b32 %r12, {%h0,%h1};
ld.global.b16 %h2, [%rd5+128];
ld.global.b16 %h3, [%rd5+144];
mov.b32 %r13, {%h2,%h3};
mad.lo.u32 %r4, %r2, 8, %r3;
mul.wide.u32 %rd4, %r4, 2;
add.u64 %rd5, %rd2, %rd4;
ld.global.u32 %r14, [%rd5];
ld.global.u32 %r15, [%rd5+128];
mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16
{%r16,%r17}, {%r8,%r9,%r10,%r11}, {%r12,%r13}, {%r14,%r15};
add.u64 %rd5, %rd3, %rd4;
st.global.u32 [%rd5], %r16;
st.global.u32 [%rd5+128], %r17;
ret;
}
)PTX";
#if defined(_MSC_VER)
using d4r::diag::binary16::half_bits;
using d4r::diag::binary16::half_value;
#else
static uint16_t half_bits(double value) {
_Float16 half = static_cast<_Float16>(value);
uint16_t bits;
std::memcpy(&bits, &half, sizeof(bits));
return bits;
}
static double half_value(uint16_t bits) {
_Float16 half;
std::memcpy(&half, &bits, sizeof(bits));
return static_cast<double>(half);
}
#endif
int main(int argc, char** argv) {
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
CudaApi cuda(args.cuda_dll);
cuda.check(cuda.cuInit(0), "cuInit");
CUdevice device = -1;
int count = 0;
cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
for (int i = 0; i < count; ++i) {
CUdevice candidate;
int bus, pci, domain;
cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
cuda.check(cuda.cuDeviceGetAttribute(&pci, 34, candidate), "PCI device");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
if (bus == props.pciBusID && pci == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
}
if (device < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
CUcontext context = nullptr;
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, device), "primary context");
struct Context { CudaApi& api; CUdevice device; ~Context() { api.cuCtxSetCurrent(nullptr); api.cuDevicePrimaryCtxRelease_v2(device); } } context_owner{cuda, device};
cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent");
CUmodule module = nullptr;
cuda.check(cuda.cuModuleLoadData(&module, source), "MMA PTX JIT");
struct Module { CudaApi& api; CUmodule module; ~Module() { api.cuModuleUnload(module); } } module_owner{cuda, module};
CUfunction kernel = nullptr;
cuda.check(cuda.cuModuleGetFunction(&kernel, module, "d4r_f16_mma_reference"), "MMA kernel");
constexpr size_t cases = 512;
std::vector<uint16_t> a(cases * 256), b(cases * 128), c(cases * 128), d(cases * 128);
uint32_t random = 0x192a321u;
auto next = [&]() { random = random * 1664525u + 1013904223u; return random; };
for (size_t n = 0; n < cases; ++n) {
for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = uint16_t(0x2800u + (next() >> 20));
for (size_t i = 0; i < 128; ++i) {
b[n * 128 + i] = uint16_t((next() & 0x8000u) | (0x3000u + (next() >> 20)));
c[n * 128 + i] = half_bits(n % 3 ? 0. : 0.015625);
}
// Sweep large cancellation terms and a tiny surviving product
// across K positions; this exposes loss hidden by integer fixtures.
if (n < 256) {
for (size_t i = 0; i < 256; ++i) a[n * 256 + i] = half_bits(1.);
std::fill(b.begin() + n * 128, b.begin() + (n + 1) * 128, half_bits(0.));
for (size_t col = 0; col < 8; ++col) {
b[n * 128 + (n % 16) * 8 + col] = half_bits(32768.);
b[n * 128 + ((n % 16 + 1) % 16) * 8 + col] = half_bits(-32768.);
b[n * 128 + ((n % 16 + 2 + n / 16 % 14) % 16) * 8 + col] = half_bits(0.0009765625);
}
}
}
struct Allocation {
CudaApi& api; CUdeviceptr pointer = 0;
Allocation(CudaApi& api, size_t bytes) : api(api) { api.check(api.cuMemAlloc_v2(&pointer, bytes), "MMA allocate"); }
~Allocation() { if (pointer) api.cuMemFree_v2(pointer); }
} da(cuda, a.size() * 2), db(cuda, b.size() * 2), dc(cuda, c.size() * 2), dd(cuda, d.size() * 2);
cuda.check(cuda.cuMemcpyHtoD_v2(da.pointer, a.data(), a.size() * 2), "MMA A upload");
cuda.check(cuda.cuMemcpyHtoD_v2(db.pointer, b.data(), b.size() * 2), "MMA B upload");
cuda.check(cuda.cuMemcpyHtoD_v2(dc.pointer, c.data(), c.size() * 2), "MMA C upload");
std::fill(d.begin(), d.end(), uint16_t(0x7e00));
cuda.check(cuda.cuMemcpyHtoD_v2(dd.pointer, d.data(), d.size() * 2), "MMA NaN sentinel");
void* parameters[] = {&da.pointer, &db.pointer, &dc.pointer, &dd.pointer};
cuda.check(cuda.cuLaunchKernel(kernel, cases, 1, 1, 32, 1, 1, 0, nullptr, parameters, nullptr), "MMA launch");
cuda.check(cuda.cuCtxSynchronize(), "MMA synchronize");
cuda.check(cuda.cuMemcpyDtoH_v2(d.data(), dd.pointer, d.size() * 2), "MMA download");
size_t mismatch = 0;
for (size_t n = 0; n < cases; ++n)
for (size_t row = 0; row < 16; ++row)
for (size_t col = 0; col < 8; ++col) {
double sum = half_value(c[n * 128 + row * 8 + col]);
for (size_t k = 0; k < 16; ++k)
sum += half_value(a[n * 256 + row * 16 + k]) * half_value(b[n * 128 + k * 8 + col]);
const uint16_t expected = half_bits(sum);
const uint16_t actual = d[n * 128 + row * 8 + col];
if (expected != actual) {
if (mismatch++ < 8) std::fprintf(stderr, "MMA_DIFFERENT case=%zu row=%zu column=%zu expected=%04x actual=%04x\n", n, row, col, expected, actual);
}
}
std::printf("MMA_REFERENCE cases=%zu elements=%zu differing=%zu\n", cases, d.size(), mismatch);
if (mismatch) return 5;
std::printf("PASS CUDA_F16_MMA architecture=%s cases=%zu exact_f16=1\n", d4r::diag::HipApi::target_arch(), cases);
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "FAIL CUDA_F16_MMA %s\n", error.what());
loaded_modules();
return 4;
}
}
tools/windows/d3d12_boundary_timing.h
-55
View File
@@ -1,55 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include <cstring>
namespace d4r::win {
// Diagnostic timestamps, read only after the copy slot's existing completion
// fence. No extra queue/CPU wait, and no image readback. The interval across
// the external fence includes HIP execution and scheduling, not just kernels.
class BoundaryTiming {
ComPtr<ID3D12QueryHeap> queries_;
ComPtr<ID3D12Resource> readback_;
uint64_t ticket_ = 0;
public:
explicit BoundaryTiming(ID3D12Device* device) {
D3D12_QUERY_HEAP_DESC query{};
query.Type = D3D12_QUERY_HEAP_TYPE_TIMESTAMP; query.Count = 4;
dx(device->CreateQueryHeap(&query, IID_PPV_ARGS(queries_.GetAddressOf())), "Boundary timestamp heap");
D3D12_HEAP_PROPERTIES heap{}; heap.Type = D3D12_HEAP_TYPE_READBACK;
D3D12_RESOURCE_DESC desc{};
desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; desc.Width = 4 * sizeof(uint64_t);
desc.Height = 1; desc.DepthOrArraySize = 1; desc.MipLevels = 1;
desc.SampleDesc.Count = 1; desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
dx(device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc,
D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(readback_.GetAddressOf())), "Boundary timestamp readback");
}
void input_begin(ID3D12GraphicsCommandList* list) { list->EndQuery(queries_.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 0); }
void input_end(ID3D12GraphicsCommandList* list) { list->EndQuery(queries_.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 1); }
void output_begin(ID3D12GraphicsCommandList* list) { list->EndQuery(queries_.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 2); }
void output_end(ID3D12GraphicsCommandList* list) {
list->EndQuery(queries_.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 3);
list->ResolveQueryData(queries_.Get(), D3D12_QUERY_TYPE_TIMESTAMP, 0, 4, readback_.Get(), 0);
}
void submitted(uint64_t ticket) { ticket_ = ticket; }
// Caller has observed the slot's completion fence, including ResolveQueryData.
void report(ID3D12CommandQueue* queue, const void* feature) {
if (!ticket_) return;
uint64_t ticks[4]{}, frequency = 0;
D3D12_RANGE read{0, sizeof(ticks)}, written{0, 0};
void* mapped = nullptr;
dx(readback_->Map(0, &read, &mapped), "Map completed boundary timestamps");
std::memcpy(ticks, mapped, sizeof(ticks)); readback_->Unmap(0, &written);
dx(queue->GetTimestampFrequency(&frequency), "Boundary timestamp frequency");
if (!frequency || !ticks[0] || ticks[0] > ticks[1] || ticks[1] > ticks[2] || ticks[2] > ticks[3])
throw std::runtime_error("Invalid completed boundary timestamps");
const double ms = 1000. / double(frequency);
std::printf("D4R_GPU_BOUNDARY feature=%p ticket=%llu input_copy_ms=%.6f external_span_ms=%.6f output_copy_ms=%.6f total_ms=%.6f input_start_ticks=%llu input_end_ticks=%llu output_start_ticks=%llu output_end_ticks=%llu frequency=%llu diagnostics_cpu_bytes=32 serializing=0\n",
feature, (unsigned long long)ticket_, double(ticks[1] - ticks[0]) * ms,
double(ticks[2] - ticks[1]) * ms, double(ticks[3] - ticks[2]) * ms,
double(ticks[3] - ticks[0]) * ms, (unsigned long long)ticks[0],
(unsigned long long)ticks[1], (unsigned long long)ticks[2],
(unsigned long long)ticks[3], (unsigned long long)frequency);
ticket_ = 0;
}
};
}
tools/windows/d3d12_command_hooks.cpp
-654
View File
@@ -1,654 +0,0 @@
#include "d3d12_command_hooks.h"
#include "d3d12_newer_commands.h"
#include "d3d12_retirement.h"
#include <MinHook.h>
#include <atomic>
#include <memory>
#include <mutex>
#include <unordered_map>
#include <vector>
#include <chrono>
namespace d4r::win::commands {
namespace {
thread_local bool internal = false;
std::mutex map_mutex, installation_mutex, error_mutex;
std::string last_error;
bool installed = false;
using Clock = std::chrono::steady_clock;
bool profile_commands() { static const bool enabled = std::getenv("D4R_PROFILE_STAGES") != nullptr; return enabled; }
double milliseconds(Clock::time_point start) { return std::chrono::duration<double, std::milli>(Clock::now() - start).count(); }
struct Event { UINT metadata; std::vector<uint8_t> data; };
using State = std::function<void(ID3D12GraphicsCommandList*)>;
struct TrackedResource {
// D3D12 command lists do not retain resource references. The caller keeps
// them alive through submission/completion. An extra tracking reference
// would keep swap-chain buffers alive and make ResizeBuffers fail.
ResourceAccess state;
};
using ResourceStates = std::unordered_map<ID3D12Resource*, TrackedResource>;
static constexpr GUID submitted_access_tag = {0x741b92df,0x9b13,0x4fc9,{0xb0,0x02,0x2c,0xe3,0x5a,0x68,0x92,0x41}};
void publish_states(const ResourceStates& states) {
for (const auto& entry : states) {
// Buffers/simultaneous-access textures have automatic legacy state
// decay at ExecuteCommandLists completion. Only ordinary textures
// retain these explicit transitions across submission boundaries.
const auto desc = resource_desc(entry.first);
if (desc.Dimension == D3D12_RESOURCE_DIMENSION_BUFFER || (desc.Flags & D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS)) continue;
dx(entry.first->SetPrivateData(submitted_access_tag, sizeof(ResourceAccess), &entry.second.state), "Publish submitted texture access");
}
}
struct Segment {
// The application's original list is a weak pointer; holding it in its
// own routing map would prevent final Release and leak the entire feature.
ID3D12GraphicsCommandList* list = nullptr;
ComPtr<ID3D12GraphicsCommandList> owner;
ComPtr<ID3D12CommandAllocator> allocator;
Boundary boundary;
ResourceStates resource_states;
};
struct Recording {
std::recursive_mutex mutex;
ID3D12GraphicsCommandList* original;
ComPtr<ID3D12GraphicsCommandList> current;
ComPtr<ID3D12CommandAllocator> current_allocator;
std::vector<Segment> segments;
std::vector<State> state;
std::vector<Event> events;
ResourceStates resource_states;
unsigned active_queries = 0;
bool render_pass = false, predication = false, indirect_state_unknown = false, closed = false;
explicit Recording(ID3D12GraphicsCommandList* list) : original(list) {}
};
std::unordered_map<ID3D12GraphicsCommandList*, std::shared_ptr<Recording>> recordings;
std::atomic<uint64_t> recording_generation{0};
struct RecordingCache {
ID3D12GraphicsCommandList* list = nullptr;
uint64_t generation = 0;
// A thread's last lookup must not retain suffix lists, allocators or game
// resources after Reset/final Release. Address reuse also needs a new
// generation, even if the old Recording is alive in a submit callback.
std::weak_ptr<Recording> value;
};
thread_local RecordingCache recording_cache;
std::shared_ptr<Recording> get_recording(ID3D12GraphicsCommandList* self, bool create = true) {
if (internal) return {};
const auto generation = recording_generation.load(std::memory_order_acquire);
if (recording_cache.list == self && recording_cache.generation == generation) {
if (auto value = recording_cache.value.lock()) return value;
}
std::lock_guard<std::mutex> lock(map_mutex);
auto found = recordings.find(self);
std::shared_ptr<Recording> value;
if (found != recordings.end()) value = found->second;
else {
if (!create) return {};
value = std::make_shared<Recording>(self);
recordings.emplace(self, value);
recording_generation.fetch_add(1, std::memory_order_release);
}
recording_cache.list = self;
recording_cache.value = value;
recording_cache.generation = recording_generation.load(std::memory_order_relaxed);
return value;
}
void forget(ID3D12GraphicsCommandList* self) {
std::shared_ptr<Recording> released;
{ std::lock_guard<std::mutex> lock(map_mutex);
auto it = recordings.find(self);
if (it != recordings.end()) {
released = std::move(it->second); recordings.erase(it);
recording_generation.fetch_add(1, std::memory_order_release);
} }
// Release retained COM objects without holding the map mutex.
}
struct HookSample {
uint32_t random = GetCurrentThreadId() | 1u;
uint64_t calls = 0, samples = 0, marked = 0;
double access_ms = 0, capture_ms = 0, driver_ms = 0;
Clock::time_point since = Clock::now();
bool choose() {
++calls; random ^= random << 13; random ^= random >> 17; random ^= random << 5;
return !(random & 63u);
}
void flush() {
const double elapsed = milliseconds(since);
if (elapsed < 2000) return;
std::printf("D4R_COMMAND_HOOK_SAMPLE thread=%lu period_ms=%.3f calls=%llu samples=%llu marked=%llu access_sum_ms=%.6f capture_sum_ms=%.6f driver_sum_ms=%.6f sampling=64\n",
GetCurrentThreadId(), elapsed, (unsigned long long)calls, (unsigned long long)samples,
(unsigned long long)marked, access_ms, capture_ms, driver_ms);
calls = samples = marked = 0; access_ms = capture_ms = driver_ms = 0; since = Clock::now();
}
};
thread_local HookSample hook_sample;
bool sample_hooks() { static const bool enabled = std::getenv("D4R_PROFILE_COMMAND_HOOKS") != nullptr; return enabled; }
struct Access {
std::shared_ptr<Recording> recording;
std::unique_lock<std::recursive_mutex> lock;
ID3D12GraphicsCommandList* self;
bool sampled = false;
Clock::time_point begin{}, acquired{}, driver{};
explicit Access(ID3D12GraphicsCommandList* list) : self(list) {
sampled = !internal && sample_hooks() && hook_sample.choose();
if (sampled) begin = Clock::now();
recording = get_recording(list);
if (recording) lock = std::unique_lock<std::recursive_mutex>(recording->mutex);
if (sampled) acquired = Clock::now();
}
void driver_call() {
if (sampled) driver = Clock::now();
}
~Access() {
if (!sampled) return;
++hook_sample.samples;
hook_sample.access_ms += std::chrono::duration<double, std::milli>(acquired - begin).count();
if (driver != Clock::time_point{}) {
++hook_sample.marked;
hook_sample.capture_ms += std::chrono::duration<double, std::milli>(driver - acquired).count();
hook_sample.driver_ms += milliseconds(driver);
}
hook_sample.flush();
}
ID3D12GraphicsCommandList* target() const {
return recording && recording->current ? recording->current.Get() : self;
}
};
template<class T> ComPtr<T> retain(T* pointer) { return ComPtr<T>(pointer); }
template<class T> std::vector<T> copy_values(const T* pointer, size_t count) {
return pointer ? std::vector<T>(pointer, pointer + count) : std::vector<T>();
}
template<class T> T* values_pointer(std::vector<T>& values) { return values.empty() ? nullptr : values.data(); }
template<class T> struct RetainedArray { std::vector<ComPtr<T>> owners; std::vector<T*> raw; };
template<class T> RetainedArray<T> retain_values(T* const* pointer, size_t count) {
RetainedArray<T> result;
for (size_t i = 0; pointer && i < count; ++i) { result.owners.emplace_back(pointer[i]); result.raw.push_back(pointer[i]); }
return result;
}
struct Hook { void* target; void* replacement; };
std::vector<Hook> attached;
void attach_method(void* target, void* replacement, void** original, const char* name) {
for (const auto& hook : attached) if (hook.target == target)
throw std::runtime_error(std::string("Shared D3D12 implementation needs an explicit alias adapter: ") + name);
const auto result = MH_CreateHook(target, replacement, original);
if (result != MH_OK) throw std::runtime_error(std::string("Cannot intercept D3D12 ") + name + ": " + MH_StatusToString(result));
attached.push_back({target, replacement});
std::printf("D4R_D3D12_HOOK method=%s\n", name);
}
#include "d3d12_command_hooks.generated.h"
static constexpr GUID signature_description_tag = {0xbdda6148,0x46f0,0x4d82,{0xa1,0x90,0x08,0x47,0x7e,0xda,0x32,0x12}};
using SignatureFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12Device*, const D3D12_COMMAND_SIGNATURE_DESC*, ID3D12RootSignature*, REFIID, void**);
SignatureFn original_signature = nullptr;
using CreateListFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12Device*, UINT, D3D12_COMMAND_LIST_TYPE,
ID3D12CommandAllocator*, ID3D12PipelineState*, REFIID, void**);
CreateListFn original_create_list = nullptr;
HRESULT STDMETHODCALLTYPE hook_create_list(ID3D12Device* device, UINT node, D3D12_COMMAND_LIST_TYPE type,
ID3D12CommandAllocator* allocator, ID3D12PipelineState* initial, REFIID iid, void** out) {
const auto result = original_create_list(device, node, type, allocator, initial, iid, out);
if (!internal && SUCCEEDED(result) && out && *out) try {
ComPtr<ID3D12GraphicsCommandList> list;
if (SUCCEEDED(static_cast<IUnknown*>(*out)->QueryInterface(IID_PPV_ARGS(list.GetAddressOf())))) {
auto recording = get_recording(list.Get());
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
auto pipeline = retain(initial);
if (pipeline) recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) {
original_SetPipelineState(target, pipeline.Get());
});
}
} catch (const std::exception& failure) {
// A public Create must retain its result even if tracking runs out of
// memory. A later split must fail explicitly instead of replaying an
// incomplete initial state.
{ std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
std::fprintf(stderr, "D4R_CREATE_LIST_TRACK_FAILURE %s\n", failure.what());
}
return result;
}
HRESULT STDMETHODCALLTYPE hook_signature(ID3D12Device* device, const D3D12_COMMAND_SIGNATURE_DESC* desc, ID3D12RootSignature* root, REFIID iid, void** out) {
const auto result = original_signature(device, desc, root, iid, out);
if (SUCCEEDED(result) && out && *out && desc && desc->NumArgumentDescs <= 4096) try {
ComPtr<ID3D12CommandSignature> signature;
if (SUCCEEDED(static_cast<IUnknown*>(*out)->QueryInterface(IID_PPV_ARGS(signature.GetAddressOf())))) {
std::vector<uint8_t> value(sizeof(UINT) + desc->NumArgumentDescs * sizeof(D3D12_INDIRECT_ARGUMENT_DESC));
std::memcpy(value.data(), &desc->NumArgumentDescs, sizeof(UINT));
if (desc->NumArgumentDescs)
std::memcpy(value.data() + sizeof(UINT), desc->pArgumentDescs, value.size() - sizeof(UINT));
const auto tagged = signature->SetPrivateData(signature_description_tag, UINT(value.size()), value.data());
if (FAILED(tagged)) std::fprintf(stderr, "D4R_SIGNATURE_TRACK_FAILURE result=0x%08x\n", unsigned(tagged));
else std::fprintf(stderr, "D4R_COMMAND_SIGNATURE arguments=%u tracked=1\n", desc->NumArgumentDescs);
}
} catch (const std::exception& failure) {
// Preserve the successful public API call. An untracked signature is
// rejected at a later split instead of unwinding into the application.
std::fprintf(stderr, "D4R_SIGNATURE_TRACK_FAILURE %s\n", failure.what());
}
return result;
}
void STDMETHODCALLTYPE hook_SetProgram(ID3D12GraphicsCommandList* self, const NewProgramDescription* desc) {
Access access(self);
if (access.recording && desc) {
auto saved = *desc;
// INITIALIZE changes backing memory. Rebinding an already initialized
// work graph in the suffix must preserve its prefix's GPU state.
if (saved.type == 5) saved.work_graph.flags &= ~1u;
access.recording->state.emplace_back([saved](ID3D12GraphicsCommandList* target) { original_SetProgram(target, &saved); });
}
original_SetProgram(access.target(), desc);
}
using ReleaseFn = ULONG(STDMETHODCALLTYPE*)(IUnknown*);
ReleaseFn original_release = nullptr;
ULONG STDMETHODCALLTYPE hook_release(IUnknown* self) {
const ULONG count = original_release(self);
if (!count) forget(reinterpret_cast<ID3D12GraphicsCommandList*>(self));
return count;
}
HRESULT STDMETHODCALLTYPE hook_Close(ID3D12GraphicsCommandList* self) {
Access access(self);
const auto result = original_Close(access.target());
if (SUCCEEDED(result) && access.recording) access.recording->closed = true;
return result;
}
HRESULT STDMETHODCALLTYPE hook_Reset(ID3D12GraphicsCommandList* self, ID3D12CommandAllocator* allocator, ID3D12PipelineState* initial_state) {
if (internal) return original_Reset(self, allocator, initial_state);
forget(self);
const auto result = original_Reset(self, allocator, initial_state);
if (SUCCEEDED(result)) {
auto recording = get_recording(self);
auto pipeline = retain(initial_state);
recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) {
if (pipeline) original_SetPipelineState(target, pipeline.Get());
});
}
return result;
}
void STDMETHODCALLTYPE hook_ClearState(ID3D12GraphicsCommandList* self, ID3D12PipelineState* pipeline_state) {
Access access(self);
if (access.recording) {
access.recording->state.clear(); access.recording->indirect_state_unknown = false;
auto pipeline = retain(pipeline_state);
access.recording->state.emplace_back([pipeline](ID3D12GraphicsCommandList* target) { original_ClearState(target, pipeline.Get()); });
}
original_ClearState(access.target(), pipeline_state);
}
void STDMETHODCALLTYPE hook_ResourceBarrier(ID3D12GraphicsCommandList* self, UINT barrier_count, const D3D12_RESOURCE_BARRIER* barriers) {
Access access(self);
if (access.recording) for (UINT i = 0; i < barrier_count; ++i)
if (barriers[i].Type == D3D12_RESOURCE_BARRIER_TYPE_TRANSITION &&
(barriers[i].Transition.Subresource == 0 || barriers[i].Transition.Subresource == D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES)) {
auto& tracked = access.recording->resource_states[barriers[i].Transition.pResource];
auto& state = tracked.state;
state = ResourceAccess(barriers[i].Transition.StateAfter);
state.pending_split = barriers[i].Flags == D3D12_RESOURCE_BARRIER_FLAG_BEGIN_ONLY;
}
original_ResourceBarrier(access.target(), barrier_count, barriers);
}
void STDMETHODCALLTYPE hook_Barrier(ID3D12GraphicsCommandList* self, UINT32 barrier_groups_count, const D3D12_BARRIER_GROUP* barrier_groups) {
Access access(self);
if (access.recording) for (UINT32 g = 0; g < barrier_groups_count; ++g) if (barrier_groups[g].Type == D3D12_BARRIER_TYPE_TEXTURE)
for (UINT32 b = 0; b < barrier_groups[g].NumBarriers; ++b) {
const auto& barrier = barrier_groups[g].pTextureBarriers[b]; const auto& range = barrier.Subresources;
const bool containsZero = range.NumMipLevels == 0 ? (range.IndexOrFirstMipLevel == 0 || range.IndexOrFirstMipLevel == UINT32_MAX) :
range.IndexOrFirstMipLevel == 0 && range.FirstArraySlice == 0 && range.FirstPlane == 0;
if (containsZero) {
auto& tracked = access.recording->resource_states[barrier.pResource];
auto& state = tracked.state; state.enhanced = true; state.inherited = false;
state.layout = barrier.LayoutAfter; state.access = barrier.AccessAfter; state.sync = barrier.SyncAfter;
state.pending_split = barrier.SyncAfter == D3D12_BARRIER_SYNC_SPLIT;
}
}
original_Barrier(access.target(), barrier_groups_count, barrier_groups);
}
void STDMETHODCALLTYPE hook_BeginEvent(ID3D12GraphicsCommandList* self, UINT metadata, const void* data, UINT size) {
Access access(self);
if (access.recording) access.recording->events.push_back({metadata, copy_values(static_cast<const uint8_t*>(data), size)});
original_BeginEvent(access.target(), metadata, data, size);
}
void STDMETHODCALLTYPE hook_EndEvent(ID3D12GraphicsCommandList* self) {
Access access(self);
if (access.recording && !access.recording->events.empty()) access.recording->events.pop_back();
original_EndEvent(access.target());
}
void STDMETHODCALLTYPE hook_BeginRenderPass(ID3D12GraphicsCommandList* self, UINT render_target_count,
const D3D12_RENDER_PASS_RENDER_TARGET_DESC* render_targets, const D3D12_RENDER_PASS_DEPTH_STENCIL_DESC* depth_stencil, D3D12_RENDER_PASS_FLAGS flags) {
Access access(self); if (access.recording) access.recording->render_pass = true;
original_BeginRenderPass(access.target(), render_target_count, render_targets, depth_stencil, flags);
}
void STDMETHODCALLTYPE hook_EndRenderPass(ID3D12GraphicsCommandList* self) {
Access access(self); if (access.recording) access.recording->render_pass = false;
original_EndRenderPass(access.target());
}
void STDMETHODCALLTYPE hook_BeginQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) {
Access access(self); if (access.recording) ++access.recording->active_queries;
original_BeginQuery(access.target(), heap, type, index);
}
void STDMETHODCALLTYPE hook_EndQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap* heap, D3D12_QUERY_TYPE type, UINT index) {
Access access(self); if (access.recording && access.recording->active_queries) --access.recording->active_queries;
original_EndQuery(access.target(), heap, type, index);
}
void STDMETHODCALLTYPE hook_SetPredication(ID3D12GraphicsCommandList* self, ID3D12Resource* buffer, UINT64 aligned_buffer_offset, D3D12_PREDICATION_OP operation) {
Access access(self);
if (access.recording) {
access.recording->predication = buffer != nullptr;
auto retained = retain(buffer);
access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) { original_SetPredication(target, retained.Get(), aligned_buffer_offset, operation); });
}
original_SetPredication(access.target(), buffer, aligned_buffer_offset, operation);
}
void STDMETHODCALLTYPE hook_ExecuteBundle(ID3D12GraphicsCommandList* self, ID3D12GraphicsCommandList* command_list) {
Access access(self);
auto bundle = get_recording(command_list, false);
if (access.recording && bundle) {
std::lock_guard<std::recursive_mutex> lock(bundle->mutex);
access.recording->state.insert(access.recording->state.end(), bundle->state.begin(), bundle->state.end());
access.recording->indirect_state_unknown |= bundle->indirect_state_unknown;
} else if (access.recording) access.recording->indirect_state_unknown = true;
original_ExecuteBundle(access.target(), command_list);
}
void STDMETHODCALLTYPE hook_ExecuteIndirect(ID3D12GraphicsCommandList* self, ID3D12CommandSignature* command_signature,
UINT max_command_count, ID3D12Resource* arg_buffer, UINT64 arg_buffer_offset, ID3D12Resource* count_buffer, UINT64 count_buffer_offset) {
Access access(self);
if (access.recording) {
UINT bytes = 0;
(void)command_signature->GetPrivateData(signature_description_tag, &bytes, nullptr);
if (bytes < sizeof(UINT) || bytes > sizeof(UINT) + 4096 * sizeof(D3D12_INDIRECT_ARGUMENT_DESC)) {
access.recording->indirect_state_unknown = true;
} else {
std::vector<uint8_t> description(bytes);
UINT count = 0;
const auto result = command_signature->GetPrivateData(signature_description_tag, &bytes, description.data());
std::memcpy(&count, description.data(), sizeof(count));
if (FAILED(result) || count > 4096 || bytes != sizeof(UINT) + count * sizeof(D3D12_INDIRECT_ARGUMENT_DESC)) access.recording->indirect_state_unknown = true;
else {
std::vector<D3D12_INDIRECT_ARGUMENT_DESC> arguments(count);
std::memcpy(arguments.data(), description.data() + sizeof(UINT), count * sizeof(D3D12_INDIRECT_ARGUMENT_DESC));
bool compute = false;
for (const auto& arg : arguments) compute |= arg.Type == D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH || arg.Type == D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH_RAYS;
// Microsoft documents that only bindings modified by the
// signature are reset to zero/NULL after ExecuteIndirect.
access.recording->state.emplace_back([arguments, compute](ID3D12GraphicsCommandList* target) {
for (const auto& arg : arguments) switch (arg.Type) {
// Public SDK addition after the pinned MinGW header:
// INCREMENTING_CONSTANT has the same first two UINTs.
case static_cast<D3D12_INDIRECT_ARGUMENT_TYPE>(11):
(compute ? original_SetComputeRoot32BitConstant : original_SetGraphicsRoot32BitConstant)
(target, arg.Constant.RootParameterIndex, 0, arg.Constant.DestOffsetIn32BitValues); break;
case D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT: {
std::vector<UINT> zeros(arg.Constant.Num32BitValuesToSet);
auto setter = compute ? original_SetComputeRoot32BitConstants : original_SetGraphicsRoot32BitConstants;
setter(target, arg.Constant.RootParameterIndex, UINT(zeros.size()), zeros.data(), arg.Constant.DestOffsetIn32BitValues); break;
}
case D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT_BUFFER_VIEW:
(compute ? original_SetComputeRootConstantBufferView : original_SetGraphicsRootConstantBufferView)(target, arg.ConstantBufferView.RootParameterIndex, 0); break;
case D3D12_INDIRECT_ARGUMENT_TYPE_SHADER_RESOURCE_VIEW:
(compute ? original_SetComputeRootShaderResourceView : original_SetGraphicsRootShaderResourceView)(target, arg.ShaderResourceView.RootParameterIndex, 0); break;
case D3D12_INDIRECT_ARGUMENT_TYPE_UNORDERED_ACCESS_VIEW:
(compute ? original_SetComputeRootUnorderedAccessView : original_SetGraphicsRootUnorderedAccessView)(target, arg.UnorderedAccessView.RootParameterIndex, 0); break;
case D3D12_INDIRECT_ARGUMENT_TYPE_INDEX_BUFFER_VIEW: original_IASetIndexBuffer(target, nullptr); break;
case D3D12_INDIRECT_ARGUMENT_TYPE_VERTEX_BUFFER_VIEW: {
D3D12_VERTEX_BUFFER_VIEW empty{}; original_IASetVertexBuffers(target, arg.VertexBuffer.Slot, 1, &empty); break;
}
default: break;
}
});
}
}
}
original_ExecuteIndirect(access.target(), command_signature, max_command_count, arg_buffer, arg_buffer_offset, count_buffer, count_buffer_offset);
}
using ExecuteFn = void(STDMETHODCALLTYPE*)(ID3D12CommandQueue*, UINT, ID3D12CommandList* const*);
ExecuteFn original_execute = nullptr;
// Queue-owned synchronization metadata has no reference back to the queue.
// It serializes a split submission against other threads using the same queue.
static constexpr GUID submission_lock_tag = {0x6c6158ba,0x209f,0x4a09,{0x97,0x10,0xa7,0x03,0x52,0x32,0xa0,0x69}};
std::mutex queue_metadata_mutex;
struct SubmissionLock final : IUnknown {
std::atomic<ULONG> references{1};
std::recursive_mutex mutex;
ComPtr<ID3D12Fence> completion_fence;
Handle completion_event;
uint64_t completion_value = 0;
HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, void** out) override {
if (!out) return E_POINTER; *out = nullptr;
if (iid != IID_IUnknown) return E_NOINTERFACE;
*out = static_cast<IUnknown*>(this); AddRef(); return S_OK;
}
ULONG STDMETHODCALLTYPE AddRef() override { return ++references; }
ULONG STDMETHODCALLTYPE Release() override { const auto n = --references; if (!n) delete this; return n; }
};
ComPtr<SubmissionLock> submission_lock(ID3D12CommandQueue* queue) {
std::lock_guard<std::mutex> lock(queue_metadata_mutex);
ComPtr<SubmissionLock> result; UINT bytes = sizeof(IUnknown*);
if (SUCCEEDED(queue->GetPrivateData(submission_lock_tag, &bytes, result.GetAddressOf()))) return result;
result.Attach(new SubmissionLock);
dx(queue->SetPrivateDataInterface(submission_lock_tag, result.Get()), "Queue submission metadata");
return result;
}
using QueueFenceFn = HRESULT(STDMETHODCALLTYPE*)(ID3D12CommandQueue*, ID3D12Fence*, UINT64);
QueueFenceFn original_queue_signal = nullptr, original_queue_wait = nullptr;
HRESULT queue_fence(QueueFenceFn function, ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) noexcept {
if (internal) return function(queue, fence, value);
try {
auto queueLock = submission_lock(queue);
std::lock_guard<std::recursive_mutex> ordered(queueLock->mutex);
return function(queue, fence, value);
} catch (const std::exception& failure) {
{ std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
std::fprintf(stderr, "D4R_QUEUE_FENCE_FAILURE %s\n", failure.what()); return E_FAIL;
}
}
HRESULT STDMETHODCALLTYPE hook_queue_signal(ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) {
return queue_fence(original_queue_signal, queue, fence, value);
}
HRESULT STDMETHODCALLTYPE hook_queue_wait(ID3D12CommandQueue* queue, ID3D12Fence* fence, UINT64 value) {
return queue_fence(original_queue_wait, queue, fence, value);
}
void complete(ID3D12CommandQueue* queue) {
auto owner = submission_lock(queue);
std::lock_guard<std::recursive_mutex> ordered(owner->mutex);
if (!owner->completion_fence) {
ComPtr<ID3D12Device> device; dx(queue->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Queue device(submit drain)");
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(owner->completion_fence.GetAddressOf())), "Queue fence(submit drain)");
}
if (!owner->completion_event.value) owner->completion_event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!owner->completion_event.value) throw std::runtime_error("CreateEvent(submit drain)");
const auto value = ++owner->completion_value;
dx(queue->Signal(owner->completion_fence.Get(), value), "Queue Signal(submit drain)");
if (owner->completion_fence->GetCompletedValue() < value) {
dx(owner->completion_fence->SetEventOnCompletion(value, owner->completion_event.value), "Queue event(submit drain)");
if (WaitForSingleObject(owner->completion_event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("GPU submit boundary timed out");
}
}
void retire(ID3D12CommandQueue* queue, std::vector<std::shared_ptr<Recording>> owners) {
auto owner = submission_lock(queue);
if (!owner->completion_fence) {
ComPtr<ID3D12Device> device; dx(queue->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Retirement device");
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(owner->completion_fence.GetAddressOf())), "Retirement fence");
}
const auto value = ++owner->completion_value;
dx(queue->Signal(owner->completion_fence.Get(), value), "Queue Signal(retirement)");
GpuRetirement::instance().submit(owner->completion_fence.Get(), value,
std::make_shared<std::vector<std::shared_ptr<Recording>>>(std::move(owners)));
}
void STDMETHODCALLTYPE hook_execute(ID3D12CommandQueue* queue, UINT count, ID3D12CommandList* const* lists) {
if (internal) { original_execute(queue, count, lists); return; }
const auto submitStart = Clock::now();
double publishMs = 0, boundaryMs = 0, drainMs = 0;
std::vector<std::shared_ptr<Recording>> owners;
std::vector<ID3D12CommandList*> pending;
std::vector<ResourceStates> pending_states;
InternalScope scope;
auto flush = [&] {
if (!pending.empty()) {
original_execute(queue, UINT(pending.size()), pending.data());
// Publish in the actual submitted order, never while recording.
// The callback's external fence waits for these producers on GPU.
const auto publishStart = Clock::now();
for (const auto& states : pending_states) publish_states(states);
publishMs += milliseconds(publishStart);
pending.clear(); pending_states.clear();
}
};
bool boundaries = false;
try {
auto queueLock = submission_lock(queue);
std::lock_guard<std::recursive_mutex> ordered(queueLock->mutex);
for (UINT i = 0; i < count; ++i) {
std::shared_ptr<Recording> recording;
{ std::lock_guard<std::mutex> lock(map_mutex);
auto found = recordings.find(static_cast<ID3D12GraphicsCommandList*>(lists[i]));
if (found != recordings.end()) recording = found->second; }
if (!recording) { pending.push_back(lists[i]); pending_states.emplace_back(); continue; }
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
if (recording->segments.empty()) {
pending.push_back(lists[i]); pending_states.push_back(recording->resource_states); continue;
}
if (!recording->closed) throw std::runtime_error("Logical command list submitted before Close");
owners.push_back(recording); boundaries = true;
for (const auto& segment : recording->segments) {
pending.push_back(segment.list); pending_states.push_back(segment.resource_states); flush();
const auto boundaryStart = Clock::now();
segment.boundary(queue);
boundaryMs += milliseconds(boundaryStart);
}
pending.push_back(recording->current.Get());
pending_states.push_back(recording->resource_states);
}
flush();
if (boundaries) {
const auto drainStart = Clock::now();
if (std::getenv("D4R_ASYNC_INTEROP")) retire(queue, std::move(owners)); else complete(queue);
drainMs = milliseconds(drainStart);
if (profile_commands()) std::printf("D4R_COMMAND_SUBMIT total_ms=%.3f boundary_ms=%.3f publish_ms=%.3f suffix_drain_ms=%.3f\n",
milliseconds(submitStart), boundaryMs, publishMs, drainMs);
}
} catch (const std::exception& failure) {
{ std::lock_guard<std::mutex> lock(error_mutex); last_error = failure.what(); }
std::fprintf(stderr, "D4R_D3D12_SUBMIT_FAILURE %s\n", failure.what());
try { complete(queue); } catch (...) {}
// Do not submit a consumer suffix after a failed DLSS boundary.
}
}
}
InternalScope::InternalScope() : previous(internal) { internal = true; }
InternalScope::~InternalScope() { internal = previous; }
std::string error() { std::lock_guard<std::mutex> lock(error_mutex); return last_error; }
size_t live_recordings() { std::lock_guard<std::mutex> lock(map_mutex); return recordings.size(); }
D3D12_RESOURCE_STATES resource_state(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES fallback) {
const auto state = resource_access(list, resource, ResourceAccess(fallback));
if (state.enhanced) throw std::runtime_error("Enhanced texture layout cannot be treated as a legacy state");
return state.legacy;
}
ResourceAccess resource_access(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, ResourceAccess fallback) {
auto recording = get_recording(list, false);
fallback.inherited = true;
if (!recording) return fallback;
std::lock_guard<std::recursive_mutex> lock(recording->mutex);
auto found = recording->resource_states.find(resource);
return found == recording->resource_states.end() ? fallback : found->second.state;
}
ResourceAccess submitted_resource_access(ID3D12Resource* resource, ResourceAccess fallback) {
if (!resource) return fallback;
ResourceAccess submitted; UINT size = sizeof(submitted);
if (SUCCEEDED(resource->GetPrivateData(submitted_access_tag, &size, &submitted)) && size == sizeof(submitted)) return submitted;
return fallback;
}
void install(ID3D12Device* device) {
std::lock_guard<std::mutex> lock(installation_mutex);
if (installed) return;
InternalScope scope;
// An application may unload its NGX handle while native D3D12 objects are
// still alive. Public method callbacks must remain callable until exit.
HMODULE owner = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_PIN,
reinterpret_cast<LPCWSTR>(&install), &owner)) throw std::runtime_error("Cannot retain the D3D12 callback module");
const auto init = MH_Initialize();
if (init != MH_OK && init != MH_ERROR_ALREADY_INITIALIZED) throw std::runtime_error(MH_StatusToString(init));
ComPtr<ID3D12CommandAllocator> allocator; ComPtr<ID3D12GraphicsCommandList> list;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Create hook allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Create hook command list");
try {
// The public COM interface inherits one ordered method table.
unsigned version = 0;
ComPtr<IUnknown> newest;
const GUID* ids[] = {&IID_ID3D12GraphicsCommandList, &IID_ID3D12GraphicsCommandList1, &IID_ID3D12GraphicsCommandList2,
&IID_ID3D12GraphicsCommandList3, &IID_ID3D12GraphicsCommandList4, &IID_ID3D12GraphicsCommandList5,
&IID_ID3D12GraphicsCommandList6, &IID_ID3D12GraphicsCommandList7,
&command_list8_iid, &command_list9_iid, &command_list10_iid};
for (unsigned n = 0; n < 11; ++n) {
ComPtr<IUnknown> candidate;
if (SUCCEEDED(list->QueryInterface(*ids[n], reinterpret_cast<void**>(candidate.GetAddressOf())))) { newest = std::move(candidate); version = n; }
}
if (newest.Get() != static_cast<IUnknown*>(list.Get())) throw std::runtime_error("D3D12 command-list versions need a separate interface identity adapter");
auto table = *reinterpret_cast<void***>(list.Get());
attach_recording_methods(table, version);
attach_method(table[2], reinterpret_cast<void*>(hook_release), reinterpret_cast<void**>(&original_release), "Release");
D3D12_COMMAND_QUEUE_DESC desc{}; desc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
ComPtr<ID3D12CommandQueue> queue; dx(device->CreateCommandQueue(&desc, IID_PPV_ARGS(queue.GetAddressOf())), "Create hook queue");
auto queueTable = *reinterpret_cast<void***>(queue.Get());
// Queue inherits IUnknown, Object, DeviceChild; UpdateTileMappings,
// CopyTileMappings precede ExecuteCommandLists in the public SDK.
attach_method(queueTable[10], reinterpret_cast<void*>(hook_execute), reinterpret_cast<void**>(&original_execute), "ExecuteCommandLists");
attach_method(queueTable[queue_signal_slot], reinterpret_cast<void*>(hook_queue_signal), reinterpret_cast<void**>(&original_queue_signal), "QueueSignal");
attach_method(queueTable[queue_wait_slot], reinterpret_cast<void*>(hook_queue_wait), reinterpret_cast<void**>(&original_queue_wait), "QueueWait");
auto deviceTable = *reinterpret_cast<void***>(device);
attach_method(deviceTable[device_command_list_slot], reinterpret_cast<void*>(hook_create_list), reinterpret_cast<void**>(&original_create_list), "CreateCommandList");
attach_method(deviceTable[device_signature_slot], reinterpret_cast<void*>(hook_signature), reinterpret_cast<void**>(&original_signature), "CreateCommandSignature");
for (const auto& hook : attached) {
const auto result = MH_QueueEnableHook(hook.target);
if (result != MH_OK) throw std::runtime_error(MH_StatusToString(result));
}
const auto result = MH_ApplyQueued();
if (result != MH_OK) throw std::runtime_error(MH_StatusToString(result));
installed = true;
std::printf("D4R_D3D12_COMMAND_BACKEND installed=1 interface=%u hooks=%zu frame_age=0\n", version, attached.size());
} catch (...) {
for (const auto& hook : attached) { (void)MH_DisableHook(hook.target); (void)MH_RemoveHook(hook.target); }
attached.clear(); throw;
}
}
void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback) {
const auto recordStart = Clock::now();
if (!installed || !list || !callback) throw std::runtime_error("D3D12 command backend is not initialized");
if (!error().empty()) throw std::runtime_error(error());
if (list->GetType() != D3D12_COMMAND_LIST_TYPE_DIRECT) throw std::runtime_error("DLSS requires a direct command list");
Access access(list);
auto& recording = *access.recording;
if (recording.closed || recording.render_pass || recording.active_queries || recording.predication || recording.indirect_state_unknown)
throw std::runtime_error("Cannot split a closed list or an active render/query/predication/unknown indirect state scope");
// Construct/replay the suffix before modifying the original list so an
// allocation failure leaves the caller's recording intact.
InternalScope scope;
ComPtr<ID3D12Device> device; dx(list->GetDevice(IID_PPV_ARGS(device.GetAddressOf())), "Boundary device");
Segment segment; segment.list = access.target(); segment.owner = recording.current;
segment.allocator = recording.current_allocator; segment.boundary = std::move(callback);
segment.resource_states = recording.resource_states;
ComPtr<ID3D12CommandAllocator> nextAllocator; ComPtr<ID3D12GraphicsCommandList> next;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(nextAllocator.GetAddressOf())), "Suffix allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, nextAllocator.Get(), nullptr, IID_PPV_ARGS(next.GetAddressOf())), "Suffix command list");
const auto replayStart = Clock::now();
for (auto& state : recording.state) state(next.Get());
const auto replayMs = milliseconds(replayStart);
recording.segments.reserve(recording.segments.size() + 1);
for (const auto& event : recording.events) original_BeginEvent(next.Get(), event.metadata, event.data.data(), UINT(event.data.size()));
for (size_t n = 0; n < recording.events.size(); ++n) original_EndEvent(segment.list);
dx(original_Close(segment.list), "Close boundary prefix");
// Each suffix owns an independent allocator until GPU completion/reset.
recording.segments.push_back(std::move(segment));
recording.current_allocator = std::move(nextAllocator);
recording.current = std::move(next);
std::printf("D4R_D3D12_RECORD boundary=%zu state_calls=%zu frame_age=0\n", recording.segments.size(), recording.state.size());
if (profile_commands()) {
// This is the interval between NGX recording calls on this thread,
// not a Present/FPS measurement or an extra queued frame.
thread_local Clock::time_point previous{};
const double interval = previous == Clock::time_point{} ? 0 : std::chrono::duration<double, std::milli>(recordStart - previous).count();
previous = recordStart;
std::printf("D4R_COMMAND_RECORD total_ms=%.3f replay_ms=%.3f state_calls=%zu interval_ms=%.3f\n",
milliseconds(recordStart), replayMs, recording.state.size(), interval);
}
}
}
tools/windows/d3d12_command_hooks.generated.h
-766
View File
@@ -1,766 +0,0 @@
// Generated by generate_d3d12_hooks.py from the public SDK. Do not edit.
using Fn_Close = HRESULT(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self);
static Fn_Close original_Close = nullptr;
static HRESULT STDMETHODCALLTYPE hook_Close(ID3D12GraphicsCommandList* self);
using Fn_Reset = HRESULT(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12CommandAllocator *allocator, ID3D12PipelineState *initial_state);
static Fn_Reset original_Reset = nullptr;
static HRESULT STDMETHODCALLTYPE hook_Reset(ID3D12GraphicsCommandList* self, ID3D12CommandAllocator *allocator, ID3D12PipelineState *initial_state);
using Fn_ClearState = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12PipelineState *pipeline_state);
static Fn_ClearState original_ClearState = nullptr;
static void STDMETHODCALLTYPE hook_ClearState(ID3D12GraphicsCommandList* self, ID3D12PipelineState *pipeline_state);
using Fn_DrawInstanced = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT vertex_count_per_instance, UINT instance_count, UINT start_vertex_location, UINT start_instance_location);
static Fn_DrawInstanced original_DrawInstanced = nullptr;
static void STDMETHODCALLTYPE hook_DrawInstanced(ID3D12GraphicsCommandList* self, UINT vertex_count_per_instance, UINT instance_count, UINT start_vertex_location, UINT start_instance_location) {
Access access(self);
access.driver_call();
return original_DrawInstanced(access.target(), vertex_count_per_instance, instance_count, start_vertex_location, start_instance_location);
}
using Fn_DrawIndexedInstanced = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT index_count_per_instance, UINT instance_count, UINT start_vertex_location, INT base_vertex_location, UINT start_instance_location);
static Fn_DrawIndexedInstanced original_DrawIndexedInstanced = nullptr;
static void STDMETHODCALLTYPE hook_DrawIndexedInstanced(ID3D12GraphicsCommandList* self, UINT index_count_per_instance, UINT instance_count, UINT start_vertex_location, INT base_vertex_location, UINT start_instance_location) {
Access access(self);
access.driver_call();
return original_DrawIndexedInstanced(access.target(), index_count_per_instance, instance_count, start_vertex_location, base_vertex_location, start_instance_location);
}
using Fn_Dispatch = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT x, UINT u, UINT z);
static Fn_Dispatch original_Dispatch = nullptr;
static void STDMETHODCALLTYPE hook_Dispatch(ID3D12GraphicsCommandList* self, UINT x, UINT u, UINT z) {
Access access(self);
access.driver_call();
return original_Dispatch(access.target(), x, u, z);
}
using Fn_CopyBufferRegion = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT64 byte_count);
static Fn_CopyBufferRegion original_CopyBufferRegion = nullptr;
static void STDMETHODCALLTYPE hook_CopyBufferRegion(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT64 byte_count) {
Access access(self);
access.driver_call();
return original_CopyBufferRegion(access.target(), dst_buffer, dst_offset, src_buffer, src_offset, byte_count);
}
using Fn_CopyTextureRegion = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const D3D12_TEXTURE_COPY_LOCATION *dst, UINT dst_x, UINT dst_y, UINT dst_z, const D3D12_TEXTURE_COPY_LOCATION *src, const D3D12_BOX *src_box);
static Fn_CopyTextureRegion original_CopyTextureRegion = nullptr;
static void STDMETHODCALLTYPE hook_CopyTextureRegion(ID3D12GraphicsCommandList* self, const D3D12_TEXTURE_COPY_LOCATION *dst, UINT dst_x, UINT dst_y, UINT dst_z, const D3D12_TEXTURE_COPY_LOCATION *src, const D3D12_BOX *src_box) {
Access access(self);
access.driver_call();
return original_CopyTextureRegion(access.target(), dst, dst_x, dst_y, dst_z, src, src_box);
}
using Fn_CopyResource = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, ID3D12Resource *src_resource);
static Fn_CopyResource original_CopyResource = nullptr;
static void STDMETHODCALLTYPE hook_CopyResource(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, ID3D12Resource *src_resource) {
Access access(self);
access.driver_call();
return original_CopyResource(access.target(), dst_resource, src_resource);
}
using Fn_CopyTiles = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *tiled_resource, const D3D12_TILED_RESOURCE_COORDINATE *tile_region_start_coordinate, const D3D12_TILE_REGION_SIZE *tile_region_size, ID3D12Resource *buffer, UINT64 buffer_offset, D3D12_TILE_COPY_FLAGS flags);
static Fn_CopyTiles original_CopyTiles = nullptr;
static void STDMETHODCALLTYPE hook_CopyTiles(ID3D12GraphicsCommandList* self, ID3D12Resource *tiled_resource, const D3D12_TILED_RESOURCE_COORDINATE *tile_region_start_coordinate, const D3D12_TILE_REGION_SIZE *tile_region_size, ID3D12Resource *buffer, UINT64 buffer_offset, D3D12_TILE_COPY_FLAGS flags) {
Access access(self);
access.driver_call();
return original_CopyTiles(access.target(), tiled_resource, tile_region_start_coordinate, tile_region_size, buffer, buffer_offset, flags);
}
using Fn_ResolveSubresource = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, UINT dst_sub_resource, ID3D12Resource *src_resource, UINT src_sub_resource, DXGI_FORMAT format);
static Fn_ResolveSubresource original_ResolveSubresource = nullptr;
static void STDMETHODCALLTYPE hook_ResolveSubresource(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, UINT dst_sub_resource, ID3D12Resource *src_resource, UINT src_sub_resource, DXGI_FORMAT format) {
Access access(self);
access.driver_call();
return original_ResolveSubresource(access.target(), dst_resource, dst_sub_resource, src_resource, src_sub_resource, format);
}
using Fn_IASetPrimitiveTopology = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_PRIMITIVE_TOPOLOGY primitive_topology);
static Fn_IASetPrimitiveTopology original_IASetPrimitiveTopology = nullptr;
static void STDMETHODCALLTYPE hook_IASetPrimitiveTopology(ID3D12GraphicsCommandList* self, D3D12_PRIMITIVE_TOPOLOGY primitive_topology) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_IASetPrimitiveTopology(target, primitive_topology);
});
access.driver_call();
return original_IASetPrimitiveTopology(access.target(), primitive_topology);
}
using Fn_RSSetViewports = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT viewport_count, const D3D12_VIEWPORT *viewports);
static Fn_RSSetViewports original_RSSetViewports = nullptr;
static void STDMETHODCALLTYPE hook_RSSetViewports(ID3D12GraphicsCommandList* self, UINT viewport_count, const D3D12_VIEWPORT *viewports) {
Access access(self);
auto saved_viewports = copy_values(viewports, viewport_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_RSSetViewports(target, viewport_count, values_pointer(saved_viewports));
});
access.driver_call();
return original_RSSetViewports(access.target(), viewport_count, viewports);
}
using Fn_RSSetScissorRects = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT rect_count, const D3D12_RECT *rects);
static Fn_RSSetScissorRects original_RSSetScissorRects = nullptr;
static void STDMETHODCALLTYPE hook_RSSetScissorRects(ID3D12GraphicsCommandList* self, UINT rect_count, const D3D12_RECT *rects) {
Access access(self);
auto saved_rects = copy_values(rects, rect_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_RSSetScissorRects(target, rect_count, values_pointer(saved_rects));
});
access.driver_call();
return original_RSSetScissorRects(access.target(), rect_count, rects);
}
using Fn_OMSetBlendFactor = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const FLOAT blend_factor[4]);
static Fn_OMSetBlendFactor original_OMSetBlendFactor = nullptr;
static void STDMETHODCALLTYPE hook_OMSetBlendFactor(ID3D12GraphicsCommandList* self, const FLOAT blend_factor[4]) {
Access access(self);
auto saved_blend_factor = copy_values(blend_factor, 4);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_OMSetBlendFactor(target, values_pointer(saved_blend_factor));
});
access.driver_call();
return original_OMSetBlendFactor(access.target(), blend_factor);
}
using Fn_OMSetStencilRef = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT stencil_ref);
static Fn_OMSetStencilRef original_OMSetStencilRef = nullptr;
static void STDMETHODCALLTYPE hook_OMSetStencilRef(ID3D12GraphicsCommandList* self, UINT stencil_ref) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_OMSetStencilRef(target, stencil_ref);
});
access.driver_call();
return original_OMSetStencilRef(access.target(), stencil_ref);
}
using Fn_SetPipelineState = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12PipelineState *pipeline_state);
static Fn_SetPipelineState original_SetPipelineState = nullptr;
static void STDMETHODCALLTYPE hook_SetPipelineState(ID3D12GraphicsCommandList* self, ID3D12PipelineState *pipeline_state) {
Access access(self);
auto saved_pipeline_state = retain(pipeline_state);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetPipelineState(target, saved_pipeline_state.Get());
});
access.driver_call();
return original_SetPipelineState(access.target(), pipeline_state);
}
using Fn_ResourceBarrier = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT barrier_count, const D3D12_RESOURCE_BARRIER *barriers);
static Fn_ResourceBarrier original_ResourceBarrier = nullptr;
static void STDMETHODCALLTYPE hook_ResourceBarrier(ID3D12GraphicsCommandList* self, UINT barrier_count, const D3D12_RESOURCE_BARRIER *barriers);
using Fn_ExecuteBundle = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12GraphicsCommandList *command_list);
static Fn_ExecuteBundle original_ExecuteBundle = nullptr;
static void STDMETHODCALLTYPE hook_ExecuteBundle(ID3D12GraphicsCommandList* self, ID3D12GraphicsCommandList *command_list);
using Fn_SetDescriptorHeaps = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT heap_count, ID3D12DescriptorHeap *const *heaps);
static Fn_SetDescriptorHeaps original_SetDescriptorHeaps = nullptr;
static void STDMETHODCALLTYPE hook_SetDescriptorHeaps(ID3D12GraphicsCommandList* self, UINT heap_count, ID3D12DescriptorHeap *const *heaps) {
Access access(self);
auto saved_heaps = retain_values(heaps, heap_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetDescriptorHeaps(target, heap_count, saved_heaps.raw.data());
});
access.driver_call();
return original_SetDescriptorHeaps(access.target(), heap_count, heaps);
}
using Fn_SetComputeRootSignature = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12RootSignature *root_signature);
static Fn_SetComputeRootSignature original_SetComputeRootSignature = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRootSignature(ID3D12GraphicsCommandList* self, ID3D12RootSignature *root_signature) {
Access access(self);
auto saved_root_signature = retain(root_signature);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRootSignature(target, saved_root_signature.Get());
});
access.driver_call();
return original_SetComputeRootSignature(access.target(), root_signature);
}
using Fn_SetGraphicsRootSignature = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12RootSignature *root_signature);
static Fn_SetGraphicsRootSignature original_SetGraphicsRootSignature = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRootSignature(ID3D12GraphicsCommandList* self, ID3D12RootSignature *root_signature) {
Access access(self);
auto saved_root_signature = retain(root_signature);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRootSignature(target, saved_root_signature.Get());
});
access.driver_call();
return original_SetGraphicsRootSignature(access.target(), root_signature);
}
using Fn_SetComputeRootDescriptorTable = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_DESCRIPTOR_HANDLE base_descriptor);
static Fn_SetComputeRootDescriptorTable original_SetComputeRootDescriptorTable = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRootDescriptorTable(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_DESCRIPTOR_HANDLE base_descriptor) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRootDescriptorTable(target, root_parameter_index, base_descriptor);
});
access.driver_call();
return original_SetComputeRootDescriptorTable(access.target(), root_parameter_index, base_descriptor);
}
using Fn_SetGraphicsRootDescriptorTable = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_DESCRIPTOR_HANDLE base_descriptor);
static Fn_SetGraphicsRootDescriptorTable original_SetGraphicsRootDescriptorTable = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRootDescriptorTable(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_DESCRIPTOR_HANDLE base_descriptor) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRootDescriptorTable(target, root_parameter_index, base_descriptor);
});
access.driver_call();
return original_SetGraphicsRootDescriptorTable(access.target(), root_parameter_index, base_descriptor);
}
using Fn_SetComputeRoot32BitConstant = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT data, UINT dst_offset);
static Fn_SetComputeRoot32BitConstant original_SetComputeRoot32BitConstant = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRoot32BitConstant(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT data, UINT dst_offset) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRoot32BitConstant(target, root_parameter_index, data, dst_offset);
});
access.driver_call();
return original_SetComputeRoot32BitConstant(access.target(), root_parameter_index, data, dst_offset);
}
using Fn_SetGraphicsRoot32BitConstant = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT data, UINT dst_offset);
static Fn_SetGraphicsRoot32BitConstant original_SetGraphicsRoot32BitConstant = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRoot32BitConstant(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT data, UINT dst_offset) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRoot32BitConstant(target, root_parameter_index, data, dst_offset);
});
access.driver_call();
return original_SetGraphicsRoot32BitConstant(access.target(), root_parameter_index, data, dst_offset);
}
using Fn_SetComputeRoot32BitConstants = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT constant_count, const void *data, UINT dst_offset);
static Fn_SetComputeRoot32BitConstants original_SetComputeRoot32BitConstants = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRoot32BitConstants(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT constant_count, const void *data, UINT dst_offset) {
Access access(self);
auto saved_data = copy_values(static_cast<const UINT*>(data), constant_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRoot32BitConstants(target, root_parameter_index, constant_count, values_pointer(saved_data), dst_offset);
});
access.driver_call();
return original_SetComputeRoot32BitConstants(access.target(), root_parameter_index, constant_count, data, dst_offset);
}
using Fn_SetGraphicsRoot32BitConstants = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT constant_count, const void *data, UINT dst_offset);
static Fn_SetGraphicsRoot32BitConstants original_SetGraphicsRoot32BitConstants = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRoot32BitConstants(ID3D12GraphicsCommandList* self, UINT root_parameter_index, UINT constant_count, const void *data, UINT dst_offset) {
Access access(self);
auto saved_data = copy_values(static_cast<const UINT*>(data), constant_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRoot32BitConstants(target, root_parameter_index, constant_count, values_pointer(saved_data), dst_offset);
});
access.driver_call();
return original_SetGraphicsRoot32BitConstants(access.target(), root_parameter_index, constant_count, data, dst_offset);
}
using Fn_SetComputeRootConstantBufferView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetComputeRootConstantBufferView original_SetComputeRootConstantBufferView = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRootConstantBufferView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRootConstantBufferView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetComputeRootConstantBufferView(access.target(), root_parameter_index, address);
}
using Fn_SetGraphicsRootConstantBufferView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetGraphicsRootConstantBufferView original_SetGraphicsRootConstantBufferView = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRootConstantBufferView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRootConstantBufferView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetGraphicsRootConstantBufferView(access.target(), root_parameter_index, address);
}
using Fn_SetComputeRootShaderResourceView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetComputeRootShaderResourceView original_SetComputeRootShaderResourceView = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRootShaderResourceView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRootShaderResourceView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetComputeRootShaderResourceView(access.target(), root_parameter_index, address);
}
using Fn_SetGraphicsRootShaderResourceView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetGraphicsRootShaderResourceView original_SetGraphicsRootShaderResourceView = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRootShaderResourceView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRootShaderResourceView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetGraphicsRootShaderResourceView(access.target(), root_parameter_index, address);
}
using Fn_SetComputeRootUnorderedAccessView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetComputeRootUnorderedAccessView original_SetComputeRootUnorderedAccessView = nullptr;
static void STDMETHODCALLTYPE hook_SetComputeRootUnorderedAccessView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetComputeRootUnorderedAccessView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetComputeRootUnorderedAccessView(access.target(), root_parameter_index, address);
}
using Fn_SetGraphicsRootUnorderedAccessView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address);
static Fn_SetGraphicsRootUnorderedAccessView original_SetGraphicsRootUnorderedAccessView = nullptr;
static void STDMETHODCALLTYPE hook_SetGraphicsRootUnorderedAccessView(ID3D12GraphicsCommandList* self, UINT root_parameter_index, D3D12_GPU_VIRTUAL_ADDRESS address) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetGraphicsRootUnorderedAccessView(target, root_parameter_index, address);
});
access.driver_call();
return original_SetGraphicsRootUnorderedAccessView(access.target(), root_parameter_index, address);
}
using Fn_IASetIndexBuffer = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const D3D12_INDEX_BUFFER_VIEW *view);
static Fn_IASetIndexBuffer original_IASetIndexBuffer = nullptr;
static void STDMETHODCALLTYPE hook_IASetIndexBuffer(ID3D12GraphicsCommandList* self, const D3D12_INDEX_BUFFER_VIEW *view) {
Access access(self);
auto saved_view = copy_values(view, 1);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_IASetIndexBuffer(target, values_pointer(saved_view));
});
access.driver_call();
return original_IASetIndexBuffer(access.target(), view);
}
using Fn_IASetVertexBuffers = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT start_slot, UINT view_count, const D3D12_VERTEX_BUFFER_VIEW *views);
static Fn_IASetVertexBuffers original_IASetVertexBuffers = nullptr;
static void STDMETHODCALLTYPE hook_IASetVertexBuffers(ID3D12GraphicsCommandList* self, UINT start_slot, UINT view_count, const D3D12_VERTEX_BUFFER_VIEW *views) {
Access access(self);
auto saved_views = copy_values(views, view_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_IASetVertexBuffers(target, start_slot, view_count, values_pointer(saved_views));
});
access.driver_call();
return original_IASetVertexBuffers(access.target(), start_slot, view_count, views);
}
using Fn_SOSetTargets = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT start_slot, UINT view_count, const D3D12_STREAM_OUTPUT_BUFFER_VIEW *views);
static Fn_SOSetTargets original_SOSetTargets = nullptr;
static void STDMETHODCALLTYPE hook_SOSetTargets(ID3D12GraphicsCommandList* self, UINT start_slot, UINT view_count, const D3D12_STREAM_OUTPUT_BUFFER_VIEW *views) {
Access access(self);
auto saved_views = copy_values(views, view_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SOSetTargets(target, start_slot, view_count, values_pointer(saved_views));
});
access.driver_call();
return original_SOSetTargets(access.target(), start_slot, view_count, views);
}
using Fn_OMSetRenderTargets = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT render_target_descriptor_count, const D3D12_CPU_DESCRIPTOR_HANDLE *render_target_descriptors, BOOL single_descriptor_handle, const D3D12_CPU_DESCRIPTOR_HANDLE *depth_stencil_descriptor);
static Fn_OMSetRenderTargets original_OMSetRenderTargets = nullptr;
static void STDMETHODCALLTYPE hook_OMSetRenderTargets(ID3D12GraphicsCommandList* self, UINT render_target_descriptor_count, const D3D12_CPU_DESCRIPTOR_HANDLE *render_target_descriptors, BOOL single_descriptor_handle, const D3D12_CPU_DESCRIPTOR_HANDLE *depth_stencil_descriptor) {
Access access(self);
auto saved_render_target_descriptors = copy_values(render_target_descriptors, (single_descriptor_handle ? 1 : render_target_descriptor_count));
auto saved_depth_stencil_descriptor = copy_values(depth_stencil_descriptor, 1);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_OMSetRenderTargets(target, render_target_descriptor_count, values_pointer(saved_render_target_descriptors), single_descriptor_handle, values_pointer(saved_depth_stencil_descriptor));
});
access.driver_call();
return original_OMSetRenderTargets(access.target(), render_target_descriptor_count, render_target_descriptors, single_descriptor_handle, depth_stencil_descriptor);
}
using Fn_ClearDepthStencilView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_CPU_DESCRIPTOR_HANDLE dsv, D3D12_CLEAR_FLAGS flags, FLOAT depth, UINT8 stencil, UINT rect_count, const D3D12_RECT *rects);
static Fn_ClearDepthStencilView original_ClearDepthStencilView = nullptr;
static void STDMETHODCALLTYPE hook_ClearDepthStencilView(ID3D12GraphicsCommandList* self, D3D12_CPU_DESCRIPTOR_HANDLE dsv, D3D12_CLEAR_FLAGS flags, FLOAT depth, UINT8 stencil, UINT rect_count, const D3D12_RECT *rects) {
Access access(self);
access.driver_call();
return original_ClearDepthStencilView(access.target(), dsv, flags, depth, stencil, rect_count, rects);
}
using Fn_ClearRenderTargetView = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_CPU_DESCRIPTOR_HANDLE rtv, const FLOAT color[4], UINT rect_count, const D3D12_RECT *rects);
static Fn_ClearRenderTargetView original_ClearRenderTargetView = nullptr;
static void STDMETHODCALLTYPE hook_ClearRenderTargetView(ID3D12GraphicsCommandList* self, D3D12_CPU_DESCRIPTOR_HANDLE rtv, const FLOAT color[4], UINT rect_count, const D3D12_RECT *rects) {
Access access(self);
access.driver_call();
return original_ClearRenderTargetView(access.target(), rtv, color, rect_count, rects);
}
using Fn_ClearUnorderedAccessViewUint = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_GPU_DESCRIPTOR_HANDLE gpu_handle, D3D12_CPU_DESCRIPTOR_HANDLE cpu_handle, ID3D12Resource *resource, const UINT values[4], UINT rect_count, const D3D12_RECT *rects);
static Fn_ClearUnorderedAccessViewUint original_ClearUnorderedAccessViewUint = nullptr;
static void STDMETHODCALLTYPE hook_ClearUnorderedAccessViewUint(ID3D12GraphicsCommandList* self, D3D12_GPU_DESCRIPTOR_HANDLE gpu_handle, D3D12_CPU_DESCRIPTOR_HANDLE cpu_handle, ID3D12Resource *resource, const UINT values[4], UINT rect_count, const D3D12_RECT *rects) {
Access access(self);
access.driver_call();
return original_ClearUnorderedAccessViewUint(access.target(), gpu_handle, cpu_handle, resource, values, rect_count, rects);
}
using Fn_ClearUnorderedAccessViewFloat = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_GPU_DESCRIPTOR_HANDLE gpu_handle, D3D12_CPU_DESCRIPTOR_HANDLE cpu_handle, ID3D12Resource *resource, const float values[4], UINT rect_count, const D3D12_RECT *rects);
static Fn_ClearUnorderedAccessViewFloat original_ClearUnorderedAccessViewFloat = nullptr;
static void STDMETHODCALLTYPE hook_ClearUnorderedAccessViewFloat(ID3D12GraphicsCommandList* self, D3D12_GPU_DESCRIPTOR_HANDLE gpu_handle, D3D12_CPU_DESCRIPTOR_HANDLE cpu_handle, ID3D12Resource *resource, const float values[4], UINT rect_count, const D3D12_RECT *rects) {
Access access(self);
access.driver_call();
return original_ClearUnorderedAccessViewFloat(access.target(), gpu_handle, cpu_handle, resource, values, rect_count, rects);
}
using Fn_DiscardResource = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *resource, const D3D12_DISCARD_REGION *region);
static Fn_DiscardResource original_DiscardResource = nullptr;
static void STDMETHODCALLTYPE hook_DiscardResource(ID3D12GraphicsCommandList* self, ID3D12Resource *resource, const D3D12_DISCARD_REGION *region) {
Access access(self);
access.driver_call();
return original_DiscardResource(access.target(), resource, region);
}
using Fn_BeginQuery = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT index);
static Fn_BeginQuery original_BeginQuery = nullptr;
static void STDMETHODCALLTYPE hook_BeginQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT index);
using Fn_EndQuery = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT index);
static Fn_EndQuery original_EndQuery = nullptr;
static void STDMETHODCALLTYPE hook_EndQuery(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT index);
using Fn_ResolveQueryData = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT start_index, UINT query_count, ID3D12Resource *dst_buffer, UINT64 aligned_dst_buffer_offset);
static Fn_ResolveQueryData original_ResolveQueryData = nullptr;
static void STDMETHODCALLTYPE hook_ResolveQueryData(ID3D12GraphicsCommandList* self, ID3D12QueryHeap *heap, D3D12_QUERY_TYPE type, UINT start_index, UINT query_count, ID3D12Resource *dst_buffer, UINT64 aligned_dst_buffer_offset) {
Access access(self);
access.driver_call();
return original_ResolveQueryData(access.target(), heap, type, start_index, query_count, dst_buffer, aligned_dst_buffer_offset);
}
using Fn_SetPredication = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *buffer, UINT64 aligned_buffer_offset, D3D12_PREDICATION_OP operation);
static Fn_SetPredication original_SetPredication = nullptr;
static void STDMETHODCALLTYPE hook_SetPredication(ID3D12GraphicsCommandList* self, ID3D12Resource *buffer, UINT64 aligned_buffer_offset, D3D12_PREDICATION_OP operation);
using Fn_SetMarker = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT metadata, const void *data, UINT size);
static Fn_SetMarker original_SetMarker = nullptr;
static void STDMETHODCALLTYPE hook_SetMarker(ID3D12GraphicsCommandList* self, UINT metadata, const void *data, UINT size) {
Access access(self);
access.driver_call();
return original_SetMarker(access.target(), metadata, data, size);
}
using Fn_BeginEvent = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT metadata, const void *data, UINT size);
static Fn_BeginEvent original_BeginEvent = nullptr;
static void STDMETHODCALLTYPE hook_BeginEvent(ID3D12GraphicsCommandList* self, UINT metadata, const void *data, UINT size);
using Fn_EndEvent = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self);
static Fn_EndEvent original_EndEvent = nullptr;
static void STDMETHODCALLTYPE hook_EndEvent(ID3D12GraphicsCommandList* self);
using Fn_ExecuteIndirect = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12CommandSignature *command_signature, UINT max_command_count, ID3D12Resource *arg_buffer, UINT64 arg_buffer_offset, ID3D12Resource *count_buffer, UINT64 count_buffer_offset);
static Fn_ExecuteIndirect original_ExecuteIndirect = nullptr;
static void STDMETHODCALLTYPE hook_ExecuteIndirect(ID3D12GraphicsCommandList* self, ID3D12CommandSignature *command_signature, UINT max_command_count, ID3D12Resource *arg_buffer, UINT64 arg_buffer_offset, ID3D12Resource *count_buffer, UINT64 count_buffer_offset);
using Fn_AtomicCopyBufferUINT = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT dependent_resource_count, ID3D12Resource *const *dependent_resources, const D3D12_SUBRESOURCE_RANGE_UINT64 *dependent_sub_resource_ranges);
static Fn_AtomicCopyBufferUINT original_AtomicCopyBufferUINT = nullptr;
static void STDMETHODCALLTYPE hook_AtomicCopyBufferUINT(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT dependent_resource_count, ID3D12Resource *const *dependent_resources, const D3D12_SUBRESOURCE_RANGE_UINT64 *dependent_sub_resource_ranges) {
Access access(self);
access.driver_call();
return original_AtomicCopyBufferUINT(access.target(), dst_buffer, dst_offset, src_buffer, src_offset, dependent_resource_count, dependent_resources, dependent_sub_resource_ranges);
}
using Fn_AtomicCopyBufferUINT64 = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT dependent_resource_count, ID3D12Resource *const *dependent_resources, const D3D12_SUBRESOURCE_RANGE_UINT64 *dependent_sub_resource_ranges);
static Fn_AtomicCopyBufferUINT64 original_AtomicCopyBufferUINT64 = nullptr;
static void STDMETHODCALLTYPE hook_AtomicCopyBufferUINT64(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_buffer, UINT64 dst_offset, ID3D12Resource *src_buffer, UINT64 src_offset, UINT dependent_resource_count, ID3D12Resource *const *dependent_resources, const D3D12_SUBRESOURCE_RANGE_UINT64 *dependent_sub_resource_ranges) {
Access access(self);
access.driver_call();
return original_AtomicCopyBufferUINT64(access.target(), dst_buffer, dst_offset, src_buffer, src_offset, dependent_resource_count, dependent_resources, dependent_sub_resource_ranges);
}
using Fn_OMSetDepthBounds = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, FLOAT min, FLOAT max);
static Fn_OMSetDepthBounds original_OMSetDepthBounds = nullptr;
static void STDMETHODCALLTYPE hook_OMSetDepthBounds(ID3D12GraphicsCommandList* self, FLOAT min, FLOAT max) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_OMSetDepthBounds(target, min, max);
});
access.driver_call();
return original_OMSetDepthBounds(access.target(), min, max);
}
using Fn_SetSamplePositions = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT sample_count, UINT pixel_count, D3D12_SAMPLE_POSITION *sample_positions);
static Fn_SetSamplePositions original_SetSamplePositions = nullptr;
static void STDMETHODCALLTYPE hook_SetSamplePositions(ID3D12GraphicsCommandList* self, UINT sample_count, UINT pixel_count, D3D12_SAMPLE_POSITION *sample_positions) {
Access access(self);
auto saved_sample_positions = copy_values(sample_positions, sample_count * pixel_count);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetSamplePositions(target, sample_count, pixel_count, values_pointer(saved_sample_positions));
});
access.driver_call();
return original_SetSamplePositions(access.target(), sample_count, pixel_count, sample_positions);
}
using Fn_ResolveSubresourceRegion = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, UINT dst_sub_resource_idx, UINT dst_x, UINT dst_y, ID3D12Resource *src_resource, UINT src_sub_resource_idx, D3D12_RECT *src_rect, DXGI_FORMAT format, D3D12_RESOLVE_MODE mode);
static Fn_ResolveSubresourceRegion original_ResolveSubresourceRegion = nullptr;
static void STDMETHODCALLTYPE hook_ResolveSubresourceRegion(ID3D12GraphicsCommandList* self, ID3D12Resource *dst_resource, UINT dst_sub_resource_idx, UINT dst_x, UINT dst_y, ID3D12Resource *src_resource, UINT src_sub_resource_idx, D3D12_RECT *src_rect, DXGI_FORMAT format, D3D12_RESOLVE_MODE mode) {
Access access(self);
access.driver_call();
return original_ResolveSubresourceRegion(access.target(), dst_resource, dst_sub_resource_idx, dst_x, dst_y, src_resource, src_sub_resource_idx, src_rect, format, mode);
}
using Fn_SetViewInstanceMask = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT mask);
static Fn_SetViewInstanceMask original_SetViewInstanceMask = nullptr;
static void STDMETHODCALLTYPE hook_SetViewInstanceMask(ID3D12GraphicsCommandList* self, UINT mask) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetViewInstanceMask(target, mask);
});
access.driver_call();
return original_SetViewInstanceMask(access.target(), mask);
}
using Fn_WriteBufferImmediate = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT count, const D3D12_WRITEBUFFERIMMEDIATE_PARAMETER *parameters, const D3D12_WRITEBUFFERIMMEDIATE_MODE *modes);
static Fn_WriteBufferImmediate original_WriteBufferImmediate = nullptr;
static void STDMETHODCALLTYPE hook_WriteBufferImmediate(ID3D12GraphicsCommandList* self, UINT count, const D3D12_WRITEBUFFERIMMEDIATE_PARAMETER *parameters, const D3D12_WRITEBUFFERIMMEDIATE_MODE *modes) {
Access access(self);
access.driver_call();
return original_WriteBufferImmediate(access.target(), count, parameters, modes);
}
using Fn_SetProtectedResourceSession = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12ProtectedResourceSession *protected_resource_session);
static Fn_SetProtectedResourceSession original_SetProtectedResourceSession = nullptr;
static void STDMETHODCALLTYPE hook_SetProtectedResourceSession(ID3D12GraphicsCommandList* self, ID3D12ProtectedResourceSession *protected_resource_session) {
Access access(self);
auto saved_protected_resource_session = retain(protected_resource_session);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetProtectedResourceSession(target, saved_protected_resource_session.Get());
});
access.driver_call();
return original_SetProtectedResourceSession(access.target(), protected_resource_session);
}
using Fn_BeginRenderPass = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT render_targets_count, const D3D12_RENDER_PASS_RENDER_TARGET_DESC *render_targets, const D3D12_RENDER_PASS_DEPTH_STENCIL_DESC *depth_stencil, D3D12_RENDER_PASS_FLAGS flags);
static Fn_BeginRenderPass original_BeginRenderPass = nullptr;
static void STDMETHODCALLTYPE hook_BeginRenderPass(ID3D12GraphicsCommandList* self, UINT render_targets_count, const D3D12_RENDER_PASS_RENDER_TARGET_DESC *render_targets, const D3D12_RENDER_PASS_DEPTH_STENCIL_DESC *depth_stencil, D3D12_RENDER_PASS_FLAGS flags);
using Fn_EndRenderPass = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self);
static Fn_EndRenderPass original_EndRenderPass = nullptr;
static void STDMETHODCALLTYPE hook_EndRenderPass(ID3D12GraphicsCommandList* self);
using Fn_InitializeMetaCommand = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12MetaCommand *meta_command, const void *initialization_parameters_data, SIZE_T initialization_parameters_data_size_in_bytes);
static Fn_InitializeMetaCommand original_InitializeMetaCommand = nullptr;
static void STDMETHODCALLTYPE hook_InitializeMetaCommand(ID3D12GraphicsCommandList* self, ID3D12MetaCommand *meta_command, const void *initialization_parameters_data, SIZE_T initialization_parameters_data_size_in_bytes) {
Access access(self);
access.driver_call();
return original_InitializeMetaCommand(access.target(), meta_command, initialization_parameters_data, initialization_parameters_data_size_in_bytes);
}
using Fn_ExecuteMetaCommand = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12MetaCommand *meta_command, const void *execution_parameters_data, SIZE_T execution_parameters_data_size_in_bytes);
static Fn_ExecuteMetaCommand original_ExecuteMetaCommand = nullptr;
static void STDMETHODCALLTYPE hook_ExecuteMetaCommand(ID3D12GraphicsCommandList* self, ID3D12MetaCommand *meta_command, const void *execution_parameters_data, SIZE_T execution_parameters_data_size_in_bytes) {
Access access(self);
access.driver_call();
return original_ExecuteMetaCommand(access.target(), meta_command, execution_parameters_data, execution_parameters_data_size_in_bytes);
}
using Fn_BuildRaytracingAccelerationStructure = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_DESC *desc, UINT postbuild_info_descs_count, const D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC *postbuild_info_descs);
static Fn_BuildRaytracingAccelerationStructure original_BuildRaytracingAccelerationStructure = nullptr;
static void STDMETHODCALLTYPE hook_BuildRaytracingAccelerationStructure(ID3D12GraphicsCommandList* self, const D3D12_BUILD_RAYTRACING_ACCELERATION_STRUCTURE_DESC *desc, UINT postbuild_info_descs_count, const D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC *postbuild_info_descs) {
Access access(self);
access.driver_call();
return original_BuildRaytracingAccelerationStructure(access.target(), desc, postbuild_info_descs_count, postbuild_info_descs);
}
using Fn_EmitRaytracingAccelerationStructurePostbuildInfo = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC *desc, UINT src_acceleration_structures_count, const D3D12_GPU_VIRTUAL_ADDRESS *src_acceleration_structure_data);
static Fn_EmitRaytracingAccelerationStructurePostbuildInfo original_EmitRaytracingAccelerationStructurePostbuildInfo = nullptr;
static void STDMETHODCALLTYPE hook_EmitRaytracingAccelerationStructurePostbuildInfo(ID3D12GraphicsCommandList* self, const D3D12_RAYTRACING_ACCELERATION_STRUCTURE_POSTBUILD_INFO_DESC *desc, UINT src_acceleration_structures_count, const D3D12_GPU_VIRTUAL_ADDRESS *src_acceleration_structure_data) {
Access access(self);
access.driver_call();
return original_EmitRaytracingAccelerationStructurePostbuildInfo(access.target(), desc, src_acceleration_structures_count, src_acceleration_structure_data);
}
using Fn_CopyRaytracingAccelerationStructure = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_GPU_VIRTUAL_ADDRESS dst_acceleration_structure_data, D3D12_GPU_VIRTUAL_ADDRESS src_acceleration_structure_data, D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE mode);
static Fn_CopyRaytracingAccelerationStructure original_CopyRaytracingAccelerationStructure = nullptr;
static void STDMETHODCALLTYPE hook_CopyRaytracingAccelerationStructure(ID3D12GraphicsCommandList* self, D3D12_GPU_VIRTUAL_ADDRESS dst_acceleration_structure_data, D3D12_GPU_VIRTUAL_ADDRESS src_acceleration_structure_data, D3D12_RAYTRACING_ACCELERATION_STRUCTURE_COPY_MODE mode) {
Access access(self);
access.driver_call();
return original_CopyRaytracingAccelerationStructure(access.target(), dst_acceleration_structure_data, src_acceleration_structure_data, mode);
}
using Fn_SetPipelineState1 = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12StateObject *state_object);
static Fn_SetPipelineState1 original_SetPipelineState1 = nullptr;
static void STDMETHODCALLTYPE hook_SetPipelineState1(ID3D12GraphicsCommandList* self, ID3D12StateObject *state_object) {
Access access(self);
auto saved_state_object = retain(state_object);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_SetPipelineState1(target, saved_state_object.Get());
});
access.driver_call();
return original_SetPipelineState1(access.target(), state_object);
}
using Fn_DispatchRays = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const D3D12_DISPATCH_RAYS_DESC *desc);
static Fn_DispatchRays original_DispatchRays = nullptr;
static void STDMETHODCALLTYPE hook_DispatchRays(ID3D12GraphicsCommandList* self, const D3D12_DISPATCH_RAYS_DESC *desc) {
Access access(self);
access.driver_call();
return original_DispatchRays(access.target(), desc);
}
using Fn_RSSetShadingRate = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_SHADING_RATE base_shading_rate, const D3D12_SHADING_RATE_COMBINER *combiners);
static Fn_RSSetShadingRate original_RSSetShadingRate = nullptr;
static void STDMETHODCALLTYPE hook_RSSetShadingRate(ID3D12GraphicsCommandList* self, D3D12_SHADING_RATE base_shading_rate, const D3D12_SHADING_RATE_COMBINER *combiners) {
Access access(self);
auto saved_combiners = copy_values(combiners, 2);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_RSSetShadingRate(target, base_shading_rate, values_pointer(saved_combiners));
});
access.driver_call();
return original_RSSetShadingRate(access.target(), base_shading_rate, combiners);
}
using Fn_RSSetShadingRateImage = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, ID3D12Resource *shading_rate_image);
static Fn_RSSetShadingRateImage original_RSSetShadingRateImage = nullptr;
static void STDMETHODCALLTYPE hook_RSSetShadingRateImage(ID3D12GraphicsCommandList* self, ID3D12Resource *shading_rate_image) {
Access access(self);
auto saved_shading_rate_image = retain(shading_rate_image);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {
original_RSSetShadingRateImage(target, saved_shading_rate_image.Get());
});
access.driver_call();
return original_RSSetShadingRateImage(access.target(), shading_rate_image);
}
using Fn_DispatchMesh = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT thread_group_count_x, UINT thread_group_count_y, UINT thread_group_count_z);
static Fn_DispatchMesh original_DispatchMesh = nullptr;
static void STDMETHODCALLTYPE hook_DispatchMesh(ID3D12GraphicsCommandList* self, UINT thread_group_count_x, UINT thread_group_count_y, UINT thread_group_count_z) {
Access access(self);
access.driver_call();
return original_DispatchMesh(access.target(), thread_group_count_x, thread_group_count_y, thread_group_count_z);
}
using Fn_Barrier = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT32 barrier_groups_count, const D3D12_BARRIER_GROUP *barrier_groups);
static Fn_Barrier original_Barrier = nullptr;
static void STDMETHODCALLTYPE hook_Barrier(ID3D12GraphicsCommandList* self, UINT32 barrier_groups_count, const D3D12_BARRIER_GROUP *barrier_groups);
using Fn_OMSetFrontAndBackStencilRef = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, UINT front_stencil, UINT back_stencil);
static Fn_OMSetFrontAndBackStencilRef original_OMSetFrontAndBackStencilRef = nullptr;
static void STDMETHODCALLTYPE hook_OMSetFrontAndBackStencilRef(ID3D12GraphicsCommandList* self, UINT front_stencil, UINT back_stencil) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) {
original_OMSetFrontAndBackStencilRef(target, front_stencil, back_stencil);
});
access.driver_call();
original_OMSetFrontAndBackStencilRef(access.target(), front_stencil, back_stencil);
}
using Fn_RSSetDepthBias = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, FLOAT depth_bias, FLOAT clamp, FLOAT slope);
static Fn_RSSetDepthBias original_RSSetDepthBias = nullptr;
static void STDMETHODCALLTYPE hook_RSSetDepthBias(ID3D12GraphicsCommandList* self, FLOAT depth_bias, FLOAT clamp, FLOAT slope) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) {
original_RSSetDepthBias(target, depth_bias, clamp, slope);
});
access.driver_call();
original_RSSetDepthBias(access.target(), depth_bias, clamp, slope);
}
using Fn_IASetIndexBufferStripCutValue = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, D3D12_INDEX_BUFFER_STRIP_CUT_VALUE value);
static Fn_IASetIndexBufferStripCutValue original_IASetIndexBufferStripCutValue = nullptr;
static void STDMETHODCALLTYPE hook_IASetIndexBufferStripCutValue(ID3D12GraphicsCommandList* self, D3D12_INDEX_BUFFER_STRIP_CUT_VALUE value) {
Access access(self);
if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) {
original_IASetIndexBufferStripCutValue(target, value);
});
access.driver_call();
original_IASetIndexBufferStripCutValue(access.target(), value);
}
using Fn_SetProgram = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const NewProgramDescription* desc);
static Fn_SetProgram original_SetProgram = nullptr;
static void STDMETHODCALLTYPE hook_SetProgram(ID3D12GraphicsCommandList* self, const NewProgramDescription* desc);
using Fn_DispatchGraph = void(STDMETHODCALLTYPE*)(ID3D12GraphicsCommandList* self, const void* desc);
static Fn_DispatchGraph original_DispatchGraph = nullptr;
static void STDMETHODCALLTYPE hook_DispatchGraph(ID3D12GraphicsCommandList* self, const void* desc) {
Access access(self);
access.driver_call();
original_DispatchGraph(access.target(), desc);
}
static void attach_recording_methods(void** table, unsigned version) {
if (version >= 0) attach_method(table[9], reinterpret_cast<void*>(hook_Close), reinterpret_cast<void**>(&original_Close), "Close");
if (version >= 0) attach_method(table[10], reinterpret_cast<void*>(hook_Reset), reinterpret_cast<void**>(&original_Reset), "Reset");
if (version >= 0) attach_method(table[11], reinterpret_cast<void*>(hook_ClearState), reinterpret_cast<void**>(&original_ClearState), "ClearState");
if (version >= 0) attach_method(table[12], reinterpret_cast<void*>(hook_DrawInstanced), reinterpret_cast<void**>(&original_DrawInstanced), "DrawInstanced");
if (version >= 0) attach_method(table[13], reinterpret_cast<void*>(hook_DrawIndexedInstanced), reinterpret_cast<void**>(&original_DrawIndexedInstanced), "DrawIndexedInstanced");
if (version >= 0) attach_method(table[14], reinterpret_cast<void*>(hook_Dispatch), reinterpret_cast<void**>(&original_Dispatch), "Dispatch");
if (version >= 0) attach_method(table[15], reinterpret_cast<void*>(hook_CopyBufferRegion), reinterpret_cast<void**>(&original_CopyBufferRegion), "CopyBufferRegion");
if (version >= 0) attach_method(table[16], reinterpret_cast<void*>(hook_CopyTextureRegion), reinterpret_cast<void**>(&original_CopyTextureRegion), "CopyTextureRegion");
if (version >= 0) attach_method(table[17], reinterpret_cast<void*>(hook_CopyResource), reinterpret_cast<void**>(&original_CopyResource), "CopyResource");
if (version >= 0) attach_method(table[18], reinterpret_cast<void*>(hook_CopyTiles), reinterpret_cast<void**>(&original_CopyTiles), "CopyTiles");
if (version >= 0) attach_method(table[19], reinterpret_cast<void*>(hook_ResolveSubresource), reinterpret_cast<void**>(&original_ResolveSubresource), "ResolveSubresource");
if (version >= 0) attach_method(table[20], reinterpret_cast<void*>(hook_IASetPrimitiveTopology), reinterpret_cast<void**>(&original_IASetPrimitiveTopology), "IASetPrimitiveTopology");
if (version >= 0) attach_method(table[21], reinterpret_cast<void*>(hook_RSSetViewports), reinterpret_cast<void**>(&original_RSSetViewports), "RSSetViewports");
if (version >= 0) attach_method(table[22], reinterpret_cast<void*>(hook_RSSetScissorRects), reinterpret_cast<void**>(&original_RSSetScissorRects), "RSSetScissorRects");
if (version >= 0) attach_method(table[23], reinterpret_cast<void*>(hook_OMSetBlendFactor), reinterpret_cast<void**>(&original_OMSetBlendFactor), "OMSetBlendFactor");
if (version >= 0) attach_method(table[24], reinterpret_cast<void*>(hook_OMSetStencilRef), reinterpret_cast<void**>(&original_OMSetStencilRef), "OMSetStencilRef");
if (version >= 0) attach_method(table[25], reinterpret_cast<void*>(hook_SetPipelineState), reinterpret_cast<void**>(&original_SetPipelineState), "SetPipelineState");
if (version >= 0) attach_method(table[26], reinterpret_cast<void*>(hook_ResourceBarrier), reinterpret_cast<void**>(&original_ResourceBarrier), "ResourceBarrier");
if (version >= 0) attach_method(table[27], reinterpret_cast<void*>(hook_ExecuteBundle), reinterpret_cast<void**>(&original_ExecuteBundle), "ExecuteBundle");
if (version >= 0) attach_method(table[28], reinterpret_cast<void*>(hook_SetDescriptorHeaps), reinterpret_cast<void**>(&original_SetDescriptorHeaps), "SetDescriptorHeaps");
if (version >= 0) attach_method(table[29], reinterpret_cast<void*>(hook_SetComputeRootSignature), reinterpret_cast<void**>(&original_SetComputeRootSignature), "SetComputeRootSignature");
if (version >= 0) attach_method(table[30], reinterpret_cast<void*>(hook_SetGraphicsRootSignature), reinterpret_cast<void**>(&original_SetGraphicsRootSignature), "SetGraphicsRootSignature");
if (version >= 0) attach_method(table[31], reinterpret_cast<void*>(hook_SetComputeRootDescriptorTable), reinterpret_cast<void**>(&original_SetComputeRootDescriptorTable), "SetComputeRootDescriptorTable");
if (version >= 0) attach_method(table[32], reinterpret_cast<void*>(hook_SetGraphicsRootDescriptorTable), reinterpret_cast<void**>(&original_SetGraphicsRootDescriptorTable), "SetGraphicsRootDescriptorTable");
if (version >= 0) attach_method(table[33], reinterpret_cast<void*>(hook_SetComputeRoot32BitConstant), reinterpret_cast<void**>(&original_SetComputeRoot32BitConstant), "SetComputeRoot32BitConstant");
if (version >= 0) attach_method(table[34], reinterpret_cast<void*>(hook_SetGraphicsRoot32BitConstant), reinterpret_cast<void**>(&original_SetGraphicsRoot32BitConstant), "SetGraphicsRoot32BitConstant");
if (version >= 0) attach_method(table[35], reinterpret_cast<void*>(hook_SetComputeRoot32BitConstants), reinterpret_cast<void**>(&original_SetComputeRoot32BitConstants), "SetComputeRoot32BitConstants");
if (version >= 0) attach_method(table[36], reinterpret_cast<void*>(hook_SetGraphicsRoot32BitConstants), reinterpret_cast<void**>(&original_SetGraphicsRoot32BitConstants), "SetGraphicsRoot32BitConstants");
if (version >= 0) attach_method(table[37], reinterpret_cast<void*>(hook_SetComputeRootConstantBufferView), reinterpret_cast<void**>(&original_SetComputeRootConstantBufferView), "SetComputeRootConstantBufferView");
if (version >= 0) attach_method(table[38], reinterpret_cast<void*>(hook_SetGraphicsRootConstantBufferView), reinterpret_cast<void**>(&original_SetGraphicsRootConstantBufferView), "SetGraphicsRootConstantBufferView");
if (version >= 0) attach_method(table[39], reinterpret_cast<void*>(hook_SetComputeRootShaderResourceView), reinterpret_cast<void**>(&original_SetComputeRootShaderResourceView), "SetComputeRootShaderResourceView");
if (version >= 0) attach_method(table[40], reinterpret_cast<void*>(hook_SetGraphicsRootShaderResourceView), reinterpret_cast<void**>(&original_SetGraphicsRootShaderResourceView), "SetGraphicsRootShaderResourceView");
if (version >= 0) attach_method(table[41], reinterpret_cast<void*>(hook_SetComputeRootUnorderedAccessView), reinterpret_cast<void**>(&original_SetComputeRootUnorderedAccessView), "SetComputeRootUnorderedAccessView");
if (version >= 0) attach_method(table[42], reinterpret_cast<void*>(hook_SetGraphicsRootUnorderedAccessView), reinterpret_cast<void**>(&original_SetGraphicsRootUnorderedAccessView), "SetGraphicsRootUnorderedAccessView");
if (version >= 0) attach_method(table[43], reinterpret_cast<void*>(hook_IASetIndexBuffer), reinterpret_cast<void**>(&original_IASetIndexBuffer), "IASetIndexBuffer");
if (version >= 0) attach_method(table[44], reinterpret_cast<void*>(hook_IASetVertexBuffers), reinterpret_cast<void**>(&original_IASetVertexBuffers), "IASetVertexBuffers");
if (version >= 0) attach_method(table[45], reinterpret_cast<void*>(hook_SOSetTargets), reinterpret_cast<void**>(&original_SOSetTargets), "SOSetTargets");
if (version >= 0) attach_method(table[46], reinterpret_cast<void*>(hook_OMSetRenderTargets), reinterpret_cast<void**>(&original_OMSetRenderTargets), "OMSetRenderTargets");
if (version >= 0) attach_method(table[47], reinterpret_cast<void*>(hook_ClearDepthStencilView), reinterpret_cast<void**>(&original_ClearDepthStencilView), "ClearDepthStencilView");
if (version >= 0) attach_method(table[48], reinterpret_cast<void*>(hook_ClearRenderTargetView), reinterpret_cast<void**>(&original_ClearRenderTargetView), "ClearRenderTargetView");
if (version >= 0) attach_method(table[49], reinterpret_cast<void*>(hook_ClearUnorderedAccessViewUint), reinterpret_cast<void**>(&original_ClearUnorderedAccessViewUint), "ClearUnorderedAccessViewUint");
if (version >= 0) attach_method(table[50], reinterpret_cast<void*>(hook_ClearUnorderedAccessViewFloat), reinterpret_cast<void**>(&original_ClearUnorderedAccessViewFloat), "ClearUnorderedAccessViewFloat");
if (version >= 0) attach_method(table[51], reinterpret_cast<void*>(hook_DiscardResource), reinterpret_cast<void**>(&original_DiscardResource), "DiscardResource");
if (version >= 0) attach_method(table[52], reinterpret_cast<void*>(hook_BeginQuery), reinterpret_cast<void**>(&original_BeginQuery), "BeginQuery");
if (version >= 0) attach_method(table[53], reinterpret_cast<void*>(hook_EndQuery), reinterpret_cast<void**>(&original_EndQuery), "EndQuery");
if (version >= 0) attach_method(table[54], reinterpret_cast<void*>(hook_ResolveQueryData), reinterpret_cast<void**>(&original_ResolveQueryData), "ResolveQueryData");
if (version >= 0) attach_method(table[55], reinterpret_cast<void*>(hook_SetPredication), reinterpret_cast<void**>(&original_SetPredication), "SetPredication");
if (version >= 0) attach_method(table[56], reinterpret_cast<void*>(hook_SetMarker), reinterpret_cast<void**>(&original_SetMarker), "SetMarker");
if (version >= 0) attach_method(table[57], reinterpret_cast<void*>(hook_BeginEvent), reinterpret_cast<void**>(&original_BeginEvent), "BeginEvent");
if (version >= 0) attach_method(table[58], reinterpret_cast<void*>(hook_EndEvent), reinterpret_cast<void**>(&original_EndEvent), "EndEvent");
if (version >= 0) attach_method(table[59], reinterpret_cast<void*>(hook_ExecuteIndirect), reinterpret_cast<void**>(&original_ExecuteIndirect), "ExecuteIndirect");
if (version >= 1) attach_method(table[60], reinterpret_cast<void*>(hook_AtomicCopyBufferUINT), reinterpret_cast<void**>(&original_AtomicCopyBufferUINT), "AtomicCopyBufferUINT");
if (version >= 1) attach_method(table[61], reinterpret_cast<void*>(hook_AtomicCopyBufferUINT64), reinterpret_cast<void**>(&original_AtomicCopyBufferUINT64), "AtomicCopyBufferUINT64");
if (version >= 1) attach_method(table[62], reinterpret_cast<void*>(hook_OMSetDepthBounds), reinterpret_cast<void**>(&original_OMSetDepthBounds), "OMSetDepthBounds");
if (version >= 1) attach_method(table[63], reinterpret_cast<void*>(hook_SetSamplePositions), reinterpret_cast<void**>(&original_SetSamplePositions), "SetSamplePositions");
if (version >= 1) attach_method(table[64], reinterpret_cast<void*>(hook_ResolveSubresourceRegion), reinterpret_cast<void**>(&original_ResolveSubresourceRegion), "ResolveSubresourceRegion");
if (version >= 1) attach_method(table[65], reinterpret_cast<void*>(hook_SetViewInstanceMask), reinterpret_cast<void**>(&original_SetViewInstanceMask), "SetViewInstanceMask");
if (version >= 2) attach_method(table[66], reinterpret_cast<void*>(hook_WriteBufferImmediate), reinterpret_cast<void**>(&original_WriteBufferImmediate), "WriteBufferImmediate");
if (version >= 3) attach_method(table[67], reinterpret_cast<void*>(hook_SetProtectedResourceSession), reinterpret_cast<void**>(&original_SetProtectedResourceSession), "SetProtectedResourceSession");
if (version >= 4) attach_method(table[68], reinterpret_cast<void*>(hook_BeginRenderPass), reinterpret_cast<void**>(&original_BeginRenderPass), "BeginRenderPass");
if (version >= 4) attach_method(table[69], reinterpret_cast<void*>(hook_EndRenderPass), reinterpret_cast<void**>(&original_EndRenderPass), "EndRenderPass");
if (version >= 4) attach_method(table[70], reinterpret_cast<void*>(hook_InitializeMetaCommand), reinterpret_cast<void**>(&original_InitializeMetaCommand), "InitializeMetaCommand");
if (version >= 4) attach_method(table[71], reinterpret_cast<void*>(hook_ExecuteMetaCommand), reinterpret_cast<void**>(&original_ExecuteMetaCommand), "ExecuteMetaCommand");
if (version >= 4) attach_method(table[72], reinterpret_cast<void*>(hook_BuildRaytracingAccelerationStructure), reinterpret_cast<void**>(&original_BuildRaytracingAccelerationStructure), "BuildRaytracingAccelerationStructure");
if (version >= 4) attach_method(table[73], reinterpret_cast<void*>(hook_EmitRaytracingAccelerationStructurePostbuildInfo), reinterpret_cast<void**>(&original_EmitRaytracingAccelerationStructurePostbuildInfo), "EmitRaytracingAccelerationStructurePostbuildInfo");
if (version >= 4) attach_method(table[74], reinterpret_cast<void*>(hook_CopyRaytracingAccelerationStructure), reinterpret_cast<void**>(&original_CopyRaytracingAccelerationStructure), "CopyRaytracingAccelerationStructure");
if (version >= 4) attach_method(table[75], reinterpret_cast<void*>(hook_SetPipelineState1), reinterpret_cast<void**>(&original_SetPipelineState1), "SetPipelineState1");
if (version >= 4) attach_method(table[76], reinterpret_cast<void*>(hook_DispatchRays), reinterpret_cast<void**>(&original_DispatchRays), "DispatchRays");
if (version >= 5) attach_method(table[77], reinterpret_cast<void*>(hook_RSSetShadingRate), reinterpret_cast<void**>(&original_RSSetShadingRate), "RSSetShadingRate");
if (version >= 5) attach_method(table[78], reinterpret_cast<void*>(hook_RSSetShadingRateImage), reinterpret_cast<void**>(&original_RSSetShadingRateImage), "RSSetShadingRateImage");
if (version >= 6) attach_method(table[79], reinterpret_cast<void*>(hook_DispatchMesh), reinterpret_cast<void**>(&original_DispatchMesh), "DispatchMesh");
if (version >= 7) attach_method(table[80], reinterpret_cast<void*>(hook_Barrier), reinterpret_cast<void**>(&original_Barrier), "Barrier");
if (version >= 8) attach_method(table[81], reinterpret_cast<void*>(hook_OMSetFrontAndBackStencilRef), reinterpret_cast<void**>(&original_OMSetFrontAndBackStencilRef), "OMSetFrontAndBackStencilRef");
if (version >= 9) attach_method(table[82], reinterpret_cast<void*>(hook_RSSetDepthBias), reinterpret_cast<void**>(&original_RSSetDepthBias), "RSSetDepthBias");
if (version >= 9) attach_method(table[83], reinterpret_cast<void*>(hook_IASetIndexBufferStripCutValue), reinterpret_cast<void**>(&original_IASetIndexBufferStripCutValue), "IASetIndexBufferStripCutValue");
if (version >= 10) attach_method(table[84], reinterpret_cast<void*>(hook_SetProgram), reinterpret_cast<void**>(&original_SetProgram), "SetProgram");
if (version >= 10) attach_method(table[85], reinterpret_cast<void*>(hook_DispatchGraph), reinterpret_cast<void**>(&original_DispatchGraph), "DispatchGraph");
}
static constexpr unsigned device_signature_slot = 41;
static constexpr unsigned device_command_list_slot = 12;
static constexpr unsigned queue_signal_slot = 14;
static constexpr unsigned queue_wait_slot = 15;
tools/windows/d3d12_command_hooks.h
-24
View File
@@ -1,24 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include <functional>
#include <string>
namespace d4r::win::commands {
// Intercept only documented D3D12 COM methods on this device's runtime.
// A boundary becomes prefix -> callback -> suffix at ExecuteCommandLists,
// preserving preceding lists in the caller's batch and current-frame output.
using Boundary = std::function<void(ID3D12CommandQueue*)>;
void install(ID3D12Device* device);
void record_boundary(ID3D12GraphicsCommandList* list, Boundary callback);
std::string error();
size_t live_recordings();
D3D12_RESOURCE_STATES resource_state(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, D3D12_RESOURCE_STATES fallback);
ResourceAccess resource_access(ID3D12GraphicsCommandList* list, ID3D12Resource* resource, ResourceAccess fallback);
ResourceAccess submitted_resource_access(ID3D12Resource* resource, ResourceAccess fallback);
// Internal copies and callback submissions must bypass logical list routing.
struct InternalScope {
bool previous;
InternalScope();
~InternalScope();
};
}
tools/windows/d3d12_command_probe.cpp
-327
View File
@@ -1,327 +0,0 @@
#include "d3d12_external.h"
#include <dxgi1_6.h>
#include <d3dcompiler.h>
#include <d3d12sdklayers.h>
#include <array>
using namespace d4r::diag;
using namespace d4r::win;
#if defined(D4R_DIAGNOSTIC_AGILITY)
extern "C" {
__declspec(dllexport) extern const UINT D3D12SDKVersion = 619;
__declspec(dllexport) extern const char* D3D12SDKPath = ".\\debug-d3d12\\";
}
#endif
namespace {
constexpr unsigned count = 256, bytes = count * sizeof(uint32_t), seed = 0x12345;
void messages(ID3D12Device* device) {
ComPtr<ID3D12InfoQueue> info;
if (FAILED(device->QueryInterface(IID_PPV_ARGS(info.GetAddressOf())))) return;
for (UINT64 i = 0; i < info->GetNumStoredMessagesAllowedByRetrievalFilter(); ++i) {
SIZE_T size = 0; (void)info->GetMessage(i, nullptr, &size);
std::vector<uint8_t> buffer(size);
auto* value = reinterpret_cast<D3D12_MESSAGE*>(buffer.data());
if (SUCCEEDED(info->GetMessage(i, value, &size)))
std::fprintf(stderr, "D3D12_VALIDATION severity=%u id=%u %s\n", unsigned(value->Severity), unsigned(value->ID), value->pDescription);
}
info->ClearStoredMessages();
}
void drain(ID3D12Device* device, ID3D12CommandQueue* queue) {
ComPtr<ID3D12Fence> fence; dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(fence.GetAddressOf())), "Probe fence");
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.value) throw std::runtime_error("Probe event");
dx(queue->Signal(fence.Get(), 1), "Probe signal");
if (fence->GetCompletedValue() < 1) {
dx(fence->SetEventOnCompletion(1, event.value), "Probe fence event");
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Probe GPU timeout");
}
}
struct NativeList {
ComPtr<ID3D12CommandAllocator> allocator;
ComPtr<ID3D12GraphicsCommandList> list;
explicit NativeList(ID3D12Device* device, ID3D12PipelineState* initial = nullptr) {
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Probe allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), initial, IID_PPV_ARGS(list.GetAddressOf())), "Probe command list");
}
void submit(ID3D12Device* device, ID3D12CommandQueue* queue) {
dx(list->Close(), "Close probe list"); ID3D12CommandList* lists[] = {list.Get()};
queue->ExecuteCommandLists(1, lists); drain(device, queue);
}
};
ComPtr<ID3D12Resource> output_buffer(ID3D12Device* device) {
D3D12_HEAP_PROPERTIES heap{}; heap.Type = D3D12_HEAP_TYPE_DEFAULT; heap.CreationNodeMask = heap.VisibleNodeMask = 1;
D3D12_RESOURCE_DESC desc{}; desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; desc.Width = bytes;
desc.Height = desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
ComPtr<ID3D12Resource> result;
dx(device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_UNORDERED_ACCESS,
nullptr, IID_PPV_ARGS(result.GetAddressOf())), "Probe UAV");
return result;
}
void upload(ID3D12Resource* buffer, uint32_t multiplier, uint32_t addend) {
void* data = nullptr; D3D12_RANGE noRead{}; dx(buffer->Map(0, &noRead, &data), "Probe upload map");
auto* values = static_cast<uint32_t*>(data);
for (unsigned i = 0; i < count; ++i) values[i] = i * multiplier + addend;
buffer->Unmap(0, nullptr);
}
bool verify(ID3D12Resource* buffer, uint32_t multiplier, uint32_t addend, uint32_t root_seed = seed) {
void* data = nullptr; D3D12_RANGE range{0, bytes}; dx(buffer->Map(0, &range, &data), "Probe verification map");
const auto* values = static_cast<const uint32_t*>(data);
bool passed = true;
for (unsigned i = 0; i < count; ++i) if (values[i] != i * multiplier + addend + root_seed) {
std::fprintf(stderr, "COMMAND_DIFFERENT index=%u expected=%u actual=%u\n", i, i * multiplier + addend + root_seed, values[i]);
passed = false; break;
}
D3D12_RANGE noWrite{}; buffer->Unmap(0, &noWrite); return passed;
}
struct Callback {
ID3D12Device* device;
ID3D12Resource* input;
ID3D12Resource* output;
ID3D12Resource* replacement;
ID3D12Resource* readback;
uint32_t expected_seed = seed;
bool prefix_verified = false;
uint32_t expected_multiplier = 17, expected_addend = 10;
};
void WINAPI boundary(ID3D12CommandQueue* queue, void* context) {
auto& callback = *static_cast<Callback*>(context);
NativeList list(callback.device);
transition(list.list.Get(), callback.output, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COPY_SOURCE);
list.list->CopyResource(callback.readback, callback.output);
transition(list.list.Get(), callback.output, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_UNORDERED_ACCESS);
transition(list.list.Get(), callback.input, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE, D3D12_RESOURCE_STATE_COPY_DEST);
list.list->CopyResource(callback.input, callback.replacement);
transition(list.list.Get(), callback.input, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
list.submit(callback.device, queue);
callback.prefix_verified = verify(callback.readback, callback.expected_multiplier,
callback.expected_addend, callback.expected_seed);
}
}
int main(int argc, char** argv) {
start();
ComPtr<ID3D12Device> device;
std::unique_ptr<Library> debug_layers;
try {
Args args(argc, argv);
if (args.module.empty()) throw std::runtime_error("--module d4r_nvngx.dll is required; NVIDIA DLLs are not needed");
if (std::getenv("D4R_COMMAND_DEBUG_SDK")) {
wchar_t exe[32768]{}; GetModuleFileNameW(nullptr, exe, 32768);
debug_layers = std::make_unique<Library>(std::filesystem::path(exe).parent_path() / L"debug-d3d12/d3d12SDKLayers.dll");
ComPtr<ID3D12Debug> debug;
auto get_interface = reinterpret_cast<decltype(&D3D12GetInterface)>(GetProcAddress(GetModuleHandleW(L"d3d12.dll"), "D3D12GetInterface"));
if (!get_interface) throw std::runtime_error("D3D12GetInterface is unavailable");
dx(get_interface(CLSID_D3D12Debug, IID_PPV_ARGS(debug.GetAddressOf())), "Diagnostic debug layer(public CLSID)");
debug->EnableDebugLayer();
}
ComPtr<ID3D12DeviceRemovedExtendedDataSettings> dred_settings;
if (SUCCEEDED(D3D12GetDebugInterface(IID_PPV_ARGS(dred_settings.GetAddressOf())))) {
dred_settings->SetAutoBreadcrumbsEnablement(D3D12_DRED_ENABLEMENT_FORCED_ON);
dred_settings->SetPageFaultEnablement(D3D12_DRED_ENABLEMENT_FORCED_ON);
}
std::unique_ptr<HipApi> hip;
hipDeviceProp_t props{};
const bool no_hip = std::getenv("D4R_COMMAND_PROBE_NO_HIP") != nullptr;
if (!no_hip) { hip = std::make_unique<HipApi>(args.hip_root); hip->select_architecture(args.device, props); }
ComPtr<IDXGIFactory4> factory; dx(CreateDXGIFactory1(IID_PPV_ARGS(factory.GetAddressOf())), "Probe DXGI");
ComPtr<IDXGIAdapter1> adapter;
for (UINT i = 0; ; ++i) {
dx(factory->EnumAdapters1(i, adapter.ReleaseAndGetAddressOf()), "Probe adapter");
DXGI_ADAPTER_DESC1 desc{}; dx(adapter->GetDesc1(&desc), "Probe adapter desc");
if (no_hip ? desc.VendorId == 0x1002 : !std::memcmp(&desc.AdapterLuid, props.luid, sizeof(LUID))) break;
}
if (std::getenv("D4R_COMMAND_PROBE_WARP")) {
dx(factory->EnumWarpAdapter(IID_PPV_ARGS(adapter.ReleaseAndGetAddressOf())), "Probe WARP adapter");
}
dx(D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_12_0, IID_PPV_ARGS(device.GetAddressOf())), "Probe D3D12");
ComPtr<ID3D12CommandQueue> queue;
D3D12_COMMAND_QUEUE_DESC qd{}; qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
dx(device->CreateCommandQueue(&qd, IID_PPV_ARGS(queue.GetAddressOf())), "Probe queue");
wchar_t system[32768]{}; if (!GetSystemDirectoryW(system, 32768)) throw std::runtime_error("System directory");
Library compiler(std::filesystem::path(system) / L"d3dcompiler_47.dll");
auto compile = compiler.symbol<decltype(&D3DCompile)>("D3DCompile");
static constexpr char shader[] = "RWStructuredBuffer<uint> Output : register(u0); StructuredBuffer<uint> Input : register(t0); cbuffer Constants : register(b0) { uint Seed; }; [numthreads(64,1,1)] void main(uint3 t:SV_DispatchThreadID) { Output[t.x] = Input[t.x] + Seed; }";
ComPtr<ID3DBlob> code, errors;
dx(compile(shader, sizeof(shader)-1, nullptr, nullptr, nullptr, "main", "cs_5_1", D3DCOMPILE_OPTIMIZATION_LEVEL3,
0, code.GetAddressOf(), errors.GetAddressOf()), "Probe HLSL");
D3D12_ROOT_PARAMETER roots[3]{};
roots[0].ParameterType = D3D12_ROOT_PARAMETER_TYPE_UAV; roots[0].Descriptor.ShaderRegister = 0;
roots[1].ParameterType = D3D12_ROOT_PARAMETER_TYPE_SRV; roots[1].Descriptor.ShaderRegister = 0;
roots[2].ParameterType = D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS; roots[2].Constants.Num32BitValues = 1;
D3D12_ROOT_SIGNATURE_DESC signature{}; signature.NumParameters = 3; signature.pParameters = roots;
ComPtr<ID3DBlob> serialized;
dx(D3D12SerializeRootSignature(&signature, D3D_ROOT_SIGNATURE_VERSION_1, serialized.GetAddressOf(), errors.ReleaseAndGetAddressOf()), "Probe root signature serialize");
ComPtr<ID3D12RootSignature> root;
dx(device->CreateRootSignature(0, serialized->GetBufferPointer(), serialized->GetBufferSize(), IID_PPV_ARGS(root.GetAddressOf())), "Probe root signature");
D3D12_COMPUTE_PIPELINE_STATE_DESC pipeline{};
pipeline.pRootSignature = root.Get(); pipeline.CS = {code->GetBufferPointer(), code->GetBufferSize()};
ComPtr<ID3D12PipelineState> pso; dx(device->CreateComputePipelineState(&pipeline, IID_PPV_ARGS(pso.GetAddressOf())), "Probe PSO");
Library shim(wide(args.module));
using Install = unsigned(*)(ID3D12Device*);
using Record = unsigned(*)(ID3D12GraphicsCommandList*, void(WINAPI*)(ID3D12CommandQueue*, void*), void*);
using Live = unsigned long long(*)();
const bool no_hooks = std::getenv("D4R_COMMAND_PROBE_NO_HOOKS") != nullptr;
if (!no_hooks && shim.symbol<Install>("d4r_D3D12_InstallCommandBackend")(device.Get()) != 1) throw std::runtime_error("Command backend installation failed");
{
ComPtr<IDXGIFactory2> newerFactory; dx(factory->QueryInterface(IID_PPV_ARGS(newerFactory.GetAddressOf())), "Resize regression factory");
DXGI_SWAP_CHAIN_DESC1 desc{}; desc.Width = desc.Height = 64;
desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; desc.SampleDesc.Count = 1;
desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; desc.BufferCount = 2;
desc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; desc.AlphaMode = DXGI_ALPHA_MODE_IGNORE;
ComPtr<IDXGISwapChain1> chain;
dx(newerFactory->CreateSwapChainForComposition(queue.Get(), &desc, nullptr, chain.GetAddressOf()), "Resize regression swap chain");
ComPtr<ID3D12Resource> backBuffer; dx(chain->GetBuffer(0, IID_PPV_ARGS(backBuffer.GetAddressOf())), "Resize regression back buffer");
NativeList recorded(device.Get());
transition(recorded.list.Get(), backBuffer.Get(), D3D12_RESOURCE_STATE_PRESENT, D3D12_RESOURCE_STATE_RENDER_TARGET);
transition(recorded.list.Get(), backBuffer.Get(), D3D12_RESOURCE_STATE_RENDER_TARGET, D3D12_RESOURCE_STATE_PRESENT);
recorded.submit(device.Get(), queue.Get()); backBuffer.Reset();
// The submitted list remains alive and unreset during resize.
dx(chain->ResizeBuffers(2, 96, 64, DXGI_FORMAT_UNKNOWN, 0), "ResizeBuffers with retained submitted list");
std::printf("D3D12_RESIZE_REGRESSION submitted_list_alive=1 old_back_buffer_released=1\n");
}
auto record = shim.symbol<Record>("d4r_D3D12_RecordDiagnosticBoundary");
auto live = shim.symbol<Live>("d4r_D3D12_LiveRecordings");
const bool root_indirect = args.interop_mode == "indirect-root-reset";
const bool initial_pso = args.interop_mode == "initial-pso";
const bool indirect_enabled = args.interop_mode == "indirect" || root_indirect;
if (args.interop_mode != "direct" && args.interop_mode != "basic" && args.interop_mode != "indirect" && !root_indirect && !initial_pso)
throw std::runtime_error("Command probe mode must be basic, indirect, indirect-root-reset or initial-pso");
D3D12_INDIRECT_ARGUMENT_DESC arguments[2]{};
arguments[0].Type = D3D12_INDIRECT_ARGUMENT_TYPE_CONSTANT;
arguments[0].Constant.RootParameterIndex = 2;
arguments[0].Constant.Num32BitValuesToSet = 1;
arguments[1].Type = D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH;
D3D12_COMMAND_SIGNATURE_DESC indirect_desc{};
indirect_desc.ByteStride = (root_indirect ? 4 : 3) * sizeof(uint32_t);
indirect_desc.NumArgumentDescs = root_indirect ? 2 : 1;
indirect_desc.pArgumentDescs = arguments + (root_indirect ? 0 : 1);
ComPtr<ID3D12CommandSignature> indirect;
dx(device->CreateCommandSignature(&indirect_desc, root_indirect ? root.Get() : nullptr, IID_PPV_ARGS(indirect.GetAddressOf())), "Probe indirect signature");
auto indirect_upload = make_buffer(device.Get(), 4 * sizeof(uint32_t), D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
auto indirect_buffer = make_buffer(device.Get(), 4 * sizeof(uint32_t), D3D12_HEAP_TYPE_DEFAULT, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
void* mapped = nullptr; D3D12_RANGE no_read{};
dx(indirect_upload->Map(0, &no_read, &mapped), "Probe indirect arguments");
const uint32_t indirect_seed = std::getenv("D4R_COMMAND_PROBE_SMALL_SEED") ? 1u : seed;
const uint32_t dispatch[] = {indirect_seed, count / 64, 1, 1};
std::memcpy(mapped, dispatch, sizeof(dispatch)); indirect_upload->Unmap(0, nullptr);
{
NativeList copy(device.Get()); copy.list->CopyResource(indirect_buffer.Get(), indirect_upload.Get());
transition(copy.list.Get(), indirect_buffer.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT);
copy.submit(device.Get(), queue.Get());
}
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
auto input = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
auto output = output_buffer(device.Get());
auto first = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
auto second = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
auto readback = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
auto prefix = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
input->SetName(L"Probe input"); output->SetName(L"Probe output"); indirect_buffer->SetName(L"Probe indirect");
std::printf("PROBE_GPU iteration=%u input=0x%llx output=0x%llx indirect=0x%llx\n", iteration,
input->GetGPUVirtualAddress(), output->GetGPUVirtualAddress(), indirect_buffer->GetGPUVirtualAddress());
upload(first.Get(), 17, 10); upload(second.Get(), 25, 100);
{
NativeList a(device.Get()), b(device.Get(), initial_pso ? pso.Get() : nullptr), c(device.Get());
a.list->CopyResource(input.Get(), first.Get());
transition(a.list.Get(), input.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
dx(a.list->Close(), "Close predecessor A");
b.list->SetComputeRootSignature(root.Get()); if (!initial_pso) b.list->SetPipelineState(pso.Get());
b.list->SetComputeRootUnorderedAccessView(0, output->GetGPUVirtualAddress());
b.list->SetComputeRootShaderResourceView(1, input->GetGPUVirtualAddress());
uint32_t temporary = seed;
b.list->SetComputeRoot32BitConstants(2, 1, &temporary, 0);
temporary = 0xdeadbeef; // Deep-copy check: replay must retain Seed.
const uint32_t expected_seed = root_indirect && iteration % 2 ? 0 : seed;
if (indirect_enabled && iteration % 2) {
b.list->ExecuteIndirect(indirect.Get(), 1, indirect_buffer.Get(), root_indirect ? 0 : sizeof(uint32_t), nullptr, 0);
D3D12_RESOURCE_BARRIER uav{}; uav.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV; uav.UAV.pResource = output.Get();
b.list->ResourceBarrier(1, &uav);
// Only the root-mutating signature resets Seed to zero.
// A dispatch-only signature must preserve every binding.
}
b.list->Dispatch(count / 64, 1, 1);
Callback callback{device.Get(), input.Get(), output.Get(), second.Get(), prefix.Get(), expected_seed};
if (no_hooks) {
dx(b.list->Close(), "Close native prefix");
messages(device.Get());
ID3D12CommandList* prefix_batch[] = {a.list.Get(), b.list.Get()};
queue->ExecuteCommandLists(2, prefix_batch); drain(device.Get(), queue.Get());
boundary(queue.Get(), &callback);
dx(b.allocator->Reset(), "Reset native prefix allocator");
dx(b.list->Reset(b.allocator.Get(), pso.Get()), "Reset native suffix");
b.list->SetComputeRootSignature(root.Get());
b.list->SetComputeRootUnorderedAccessView(0, output->GetGPUVirtualAddress());
b.list->SetComputeRootShaderResourceView(1, input->GetGPUVirtualAddress());
b.list->SetComputeRoot32BitConstant(2, expected_seed, 0);
} else if (record(b.list.Get(), boundary, &callback) != 1) throw std::runtime_error("Record split failed");
b.list->Dispatch(count / 64, 1, 1); // No rebinding after the split.
dx(b.list->Close(), "Close logical list B");
transition(c.list.Get(), output.Get(), D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COPY_SOURCE);
c.list->CopyResource(readback.Get(), output.Get());
transition(c.list.Get(), output.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_UNORDERED_ACCESS);
dx(c.list->Close(), "Close consumer C");
messages(device.Get());
ID3D12CommandList* batch[] = {a.list.Get(), b.list.Get(), c.list.Get()};
queue->ExecuteCommandLists(no_hooks ? 2 : 3, batch + (no_hooks ? 1 : 0)); drain(device.Get(), queue.Get());
if (!callback.prefix_verified || !verify(readback.Get(), 25, 100, expected_seed)) throw std::runtime_error("Queue order or suffix root state mismatch");
if (!no_hooks) for (unsigned reuse = 0; reuse < 2; ++reuse) {
// Populate the same thread's routing cache with a split
// list, Reset that exact COM object, then split again with
// different root state. A cached previous Recording would
// keep targeting the old suffix or replay the old Seed.
dx(b.allocator->Reset(), "Reset reused allocator");
dx(b.list->Reset(b.allocator.Get(), initial_pso ? pso.Get() : nullptr), "Reset reused logical list");
b.list->SetComputeRootSignature(root.Get());
if (!initial_pso) b.list->SetPipelineState(pso.Get());
b.list->SetComputeRootUnorderedAccessView(0, output->GetGPUVirtualAddress());
b.list->SetComputeRootShaderResourceView(1, input->GetGPUVirtualAddress());
callback.expected_seed = seed + iteration + reuse + 1;
callback.expected_multiplier = 25; callback.expected_addend = 100;
callback.prefix_verified = false;
b.list->SetComputeRoot32BitConstant(2, callback.expected_seed, 0);
b.list->Dispatch(count / 64, 1, 1);
if (record(b.list.Get(), boundary, &callback) != 1) throw std::runtime_error("Record reused split failed");
b.list->Dispatch(count / 64, 1, 1);
transition(b.list.Get(), output.Get(), D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COPY_SOURCE);
b.list->CopyResource(readback.Get(), output.Get());
transition(b.list.Get(), output.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_UNORDERED_ACCESS);
b.submit(device.Get(), queue.Get());
if (!callback.prefix_verified || !verify(readback.Get(), 25, 100, callback.expected_seed))
throw std::runtime_error("Reset reused logical list retained stale routing/root state");
}
}
if (live() != 0) throw std::runtime_error("D3D12 recording metadata leaked after command-list Release");
}
const bool warp = std::getenv("D4R_COMMAND_PROBE_WARP") != nullptr;
std::printf("PASS D3D12_COMMAND_BACKEND adapter=%s architecture=%s backend=%u iterations=%u batch_order=1 root_state=1 deep_copy=1 indirect=%u indirect_reset=%u reset_reuse=%u live_recordings=0 frame_age=0\n",
warp ? "WARP" : "AMD", warp ? "software" : no_hip ? "not_queried" : HipApi::target_arch(), !no_hooks,
args.iterations, indirect_enabled, root_indirect, no_hooks ? 0 : 2 * args.iterations);
return 0;
} catch (const std::exception& failure) {
std::fprintf(stderr, "FAIL D3D12_COMMAND_BACKEND %s\n", failure.what());
if (device) {
messages(device.Get());
std::fprintf(stderr, "DEVICE_REMOVED result=0x%08x\n", unsigned(device->GetDeviceRemovedReason()));
ComPtr<ID3D12DeviceRemovedExtendedData> dred;
if (SUCCEEDED(device->QueryInterface(IID_PPV_ARGS(dred.GetAddressOf())))) {
D3D12_DRED_AUTO_BREADCRUMBS_OUTPUT breadcrumbs{};
if (SUCCEEDED(dred->GetAutoBreadcrumbsOutput(&breadcrumbs)))
for (auto* node = breadcrumbs.pHeadAutoBreadcrumbNode; node; node = node->pNext) {
UINT completed = node->pLastBreadcrumbValue ? *node->pLastBreadcrumbValue : 0;
std::fprintf(stderr, "DRED list=%p completed=%u total=%u\n", node->pCommandList, completed, node->BreadcrumbCount);
for (UINT i = completed > 2 ? completed - 2 : 0; i < std::min(node->BreadcrumbCount, completed + 3); ++i)
std::fprintf(stderr, "DRED command=%u operation=%u\n", i, unsigned(node->pCommandHistory[i]));
}
D3D12_DRED_PAGE_FAULT_OUTPUT fault{};
if (SUCCEEDED(dred->GetPageFaultAllocationOutput(&fault)))
std::fprintf(stderr, "DRED fault_address=0x%llx\n", fault.PageFaultVA);
for (auto* allocation = fault.pHeadExistingAllocationNode; allocation; allocation = allocation->pNext)
std::fprintf(stderr, "DRED existing=%ls type=%u\n", allocation->ObjectNameW ? allocation->ObjectNameW : L"(unnamed)", unsigned(allocation->AllocationType));
for (auto* allocation = fault.pHeadRecentFreedAllocationNode; allocation; allocation = allocation->pNext)
std::fprintf(stderr, "DRED freed=%ls type=%u\n", allocation->ObjectNameW ? allocation->ObjectNameW : L"(unnamed)", unsigned(allocation->AllocationType));
}
}
loaded_modules(); return 4;
}
}
tools/windows/d3d12_dlss_probe.cpp
-413
View File
@@ -1,413 +0,0 @@
#include "d3d12_external.h"
#include "ngx_parameters.h"
#include "ngx_cuda_evaluate.h"
#include "ngx_public_abi.h"
#include <dxgi1_6.h>
#include <memory>
#include <array>
#include <d3dcompiler.h>
int main(int argc, char** argv) {
using namespace d4r::diag;
using namespace d4r::win;
start();
try {
Args args(argc, argv);
if (args.cuda_dll.empty() || (args.interop_mode != "images" && (args.module.empty() || args.ngx_core.empty() || args.dlss_dll.empty())))
throw std::runtime_error("--module shim DLL, --cuda-dll, --ngx-core and --dlss-dll required");
const char* diagDir = std::getenv("D4R_DIAG_DIR");
if (!diagDir && args.interop_mode != "images") throw std::runtime_error("Set D4R_DIAG_DIR");
const std::filesystem::path directory(diagDir ? wide(diagDir) : std::filesystem::current_path().wstring());
std::filesystem::create_directories(directory);
const bool commandBackend = args.interop_mode == "command-list";
const bool inherited = args.barrier_mode.rfind("inherited-", 0) == 0;
const bool enhanced = args.barrier_mode == "enhanced" || args.barrier_mode == "inherited-enhanced";
if (inherited && !commandBackend) throw std::runtime_error("Inherited state diagnostic requires the command-list backend");
if (args.interop_mode == "images" && args.pixel_profile != "baseline") throw std::runtime_error("Raw CUDA image diagnostic requires baseline formats");
std::printf("D3D12_PIXEL_PROFILE %s\n", args.pixel_profile.c_str());
if (commandBackend && _putenv_s("D4R_D3D12_COMMAND_BACKEND", "1")) throw std::runtime_error("Cannot enable command backend");
for (const auto& pair : {std::make_pair("D4R_HIP_ROOT", args.hip_root),
std::make_pair("D4R_NVCUDA_DLL", args.cuda_dll), std::make_pair("D4R_NVAPI_DLL", args.nvapi_dll),
std::make_pair("D4R_NGX_CORE", args.ngx_core), std::make_pair("D4R_DLSS_DLL", args.dlss_dll)})
if (_putenv_s(pair.first, pair.second.c_str())) throw std::runtime_error("Cannot set runtime environment");
HipApi hip(args.hip_root); hipDeviceProp_t props{}; hip.select_architecture(args.device, props);
ComPtr<IDXGIFactory4> factory; dx(CreateDXGIFactory1(IID_PPV_ARGS(factory.GetAddressOf())), "CreateDXGIFactory1");
ComPtr<IDXGIAdapter1> adapter;
bool found = false;
for (UINT i = 0; ; ++i) {
const HRESULT result = factory->EnumAdapters1(i, adapter.ReleaseAndGetAddressOf());
if (result == DXGI_ERROR_NOT_FOUND) break;
dx(result, "EnumAdapters"); DXGI_ADAPTER_DESC1 desc{}; dx(adapter->GetDesc1(&desc), "GetDesc1");
if (!std::memcmp(&desc.AdapterLuid, props.luid, sizeof(LUID))) { found = true; break; }
}
if (!found) throw std::runtime_error("HIP adapter missing from DXGI");
ComPtr<ID3D12Device> device;
dx(D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_12_0, IID_PPV_ARGS(device.GetAddressOf())), "D3D12CreateDevice");
if (enhanced) { D3D12_FEATURE_DATA_D3D12_OPTIONS12 support{}; dx(device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS12, &support, sizeof(support)), "Enhanced barrier support"); if (!support.EnhancedBarriersSupported) throw std::runtime_error("Enhanced barriers unavailable"); }
auto enhancedBarrier = [&](ID3D12GraphicsCommandList* command, ID3D12Resource* texture, unsigned plane, bool initial) {
ComPtr<ID3D12GraphicsCommandList7> newer; dx(command->QueryInterface(IID_PPV_ARGS(newer.GetAddressOf())), "Harness enhanced list");
D3D12_TEXTURE_BARRIER barrier{}; barrier.pResource = texture;
barrier.LayoutAfter = plane == 3 ? D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_UNORDERED_ACCESS : D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_SHADER_RESOURCE;
barrier.LayoutBefore = initial ? D3D12_BARRIER_LAYOUT_COMMON : barrier.LayoutAfter;
barrier.AccessAfter = plane == 3 ? D3D12_BARRIER_ACCESS_UNORDERED_ACCESS : D3D12_BARRIER_ACCESS_SHADER_RESOURCE;
barrier.AccessBefore = initial ? D3D12_BARRIER_ACCESS_COMMON : barrier.AccessAfter;
barrier.SyncBefore = barrier.SyncAfter = D3D12_BARRIER_SYNC_ALL;
D3D12_BARRIER_GROUP group{}; group.Type = D3D12_BARRIER_TYPE_TEXTURE; group.NumBarriers = 1; group.pTextureBarriers = &barrier; newer->Barrier(1, &group);
};
ComPtr<ID3D12CommandQueue> queue;
D3D12_COMMAND_QUEUE_DESC qd{}; qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
dx(device->CreateCommandQueue(&qd, IID_PPV_ARGS(queue.GetAddressOf())), "CreateCommandQueue");
ComPtr<ID3D12Fence> completion;
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(completion.GetAddressOf())), "CreateFence(harness)");
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.value) throw std::runtime_error("CreateEvent(harness)");
uint64_t fenceValue = 0;
auto drain = [&] {
dx(queue->Signal(completion.Get(), ++fenceValue), "Queue Signal(harness)");
if (completion->GetCompletedValue() < fenceValue) {
dx(completion->SetEventOnCompletion(fenceValue, event.value), "SetEventOnCompletion(harness)");
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Harness GPU timeout");
}
};
ComPtr<ID3D12CommandAllocator> allocator;
ComPtr<ID3D12GraphicsCommandList> list;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "CreateCommandList");
const unsigned outWidth = args.output_width, outHeight = args.output_height;
const unsigned width = args.input_width ? args.input_width : outWidth / 2;
const unsigned height = args.input_height ? args.input_height : outHeight / 2;
std::printf("D3D12_NGX_DIMENSIONS input=%ux%u output=%ux%u\n", width, height, outWidth, outHeight);
struct Texture { ComPtr<ID3D12Resource> image, upload; D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint{}; UINT64 bytes; } textures[5];
for (unsigned i = 0; i < 5; ++i) {
auto& texture = textures[i];
D3D12_RESOURCE_DESC desc{}; desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D;
desc.Width = i == 4 ? 1 : i == 3 ? outWidth : width;
desc.Height = i == 4 ? 1 : i == 3 ? outHeight : height;
desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
desc.Format = i == 0 || i == 3 ? DXGI_FORMAT_R16G16B16A16_FLOAT : i == 2 ? DXGI_FORMAT_R16G16_FLOAT : DXGI_FORMAT_R32_FLOAT;
if (args.pixel_profile == "packed") desc.Format = i == 0 || i == 3 ? DXGI_FORMAT_R11G11B10_FLOAT :
i == 2 ? DXGI_FORMAT_R32G32B32A32_FLOAT : i == 4 ? DXGI_FORMAT_R16_FLOAT : DXGI_FORMAT_R32_FLOAT;
if (args.pixel_profile == "unorm") desc.Format = i == 3 ? DXGI_FORMAT_B8G8R8A8_UNORM :
i == 2 ? DXGI_FORMAT_R16G16_SNORM : i == 4 ? DXGI_FORMAT_R32G32B32A32_FLOAT : desc.Format;
if (args.pixel_profile == "depth-stencil" && i == 1) { desc.Format = DXGI_FORMAT_R32G8X24_TYPELESS; desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL; }
if (i == 3) desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
D3D12_HEAP_PROPERTIES heap{}; heap.Type = D3D12_HEAP_TYPE_DEFAULT;
heap.CreationNodeMask = heap.VisibleNodeMask = 1;
dx(device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_COPY_DEST,
nullptr, IID_PPV_ARGS(texture.image.GetAddressOf())), "CreateCommittedResource(texture)");
UINT rows; UINT64 rowBytes;
device->GetCopyableFootprints(&desc, 0, 1, 0, &texture.footprint, &rows, &rowBytes, &texture.bytes);
texture.upload = make_buffer(device.Get(), texture.bytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
void* raw = nullptr; D3D12_RANGE noRead{};
dx(texture.upload->Map(0, &noRead, &raw), "Map(synthetic initialization)");
std::memset(raw, 0, size_t(texture.bytes));
for (unsigned y = 0; y < desc.Height; ++y) for (unsigned x = 0; x < desc.Width; ++x) {
auto* row = static_cast<uint8_t*>(raw) + y * texture.footprint.Footprint.RowPitch;
if (i == 0) {
uint16_t pixel[] = {uint16_t(((x / 16 + y / 16) & 1) ? 0x3a00 : 0x3400),
uint16_t((x & 32) ? 0x3800 : 0x3400), uint16_t((y & 32) ? 0x3a00 : 0x3800), 0x3c00};
if (desc.Format == DXGI_FORMAT_R11G11B10_FLOAT) {
const uint32_t packed = uint32_t(pixel[0] >> 4) | (uint32_t(pixel[1] >> 4) << 11) | (uint32_t(pixel[2] >> 5) << 22);
std::memcpy(row + x * 4, &packed, 4);
} else std::memcpy(row + x * 8, pixel, sizeof(pixel));
} else if (i == 1 || i == 4) {
if (desc.Format == DXGI_FORMAT_R16_FLOAT) { const uint16_t one = 0x3c00; std::memcpy(row + x * 2, &one, 2); }
else { const float value = i == 1 ? .5f : 1.f; std::memcpy(row + x * 4, &value, 4); }
} else if (i == 3) {
if (desc.Format == DXGI_FORMAT_R16G16B16A16_FLOAT) { const uint16_t pixel[] = {0x7e00, 0x7e00, 0x7e00, 0x7e00}; std::memcpy(row + x * 8, pixel, 8); }
else { const uint32_t sentinel = 0xffffffff; std::memcpy(row + x * 4, &sentinel, 4); }
}
}
texture.upload->Unmap(0, nullptr);
D3D12_TEXTURE_COPY_LOCATION src{}, dst{};
src.pResource = texture.upload.Get(); src.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; src.PlacedFootprint = texture.footprint;
dst.pResource = texture.image.Get(); dst.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
list->CopyTextureRegion(&dst, 0, 0, 0, &src, nullptr);
transition(list.Get(), texture.image.Get(), D3D12_RESOURCE_STATE_COPY_DEST,
(enhanced || inherited) ? D3D12_RESOURCE_STATE_COMMON : i == 3 ? D3D12_RESOURCE_STATE_UNORDERED_ACCESS : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
if (enhanced && !inherited) enhancedBarrier(list.Get(), texture.image.Get(), i, true);
}
if (!inherited) dx(list->Close(), "Close(synthetic producers)"); ID3D12CommandList* producer[] = {list.Get()};
if (!commandBackend) { queue->ExecuteCommandLists(1, producer); drain(); }
if (args.interop_mode == "images") {
CudaApi cuda(args.cuda_dll); cuda.check(cuda.cuInit(0), "cuInit(image interop)");
CUdevice ordinal = 0; CUcontext context = nullptr;
cuda.check(cuda.cuDeviceGet(&ordinal, 0), "cuDeviceGet(image interop)");
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain(image interop)");
struct ContextCleanup { CudaApi& cuda; CUdevice ordinal; ~ContextCleanup() { (void)cuda.cuCtxSetCurrent(nullptr); (void)cuda.cuDevicePrimaryCtxRelease_v2(ordinal); } } contextCleanup{cuda, ordinal};
cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent(image interop)");
ExternalApi external{hip}; SharedTimeline timeline(external, device.Get());
d4r::cuda::ImageApi images(cuda);
std::unique_ptr<SharedPlane> planes[5];
std::unique_ptr<d4r::cuda::Image> arrays[5];
dx(allocator->Reset(), "Reset(image interop allocator)"); dx(list->Reset(allocator.Get(), nullptr), "Reset(image interop list)");
for (unsigned i = 0; i < 5; ++i) {
const auto desc = resource_desc(textures[i].image.Get());
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
const bool half = i == 0 || i == 2 || i == 3;
planes[i] = std::make_unique<SharedPlane>(external, device.Get(), desc);
arrays[i] = std::make_unique<d4r::cuda::Image>(images, unsigned(desc.Width), desc.Height, half ? 16 : 32, channels, i == 3, i == 0 ? 1 : 0);
planes[i]->copy_input(list.Get(), textures[i].image.Get(), i == 3 ? D3D12_RESOURCE_STATE_UNORDERED_ACCESS : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
}
dx(list->Close(), "Close(image interop input)"); ID3D12CommandList* copies[] = {list.Get()}; queue->ExecuteCommandLists(1, copies);
timeline.wait_input(queue.Get());
for (unsigned i = 0; i < 5; ++i) arrays[i]->upload_device(reinterpret_cast<uintptr_t>(planes[i]->mapped), planes[i]->footprint.Footprint.RowPitch);
for (unsigned i = 0; i < 5; ++i) {
const auto desc = resource_desc(textures[i].image.Get());
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
const bool half = i == 0 || i == 2 || i == 3;
const size_t rowBytes = size_t(desc.Width) * channels * (half ? 2 : 4);
std::vector<uint8_t> actual(rowBytes * desc.Height);
arrays[i]->download(actual.data());
void* expected = nullptr; D3D12_RANGE read{0, size_t(textures[i].bytes)};
dx(textures[i].upload->Map(0, &read, &expected), "Map(expected diagnostic image)");
bool matches = true;
for (unsigned y = 0; y < desc.Height; ++y) if (std::memcmp(actual.data() + y * rowBytes,
static_cast<uint8_t*>(expected) + y * textures[i].footprint.Footprint.RowPitch, rowBytes)) { matches = false; break; }
D3D12_RANGE noWrite{}; textures[i].upload->Unmap(0, &noWrite);
std::printf("D3D12_ARRAY_COPY plane=%u format=%u matches=%u first=%02x%02x%02x%02x\n", i, desc.Format, matches, actual[0], actual[1], actual[2], actual[3]);
if (!matches) throw std::runtime_error("Imported D3D12 VRAM to CUDA array mismatch on plane " + std::to_string(i));
timeline.drain(queue.Get());
}
std::printf("PASS D3D12_ARRAY_COPY architecture=%s planes=5 cpu_copies_between_apis=0\n", d4r::diag::HipApi::target_arch());
return 0;
}
Library shim(wide(args.module));
ComPtr<ID3D12CommandSignature> earlySignature;
ComPtr<ID3D12RootSignature> earlyRoot;
ComPtr<ID3D12PipelineState> earlyPso;
ComPtr<ID3D12Resource> earlyArguments;
if (args.early_indirect) {
if (!commandBackend) throw std::runtime_error("Early indirect diagnostic requires command-list backend");
// Discover a device through the frontend before NGX Init. This is
// the startup order used by games which create signatures early.
ComPtr<ID3D12Device> discovered;
dx(D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_12_0, IID_PPV_ARGS(discovered.GetAddressOf())), "Early frontend device discovery");
D3D12_ROOT_SIGNATURE_DESC rootDesc{};
ComPtr<ID3DBlob> rootCode, errors, code;
dx(D3D12SerializeRootSignature(&rootDesc, D3D_ROOT_SIGNATURE_VERSION_1, rootCode.GetAddressOf(), errors.GetAddressOf()), "Early empty root signature");
dx(device->CreateRootSignature(0, rootCode->GetBufferPointer(), rootCode->GetBufferSize(), IID_PPV_ARGS(earlyRoot.GetAddressOf())), "Early root signature");
constexpr char shader[] = "[numthreads(1,1,1)] void main() {}";
dx(D3DCompile(shader, sizeof(shader) - 1, "early-indirect", nullptr, nullptr, "main", "cs_5_0", 0, 0, code.GetAddressOf(), errors.ReleaseAndGetAddressOf()), "Early noop shader");
D3D12_COMPUTE_PIPELINE_STATE_DESC pipeline{};
pipeline.pRootSignature = earlyRoot.Get(); pipeline.CS = {code->GetBufferPointer(), code->GetBufferSize()};
dx(device->CreateComputePipelineState(&pipeline, IID_PPV_ARGS(earlyPso.GetAddressOf())), "Early noop PSO");
D3D12_INDIRECT_ARGUMENT_DESC argument{}; argument.Type = D3D12_INDIRECT_ARGUMENT_TYPE_DISPATCH;
D3D12_COMMAND_SIGNATURE_DESC signature{}; signature.ByteStride = 12; signature.NumArgumentDescs = 1; signature.pArgumentDescs = &argument;
dx(device->CreateCommandSignature(&signature, nullptr, IID_PPV_ARGS(earlySignature.GetAddressOf())), "Pre-NGX command signature");
earlyArguments = make_buffer(device.Get(), 12, D3D12_HEAP_TYPE_DEFAULT, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
auto upload = make_buffer(device.Get(), 12, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
void* mapped = nullptr; D3D12_RANGE noRead{}; dx(upload->Map(0, &noRead, &mapped), "Early indirect upload");
const uint32_t dimensions[] = {0, 1, 1}; std::memcpy(mapped, dimensions, sizeof(dimensions)); upload->Unmap(0, nullptr);
ComPtr<ID3D12CommandAllocator> copyAllocator; ComPtr<ID3D12GraphicsCommandList> copy;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(copyAllocator.GetAddressOf())), "Early copy allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, copyAllocator.Get(), nullptr, IID_PPV_ARGS(copy.GetAddressOf())), "Early copy list");
copy->CopyResource(earlyArguments.Get(), upload.Get());
transition(copy.Get(), earlyArguments.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT);
dx(copy->Close(), "Early copy close"); ID3D12CommandList* batch[] = {copy.Get()}; queue->ExecuteCommandLists(1, batch); drain();
std::printf("D3D12_EARLY_INDIRECT signature_created_before_ngx_init=1\n");
}
using Init = unsigned(*)(unsigned long long, const wchar_t*, ID3D12Device*, unsigned, const void*);
using Allocate = unsigned(*)(void**); using Destroy = unsigned(*)(void*);
using Create = unsigned(*)(ID3D12GraphicsCommandList*, unsigned, void*, void**);
using Evaluate = unsigned(*)(ID3D12CommandQueue*, void*, void*);
using Release = unsigned(*)(void*); using Shutdown = unsigned(*)();
auto check = [](unsigned result, const char* call) { if (result != 1) throw std::runtime_error(std::string(call) + " NGX=" + std::to_string(result)); };
d4r::ngx::FeatureDiscoveryInfo discovery{}; discovery.SDKVersion = 0x15; discovery.FeatureID = 1;
d4r::ngx::FeatureRequirement requirements{};
using Requirements = unsigned(*)(IDXGIAdapter*,const d4r::ngx::FeatureDiscoveryInfo*,d4r::ngx::FeatureRequirement*);
check(shim.symbol<Requirements>("NVSDK_NGX_D3D12_GetFeatureRequirements")(adapter.Get(),&discovery,&requirements),"D3D12 feature requirements");
if (requirements.FeatureSupported != 0) throw std::runtime_error("D3D12 DLSS requirements unsupported");
if (args.ngx_frontend != "optiscaler") {
ComPtr<IDXGIAdapter> warp;
dx(factory->EnumWarpAdapter(IID_PPV_ARGS(warp.GetAddressOf())), "Requirements WARP adapter");
d4r::ngx::FeatureRequirement unsupported{};
check(shim.symbol<Requirements>("NVSDK_NGX_D3D12_GetFeatureRequirements")(warp.Get(),&discovery,&unsupported),"WARP feature requirements");
if (unsupported.FeatureSupported != 4) throw std::runtime_error("WARP must report AdapterUnsupported without an initialization failure");
std::printf("D3D12_REQUIREMENTS warp_supported=0 enumeration_success=1\n");
}
if (args.ngx_abi == "project-legacy") {
using ProjectInit = unsigned(*)(const char*,int,const char*,const wchar_t*,ID3D12Device*,const void*,unsigned);
check(shim.symbol<ProjectInit>("NVSDK_NGX_D3D12_Init_with_ProjectID")("24480451-f00d-face-1304-0308dabad187",0,"1.0",directory.c_str(),device.Get(),nullptr,0x15),"D3D12 legacy project Init");
} else if (args.ngx_abi == "project") {
using ProjectInit = unsigned(*)(const char*,int,const char*,const wchar_t*,ID3D12Device*,unsigned,const void*);
check(shim.symbol<ProjectInit>("NVSDK_NGX_D3D12_Init_ProjectID")("24480451-f00d-face-1304-0308dabad187",0,"1.0",directory.c_str(),device.Get(),0x15,nullptr),"D3D12 project Init");
} else check(shim.symbol<Init>("NVSDK_NGX_D3D12_Init_Ext")(args.ngx_app_id, directory.c_str(), device.Get(), 0x15, nullptr), "D3D12 Init");
auto shutdown = shim.symbol<Shutdown>("NVSDK_NGX_D3D12_Shutdown");
struct ShutdownCleanup { Shutdown fn; ~ShutdownCleanup() { (void)fn(); } } shutdownCleanup{shutdown};
void* parameters = nullptr; check(shim.symbol<Allocate>("NVSDK_NGX_D3D12_AllocateParameters")(&parameters), "AllocateParameters");
auto destroy = shim.symbol<Destroy>("NVSDK_NGX_D3D12_DestroyParameters");
struct ParameterCleanup { Destroy fn; void* p; ~ParameterCleanup() { (void)fn(p); } } parameterCleanup{destroy, parameters};
for (const auto& pair : {std::make_pair("Width", width), std::make_pair("Height", height),
std::make_pair("OutWidth", outWidth), std::make_pair("OutHeight", outHeight),
std::make_pair("CreationNodeMask", 1u), std::make_pair("VisibilityNodeMask", 1u)}) d4r_ngx_set_uint(parameters, pair.first, pair.second);
d4r_ngx_set_int(parameters, "PerfQualityValue", 2); d4r_ngx_set_int(parameters, "DLSS.Feature.Create.Flags", args.ngx_create_flags);
std::printf("D3D12_NGX_CREATE_FLAGS value=%u\n", args.ngx_create_flags);
for (const char* name : {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"}) d4r_ngx_set_uint(parameters, name, args.preset);
void* handle = nullptr;
// Public NGX and OptiScaler require an open creation command list.
// Submit it independently from the pending synthetic input producer.
ComPtr<ID3D12CommandAllocator> creationAllocator;
ComPtr<ID3D12GraphicsCommandList> creationList;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(creationAllocator.GetAddressOf())), "Create feature allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, creationAllocator.Get(), nullptr,
IID_PPV_ARGS(creationList.GetAddressOf())), "Create feature command list");
check(shim.symbol<Create>("NVSDK_NGX_D3D12_CreateFeature")(creationList.Get(), 1, parameters, &handle), "D3D12 CreateFeature");
dx(creationList->Close(), "Close feature creation list");
ID3D12CommandList* creationBatch[] = {creationList.Get()}; queue->ExecuteCommandLists(1, creationBatch); drain();
auto release = shim.symbol<Release>("NVSDK_NGX_D3D12_ReleaseFeature");
struct FeatureCleanup { Release fn; void* h; ~FeatureCleanup() { (void)fn(h); } } featureCleanup{release, handle};
static const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) d4r_ngx_set_d3d12_resource(parameters, names[i], textures[i].image.Get());
if (enhanced && !commandBackend) {
const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
for (unsigned i = 0; i < 5; ++i) { const std::string prefix(states[i]); d4r_ngx_set_uint(parameters,(prefix+".Enhanced").c_str(),1);
d4r_ngx_set_uint(parameters,(prefix+".Layout").c_str(), i == 3 ? D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_UNORDERED_ACCESS : D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_SHADER_RESOURCE);
d4r_ngx_set_uint(parameters,(prefix+".Access").c_str(), i == 3 ? D3D12_BARRIER_ACCESS_UNORDERED_ACCESS : D3D12_BARRIER_ACCESS_SHADER_RESOURCE); }
}
d4r_ngx_set_int(parameters, "Disable.Watermark", 1);
auto readback = make_buffer(device.Get(), textures[3].bytes, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
ComPtr<ID3D12GraphicsCommandList> predecessor;
ComPtr<ID3D12CommandAllocator> predecessorAllocator;
if (commandBackend) {
if (inherited) {
// Record state only in predecessor A after the backend has
// initialized. List B's Evaluate must resolve it at submission,
// after A, and retain it across subsequent frames/resets.
for (unsigned i = 0; i < 5; ++i) {
if (enhanced) enhancedBarrier(list.Get(), textures[i].image.Get(), i, true);
else transition(list.Get(), textures[i].image.Get(), D3D12_RESOURCE_STATE_COMMON,
i == 3 ? D3D12_RESOURCE_STATE_UNORDERED_ACCESS : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
}
dx(list->Close(), "Close tracked predecessor A");
}
// Keep initial synthetic producers unsubmitted until the first
// batch, before the list which contains the NGX evaluation.
predecessor = list; predecessorAllocator = allocator;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.ReleaseAndGetAddressOf())), "Create game-style allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(list.ReleaseAndGetAddressOf())), "Create game-style command list");
dx(list->Close(), "Close initial game-style list");
}
const bool burst = std::getenv("D4R_DIAG_BURST") != nullptr;
const bool recreate = std::getenv("D4R_DIAG_RECREATE") != nullptr;
if (burst && (!commandBackend || args.iterations > 8)) throw std::runtime_error("Burst requires command-list backend and at most 8 frames");
struct SubmittedFrame {
ComPtr<ID3D12CommandAllocator> allocator;
ComPtr<ID3D12GraphicsCommandList> list;
ComPtr<ID3D12Resource> readback;
};
std::vector<SubmittedFrame> submitted;
std::vector<std::pair<ComPtr<ID3D12CommandAllocator>, ComPtr<ID3D12GraphicsCommandList>>> creation_owners;
auto validate_output = [&](ID3D12Resource* image_readback, unsigned frame) {
void* raw = nullptr; D3D12_RANGE range{0, size_t(textures[3].bytes)};
dx(image_readback->Map(0, &range, &raw), "Map(verification only)");
std::vector<uint16_t> pixels(size_t(outWidth) * outHeight * 4);
const auto format = resource_desc(textures[3].image.Get()).Format;
if (format == DXGI_FORMAT_R16G16B16A16_FLOAT) {
for (unsigned y = 0; y < outHeight; ++y) std::memcpy(pixels.data() + size_t(y) * outWidth * 4,
static_cast<uint8_t*>(raw) + y * textures[3].footprint.Footprint.RowPitch, outWidth * 8);
} else {
std::ofstream native(directory / ("output-" + std::to_string(frame) + (args.pixel_profile == "packed" ? ".r11g11b10f" : ".bgra8")), std::ios::binary);
// Independently decode the diagnostic readback for previews and
// frame comparisons. No CPU image copies are in the backend.
static const auto unormHalf = [] {
std::array<uint16_t, 256> lut{};
for (unsigned b = 0; b < 256; ++b) { double best = INFINITY; for (unsigned h = 0; h <= 0x3c00; ++h) { double error = std::abs(double(half_value(uint16_t(h))) - double(b)/255.); if (error < best || (error == best && !(h & 1))) { best = error; lut[b] = uint16_t(h); } } }
return lut;
}();
for (unsigned y = 0; y < outHeight; ++y) {
const auto* row = static_cast<uint8_t*>(raw) + y * textures[3].footprint.Footprint.RowPitch;
native.write(reinterpret_cast<const char*>(row), outWidth * 4);
for (unsigned x = 0; x < outWidth; ++x) {
auto* p = pixels.data() + (size_t(y) * outWidth + x) * 4;
if (format == DXGI_FORMAT_R11G11B10_FLOAT) { uint32_t value = 0; std::memcpy(&value,row + x * 4,4); p[0] = uint16_t((value & 2047) << 4); p[1] = uint16_t(((value >> 11) & 2047) << 4); p[2] = uint16_t((value >> 22) << 5); p[3] = 0x3c00; }
else for (unsigned c = 0; c < 4; ++c) p[c] = unormHalf[row[x * 4 + (c == 0 ? 2 : c == 2 ? 0 : c)]];
}
}
if (!native) throw std::runtime_error("Saving native output format failed");
}
D3D12_RANGE noWrite{}; image_readback->Unmap(0, &noWrite);
double sum = 0, square = 0;
for (size_t i = 0; i < pixels.size(); ++i) {
const float value = half_value(pixels[i]);
if (!std::isfinite(value)) throw std::runtime_error("D3D12 output contains NaN/Inf or unwritten pixels");
if ((i & 3) < 3) { sum += value; square += double(value) * value; }
}
const double count = outWidth * outHeight * 3, variance = square / count - (sum / count) * (sum / count);
save_output(directory, pixels, outWidth, outHeight, frame);
std::printf("D3D12_OUTPUT preset=%u frame=%u finite=1 mean=%.9g variance=%.9g frame_age=0\n", args.preset, frame, sum/count, variance);
if (variance < 1e-6) throw std::runtime_error("D3D12 output lost the synthetic pattern");
};
for (unsigned frame = 0; frame < args.iterations; ++frame) {
if (burst) {
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.ReleaseAndGetAddressOf())), "Burst allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.ReleaseAndGetAddressOf())), "Burst list");
dx(list->Close(), "Burst initial Close");
readback = make_buffer(device.Get(), textures[3].bytes, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
}
d4r_ngx_set_int(parameters, "Reset", frame == 0 ? 1 : 0);
if (commandBackend) {
dx(allocator->Reset(), "Reset(game-style allocator)"); dx(list->Reset(allocator.Get(), nullptr), "Reset(game-style list)");
if (earlySignature) {
list->SetComputeRootSignature(earlyRoot.Get()); list->SetPipelineState(earlyPso.Get());
list->ExecuteIndirect(earlySignature.Get(), 1, earlyArguments.Get(), 0, nullptr, 0);
}
if (enhanced && !inherited) for (unsigned i = 0; i < 5; ++i) enhancedBarrier(list.Get(), textures[i].image.Get(), i, false);
using RecordedEvaluate = unsigned(*)(ID3D12GraphicsCommandList*, void*, void*, void*);
check(shim.symbol<RecordedEvaluate>("NVSDK_NGX_D3D12_EvaluateFeature")(list.Get(), handle, parameters, nullptr), "D3D12 recorded same-frame Evaluate");
// The caller may reuse/mutate its parameter object as soon as
// Evaluate returns. The queued snapshot must remain immutable.
d4r_ngx_set_d3d12_resource(parameters, "Color", nullptr);
d4r_ngx_set_int(parameters, "Reset", 0);
} else {
check(shim.symbol<Evaluate>("d4r_D3D12_EvaluateAtBoundary")(queue.Get(), handle, parameters), "D3D12 same-frame Evaluate");
dx(allocator->Reset(), "Reset(harness allocator)"); dx(list->Reset(allocator.Get(), nullptr), "Reset(harness list)");
}
ResourceAccess outputAccess(D3D12_RESOURCE_STATE_UNORDERED_ACCESS); outputAccess.enhanced = enhanced;
outputAccess.layout = D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_UNORDERED_ACCESS; outputAccess.access = D3D12_BARRIER_ACCESS_UNORDERED_ACCESS;
texture_copy_barrier(list.Get(), textures[3].image.Get(), outputAccess, false, false);
D3D12_TEXTURE_COPY_LOCATION src{}, dst{};
src.pResource = textures[3].image.Get(); src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
dst.pResource = readback.Get(); dst.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; dst.PlacedFootprint = textures[3].footprint;
list->CopyTextureRegion(&dst, 0, 0, 0, &src, nullptr);
texture_copy_barrier(list.Get(), textures[3].image.Get(), outputAccess, false, true);
dx(list->Close(), "Close(verification)");
if (commandBackend && frame == 0) {
ID3D12CommandList* batch[] = {predecessor.Get(), list.Get()}; queue->ExecuteCommandLists(2, batch);
} else { ID3D12CommandList* verify[] = {list.Get()}; queue->ExecuteCommandLists(1, verify); }
if (!burst) drain(); else submitted.push_back({allocator, list, readback});
if (commandBackend) d4r_ngx_set_d3d12_resource(parameters, "Color", textures[0].image.Get());
if (!burst) validate_output(readback.Get(), frame);
if (recreate && frame + 1 < args.iterations) {
// Release/create without an explicit harness drain. In burst
// mode earlier frames can still be using the old feature.
check(release(handle), "Recreate ReleaseFeature"); featureCleanup.h = nullptr;
dx(creationAllocator->Reset(), "Recreate allocator Reset");
dx(creationList->Reset(creationAllocator.Get(), nullptr), "Recreate list Reset");
handle = nullptr;
check(shim.symbol<Create>("NVSDK_NGX_D3D12_CreateFeature")(creationList.Get(), 1, parameters, &handle), "Recreate CreateFeature");
featureCleanup.h = handle;
dx(creationList->Close(), "Recreate list Close");
ID3D12CommandList* created[] = {creationList.Get()}; queue->ExecuteCommandLists(1, created);
creation_owners.emplace_back(creationAllocator, creationList);
// This empty creation list owns no work; the next recreation
// allocates independent command memory without waiting on GPU.
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(creationAllocator.ReleaseAndGetAddressOf())), "Next recreate allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, creationAllocator.Get(), nullptr, IID_PPV_ARGS(creationList.ReleaseAndGetAddressOf())), "Next recreate list");
dx(creationList->Close(), "Next recreate initial Close");
std::printf("D4R_RECREATE_VALIDATION submitted_frame=%u next_feature_created=1\n", frame);
}
}
if (burst) {
drain();
for (unsigned frame = 0; frame < submitted.size(); ++frame) validate_output(submitted[frame].readback.Get(), frame);
std::printf("D4R_BURST_VALIDATION frames=%zu cpu_readback_after_all_submissions=1\n", submitted.size());
}
loaded_modules();
std::printf("PASS D3D12_DLSS architecture=%s preset=%u frames=%u fast_path_cpu_copies=0 frame_age=0 enhanced=%u queue_integration=%s\n", d4r::diag::HipApi::target_arch(),
args.preset, args.iterations, enhanced, commandBackend ? "recorded_command_list" : "explicit_harness");
return 0;
} catch (const std::exception& error) { std::fprintf(stderr, "FAIL D3D12_DLSS %s\n", error.what()); loaded_modules(); return 4; }
}
tools/windows/d3d12_external.h
-246
View File
@@ -1,246 +0,0 @@
#pragma once
#if defined(__MINGW32__)
#define WIDL_EXPLICIT_AGGREGATE_RETURNS
#endif
#include "hip_api.h"
#include <d3d12.h>
#include <wrl/client.h>
namespace d4r::win {
using Microsoft::WRL::ComPtr;
using diag::HipApi;
inline void dx(HRESULT result, const char* operation) {
if (FAILED(result)) throw std::runtime_error(std::string(operation) + " HRESULT=" + std::to_string(result));
}
struct Handle {
HANDLE value = nullptr;
Handle() = default;
Handle(const Handle&) = delete;
~Handle() { if (value) CloseHandle(value); }
};
struct ExternalApi {
HipApi& hip;
#define D4R_EXTERNAL(name) decltype(&::name) name = hip.library.symbol<decltype(&::name)>(#name)
D4R_EXTERNAL(hipImportExternalMemory);
D4R_EXTERNAL(hipExternalMemoryGetMappedBuffer);
D4R_EXTERNAL(hipDestroyExternalMemory);
D4R_EXTERNAL(hipImportExternalSemaphore);
D4R_EXTERNAL(hipDestroyExternalSemaphore);
D4R_EXTERNAL(hipWaitExternalSemaphoresAsync);
D4R_EXTERNAL(hipSignalExternalSemaphoresAsync);
#undef D4R_EXTERNAL
};
inline D3D12_RESOURCE_DESC resource_desc(ID3D12Resource* resource) {
#if defined(__MINGW32__)
D3D12_RESOURCE_DESC desc{}; resource->GetDesc(&desc); return desc;
#else
return resource->GetDesc();
#endif
}
inline LUID adapter_luid(ID3D12Device* device) {
#if defined(__MINGW32__)
LUID luid{}; device->GetAdapterLuid(&luid); return luid;
#else
return device->GetAdapterLuid();
#endif
}
inline ComPtr<ID3D12Resource> make_buffer(ID3D12Device* device, uint64_t bytes,
D3D12_HEAP_TYPE heapType, D3D12_HEAP_FLAGS flags, D3D12_RESOURCE_STATES state) {
D3D12_HEAP_PROPERTIES heap{}; heap.Type = heapType;
heap.CreationNodeMask = heap.VisibleNodeMask = 1;
D3D12_RESOURCE_DESC desc{};
desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; desc.Width = bytes;
desc.Height = desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
ComPtr<ID3D12Resource> resource;
dx(device->CreateCommittedResource(&heap, flags, &desc, state, nullptr,
IID_PPV_ARGS(resource.GetAddressOf())), "CreateCommittedResource(buffer)");
return resource;
}
inline void transition(ID3D12GraphicsCommandList* list, ID3D12Resource* resource,
D3D12_RESOURCE_STATES before, D3D12_RESOURCE_STATES after) {
if (before == after) return;
D3D12_RESOURCE_BARRIER barrier{};
barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barrier.Transition = {resource, D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, before, after};
list->ResourceBarrier(1, &barrier);
}
struct ResourceAccess {
D3D12_RESOURCE_STATES legacy = D3D12_RESOURCE_STATE_COMMON;
bool enhanced = false, pending_split = false, inherited = false;
D3D12_BARRIER_LAYOUT layout = D3D12_BARRIER_LAYOUT_COMMON;
D3D12_BARRIER_ACCESS access = D3D12_BARRIER_ACCESS_COMMON;
D3D12_BARRIER_SYNC sync = D3D12_BARRIER_SYNC_ALL;
ResourceAccess() = default;
ResourceAccess(D3D12_RESOURCE_STATES state) : legacy(state) {}
};
inline void texture_copy_barrier(ID3D12GraphicsCommandList* list, ID3D12Resource* texture,
const ResourceAccess& original, bool output, bool restore) {
if (original.pending_split) throw std::runtime_error("Cannot use a texture inside an unfinished split barrier");
if (!original.enhanced) {
const auto copy = output ? D3D12_RESOURCE_STATE_COPY_DEST : D3D12_RESOURCE_STATE_COPY_SOURCE;
if (original.legacy == copy) return;
D3D12_RESOURCE_BARRIER barrier{}; barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barrier.Transition = {texture, 0, restore ? copy : original.legacy, restore ? original.legacy : copy};
list->ResourceBarrier(1, &barrier); return;
}
ComPtr<ID3D12GraphicsCommandList7> newer;
dx(list->QueryInterface(IID_PPV_ARGS(newer.GetAddressOf())), "Enhanced interop command list");
D3D12_TEXTURE_BARRIER barrier{}; barrier.pResource = texture;
barrier.LayoutBefore = restore ? (output ? D3D12_BARRIER_LAYOUT_COPY_DEST : D3D12_BARRIER_LAYOUT_COPY_SOURCE) : original.layout;
barrier.LayoutAfter = restore ? original.layout : (output ? D3D12_BARRIER_LAYOUT_COPY_DEST : D3D12_BARRIER_LAYOUT_COPY_SOURCE);
barrier.AccessBefore = restore ? (output ? D3D12_BARRIER_ACCESS_COPY_DEST : D3D12_BARRIER_ACCESS_COPY_SOURCE) : original.access;
barrier.AccessAfter = restore ? original.access : (output ? D3D12_BARRIER_ACCESS_COPY_DEST : D3D12_BARRIER_ACCESS_COPY_SOURCE);
// Interop runs in a separate submitted direct list. ALL is a conservative
// synchronization scope for the app's original access class/layout.
barrier.SyncBefore = restore ? D3D12_BARRIER_SYNC_COPY : D3D12_BARRIER_SYNC_ALL;
barrier.SyncAfter = restore ? D3D12_BARRIER_SYNC_ALL : D3D12_BARRIER_SYNC_COPY;
barrier.Subresources.IndexOrFirstMipLevel = 0; // Single subresource 0.
D3D12_BARRIER_GROUP group{}; group.Type = D3D12_BARRIER_TYPE_TEXTURE; group.NumBarriers = 1; group.pTextureBarriers = &barrier;
newer->Barrier(1, &group);
}
class SharedPlane {
ExternalApi& api_;
hipExternalMemory_t memory_ = nullptr;
void release() noexcept {
if (mapped) { (void)api_.hip.hipFree(mapped); mapped = nullptr; }
if (memory_) { (void)api_.hipDestroyExternalMemory(memory_); memory_ = nullptr; }
}
public:
ComPtr<ID3D12Resource> buffer;
D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint{};
UINT64 bytes = 0;
void* mapped = nullptr;
SharedPlane(ExternalApi& api, ID3D12Device* device, const D3D12_RESOURCE_DESC& texture) : api_(api) {
UINT rows = 0; UINT64 rowBytes = 0;
device->GetCopyableFootprints(&texture, 0, 1, 0, &footprint, &rows, &rowBytes, &bytes);
if (texture.Format == DXGI_FORMAT_R32G8X24_TYPELESS || texture.Format == DXGI_FORMAT_D32_FLOAT_S8X24_UINT || texture.Format == DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS) {
// D3D12 exposes D32S8 as separate depth/stencil copy planes. Only
// subresource 0 (R32 depth) belongs to NGX; never copy interleaved
// 64-bit texels or touch the stencil plane.
if ((footprint.Footprint.Format != DXGI_FORMAT_R32_FLOAT && footprint.Footprint.Format != DXGI_FORMAT_R32_TYPELESS) || rowBytes != texture.Width * 4)
throw std::runtime_error("Unexpected D32S8 depth-plane copy footprint");
}
if (!bytes || bytes == UINT64_MAX || !footprint.Footprint.RowPitch)
throw std::runtime_error("Invalid shared texture footprint");
buffer = make_buffer(device, bytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_HEAP_FLAG_SHARED, D3D12_RESOURCE_STATE_COMMON);
const auto desc = resource_desc(buffer.Get());
D3D12_RESOURCE_ALLOCATION_INFO allocation{};
#if defined(__MINGW32__)
device->GetResourceAllocationInfo(&allocation, 0, 1, &desc);
#else
allocation = device->GetResourceAllocationInfo(0, 1, &desc);
#endif
Handle handle;
dx(device->CreateSharedHandle(buffer.Get(), nullptr, GENERIC_ALL, nullptr, &handle.value), "CreateSharedHandle(VRAM)");
try {
hipExternalMemoryHandleDesc import{};
import.type = hipExternalMemoryHandleTypeD3D12Resource;
import.handle.win32.handle = handle.value;
import.size = allocation.SizeInBytes; import.flags = hipExternalMemoryDedicated;
api.hip.check(api.hipImportExternalMemory(&memory_, &import), "hipImportExternalMemory(D3D12)");
hipExternalMemoryBufferDesc map{}; map.size = bytes;
api.hip.check(api.hipExternalMemoryGetMappedBuffer(&mapped, memory_, &map), "hipExternalMemoryGetMappedBuffer");
} catch (...) { release(); throw; }
// Win32 imports retain their own reference; the exporter closes its HANDLE.
std::printf("D4R_IMPORT width=%u height=%u pitch=%u allocation=%llu cpu_copy=0\n",
footprint.Footprint.Width, footprint.Footprint.Height, footprint.Footprint.RowPitch, allocation.SizeInBytes);
}
SharedPlane(const SharedPlane&) = delete;
~SharedPlane() { release(); }
void copy_input(ID3D12GraphicsCommandList* list, ID3D12Resource* texture, ResourceAccess state) {
texture_copy_barrier(list, texture, state, false, false);
transition(list, buffer.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_DEST);
D3D12_TEXTURE_COPY_LOCATION source{}, dest{};
source.pResource = texture; source.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
dest.pResource = buffer.Get(); dest.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; dest.PlacedFootprint = footprint;
list->CopyTextureRegion(&dest, 0, 0, 0, &source, nullptr);
transition(list, buffer.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_COMMON);
texture_copy_barrier(list, texture, state, false, true);
}
void copy_output(ID3D12GraphicsCommandList* list, ID3D12Resource* texture, ResourceAccess state) {
texture_copy_barrier(list, texture, state, true, false);
transition(list, buffer.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_SOURCE);
D3D12_TEXTURE_COPY_LOCATION source{}, dest{};
source.pResource = buffer.Get(); source.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; source.PlacedFootprint = footprint;
dest.pResource = texture; dest.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
list->CopyTextureRegion(&dest, 0, 0, 0, &source, nullptr);
transition(list, buffer.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COMMON);
texture_copy_barrier(list, texture, state, true, true);
}
};
class SharedTimeline {
ExternalApi& api_;
hipExternalSemaphore_t semaphore_ = nullptr;
hipStream_t stream_ = nullptr;
Handle event_;
uint64_t value_ = 0;
public:
struct Ticket { uint64_t input, output; };
ComPtr<ID3D12Fence> fence;
SharedTimeline(ExternalApi& api, ID3D12Device* device) : api_(api) {
dx(device->CreateFence(0, D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS(fence.GetAddressOf())), "CreateFence(D3D12/HIP)");
Handle handle;
dx(device->CreateSharedHandle(fence.Get(), nullptr, GENERIC_ALL, nullptr, &handle.value), "CreateSharedHandle(fence)");
event_.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event_.value) throw std::runtime_error("CreateEvent(fence) failed");
hipExternalSemaphoreHandleDesc import{};
import.type = hipExternalSemaphoreHandleTypeD3D12Fence;
import.handle.win32.handle = handle.value;
api.hip.check(api.hipImportExternalSemaphore(&semaphore_, &import), "hipImportExternalSemaphore(D3D12Fence)");
try { api.hip.check(api.hip.hipStreamCreateWithFlags(&stream_, hipStreamNonBlocking), "hipStreamCreate(interop)"); }
catch (...) { (void)api.hipDestroyExternalSemaphore(semaphore_); semaphore_ = nullptr; throw; }
}
~SharedTimeline() {
if (stream_) { (void)api_.hip.hipStreamSynchronize(stream_); (void)api_.hip.hipStreamDestroy(stream_); }
if (semaphore_) (void)api_.hipDestroyExternalSemaphore(semaphore_);
}
// Caller must serialize transactions and submit the producer prefix first.
void wait_input(ID3D12CommandQueue* queue) {
const uint64_t ready = ++value_;
dx(queue->Signal(fence.Get(), ready), "Queue Signal(input)");
wait_ready(ready);
}
// Reserve both dependencies on the submission thread. The CUDA worker
// never submits to the D3D12 queue, so later game Signal/Wait calls retain
// their original ordering without waiting on a CPU callback.
Ticket reserve() { return {++value_, ++value_}; }
void enqueue(ID3D12CommandQueue* queue, Ticket ticket) {
dx(queue->Signal(fence.Get(), ticket.input), "Queue Signal(async input)");
dx(queue->Wait(fence.Get(), ticket.output), "Queue Wait(async output)");
}
void wait_ready(uint64_t ready) {
hipExternalSemaphoreWaitParams wait{}; wait.params.fence.value = ready;
api_.hip.check(api_.hipWaitExternalSemaphoresAsync(&semaphore_, &wait, 1, stream_), "HIP wait(D3D12 input)");
api_.hip.check(api_.hip.hipStreamSynchronize(stream_), "HIP input wait completion");
// Establish host-side completion as well: the Windows HIP semaphore
// wait currently needs a separately verified completion boundary before
// work is launched on CUDA's other streams. No image crosses the CPU.
if (fence->GetCompletedValue() < ready) {
dx(fence->SetEventOnCompletion(ready, event_.value), "SetEventOnCompletion(input)");
if (WaitForSingleObject(event_.value, 30000) != WAIT_OBJECT_0)
throw std::runtime_error("D3D12 input completion timeout");
}
}
void signal_output(ID3D12CommandQueue* queue) {
const uint64_t done = ++value_;
signal_ready(done);
dx(queue->Wait(fence.Get(), done), "Queue Wait(HIP output)");
}
void signal_ready(uint64_t done) {
hipExternalSemaphoreSignalParams signal{}; signal.params.fence.value = done;
api_.hip.check(api_.hipSignalExternalSemaphoresAsync(&semaphore_, &signal, 1, stream_), "HIP signal(output)");
}
void drain(ID3D12CommandQueue* queue) {
const uint64_t done = ++value_;
dx(queue->Signal(fence.Get(), done), "Queue Signal(completion)");
if (fence->GetCompletedValue() < done) {
dx(fence->SetEventOnCompletion(done, event_.value), "SetEventOnCompletion");
if (WaitForSingleObject(event_.value, 30000) != WAIT_OBJECT_0)
throw std::runtime_error("D3D12 completion timeout; resources must remain alive");
}
}
};
}
tools/windows/d3d12_hip_interop_probe.cpp
-346
View File
@@ -1,346 +0,0 @@
#if defined(__MINGW32__)
#define WIDL_EXPLICIT_AGGREGATE_RETURNS
#endif
#include "hip_api.h"
#include <d3d12.h>
#include <dxgi1_6.h>
#include <wrl/client.h>
using Microsoft::WRL::ComPtr;
using namespace d4r::diag;
static void handle_checkpoint(const char* stage)
{
DWORD count = 0;
if (!GetProcessHandleCount(GetCurrentProcess(), &count))
throw std::runtime_error("GetProcessHandleCount failed");
std::printf("HANDLE_STAGE %s count=%lu\n", stage, count);
}
static void dx(HRESULT hr, const char* call)
{
std::printf("D3D12 %s -> 0x%08lx\n", call, static_cast<unsigned long>(hr));
if (FAILED(hr)) throw std::runtime_error(std::string(call) + " HRESULT=" + std::to_string(hr));
}
struct NativeHandle {
HANDLE handle = nullptr;
~NativeHandle() { if (handle) CloseHandle(handle); }
};
struct ExternalApi {
HipApi& hip;
#define EXT(name) decltype(&::name) name = hip.library.symbol<decltype(&::name)>(#name)
EXT(hipImportExternalMemory);
EXT(hipExternalMemoryGetMappedBuffer);
EXT(hipDestroyExternalMemory);
EXT(hipImportExternalSemaphore);
EXT(hipDestroyExternalSemaphore);
EXT(hipWaitExternalSemaphoresAsync);
EXT(hipSignalExternalSemaphoresAsync);
#undef EXT
};
struct Imported {
ExternalApi& api;
hipExternalMemory_t memory = nullptr;
hipExternalSemaphore_t semaphore = nullptr;
hipStream_t stream = nullptr;
bool owns_stream = true;
void* mapped = nullptr;
void* errors = nullptr;
~Imported() {
// No resource may be released while the HIP stream is still using it.
if (stream) (void)api.hip.hipStreamSynchronize(stream);
if (stream && owns_stream) (void)api.hip.hipStreamDestroy(stream);
if (errors) (void)api.hip.hipFree(errors);
if (mapped) (void)api.hip.hipFree(mapped);
if (memory) (void)api.hipDestroyExternalMemory(memory);
if (semaphore) (void)api.hipDestroyExternalSemaphore(semaphore);
}
};
static D3D12_RESOURCE_DESC buffer_desc(uint64_t bytes)
{
D3D12_RESOURCE_DESC desc{};
desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
desc.Width = bytes; desc.Height = 1; desc.DepthOrArraySize = 1;
desc.MipLevels = 1; desc.SampleDesc.Count = 1;
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
return desc;
}
static ComPtr<ID3D12Resource> make_buffer(ID3D12Device* device, uint64_t bytes,
D3D12_HEAP_TYPE type, D3D12_HEAP_FLAGS flags, D3D12_RESOURCE_STATES state)
{
D3D12_HEAP_PROPERTIES properties{};
properties.Type = type;
properties.CreationNodeMask = properties.VisibleNodeMask = 1;
auto desc = buffer_desc(bytes);
ComPtr<ID3D12Resource> resource;
dx(device->CreateCommittedResource(&properties, flags, &desc, state, nullptr,
IID_PPV_ARGS(resource.GetAddressOf())), "CreateCommittedResource(buffer)");
return resource;
}
static void transition(ID3D12GraphicsCommandList* list, ID3D12Resource* resource,
D3D12_RESOURCE_STATES before, D3D12_RESOURCE_STATES after)
{
D3D12_RESOURCE_BARRIER barrier{};
barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
barrier.Transition.pResource = resource;
barrier.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
barrier.Transition.StateBefore = before; barrier.Transition.StateAfter = after;
list->ResourceBarrier(1, &barrier);
}
static void import_lifetime(ID3D12Device* device, ExternalApi& api, const Args& args)
{
if (args.interop_mode != "resource" && args.interop_mode != "import" && args.interop_mode != "map")
throw std::runtime_error("--interop-mode must be roundtrip, resource, import or map");
DWORD baseline = 0, handles = 0;
for (unsigned i = 0; i < args.iterations; ++i) {
{
auto resource = make_buffer(device, 65536, D3D12_HEAP_TYPE_DEFAULT,
D3D12_HEAP_FLAG_SHARED, D3D12_RESOURCE_STATE_COMMON);
NativeHandle handle;
dx(device->CreateSharedHandle(resource.Get(), nullptr, GENERIC_ALL, nullptr,
&handle.handle), "CreateSharedHandle(lifecycle)");
Imported imported{api};
if (args.interop_mode != "resource") {
hipExternalMemoryHandleDesc desc{};
desc.type = hipExternalMemoryHandleTypeD3D12Resource;
desc.handle.win32.handle = handle.handle;
desc.size = 65536;
desc.flags = hipExternalMemoryDedicated;
api.hip.check(api.hipImportExternalMemory(&imported.memory, &desc), "hipImportExternalMemory(lifecycle)");
if (args.interop_mode == "map") {
hipExternalMemoryBufferDesc map{}; map.size = 65536;
api.hip.check(api.hipExternalMemoryGetMappedBuffer(&imported.mapped, imported.memory, &map), "hipExternalMemoryGetMappedBuffer(lifecycle)");
api.hip.check(api.hip.hipFree(imported.mapped), "hipFree(lifecycle)"); imported.mapped = nullptr;
}
api.hip.check(api.hipDestroyExternalMemory(imported.memory), "hipDestroyExternalMemory(lifecycle)"); imported.memory = nullptr;
}
}
if (!GetProcessHandleCount(GetCurrentProcess(), &handles)) throw std::runtime_error("GetProcessHandleCount failed");
size_t free_bytes = 0, total_bytes = 0;
api.hip.check(api.hip.hipMemGetInfo(&free_bytes, &total_bytes), "hipMemGetInfo");
if (i == 1 || (args.iterations == 1 && i == 0)) baseline = handles;
std::printf("IMPORT_CLEANUP mode=%s iteration=%u handles=%lu baseline=%lu free_vram=%zu\n",
args.interop_mode.c_str(), i, handles, baseline, free_bytes);
}
if (handles > baseline + 4) throw std::runtime_error("External memory lifecycle handle growth");
std::printf("PASS INTEROP_LIFETIME mode=%s iterations=%u\n", args.interop_mode.c_str(), args.iterations);
}
int main(int argc, char** argv)
{
start();
try {
Args args(argc, argv);
if (args.module.empty() && args.interop_mode == "roundtrip") throw std::runtime_error("--module required");
if (args.stream_lifetime != "persistent" && args.stream_lifetime != "cycle")
throw std::runtime_error("--stream-lifetime must be persistent or cycle");
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
ExternalApi external{hip};
ComPtr<IDXGIFactory4> factory;
dx(CreateDXGIFactory1(IID_PPV_ARGS(factory.GetAddressOf())), "CreateDXGIFactory1");
ComPtr<IDXGIAdapter1> adapter;
bool matched = false;
for (UINT i = 0; ; ++i) {
const HRESULT enumerated = factory->EnumAdapters1(i, adapter.ReleaseAndGetAddressOf());
if (enumerated == DXGI_ERROR_NOT_FOUND) break;
dx(enumerated, "EnumAdapters1");
DXGI_ADAPTER_DESC1 desc{};
dx(adapter->GetDesc1(&desc), "GetDesc1");
std::printf("DXGI adapter=%s LUID=%08lx:%08lx\n", utf8(desc.Description).c_str(),
static_cast<unsigned long>(desc.AdapterLuid.HighPart), desc.AdapterLuid.LowPart);
if (std::memcmp(&desc.AdapterLuid, props.luid, sizeof(LUID)) == 0) { matched = true; break; }
}
if (!matched) throw std::runtime_error("HIP/DXGI LUID mismatch: cross-adapter import prohibited");
ComPtr<ID3D12Device> device;
dx(D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_12_0, IID_PPV_ARGS(device.GetAddressOf())), "D3D12CreateDevice");
if (args.interop_mode != "roundtrip") { import_lifetime(device.Get(), external, args); return 0; }
ComPtr<ID3D12CommandQueue> queue;
D3D12_COMMAND_QUEUE_DESC qd{};
qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
dx(device->CreateCommandQueue(&qd, IID_PPV_ARGS(queue.GetAddressOf())), "CreateCommandQueue");
ComPtr<ID3D12Fence> shared_fence, complete;
dx(device->CreateFence(0, D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS(shared_fence.GetAddressOf())), "CreateFence(shared)");
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(complete.GetAddressOf())), "CreateFence(completion)");
NativeHandle fence_handle, event;
event.handle = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.handle) throw std::runtime_error("CreateEvent failed");
dx(device->CreateSharedHandle(shared_fence.Get(), nullptr, GENERIC_ALL, nullptr,
&fence_handle.handle), "CreateSharedHandle(fence)");
// R32_UINT texture -> linear shared buffer -> HIP -> texture -> verification readback.
constexpr UINT width = 256, height = 64, count = width * height;
const uint64_t bytes = static_cast<uint64_t>(count) * sizeof(uint32_t);
auto upload = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
auto readback = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
D3D12_RESOURCE_DESC td{};
td.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D;
td.Width = width; td.Height = height; td.DepthOrArraySize = 1; td.MipLevels = 1;
td.SampleDesc.Count = 1; td.Format = DXGI_FORMAT_R32_UINT;
D3D12_HEAP_PROPERTIES hp{}; hp.Type = D3D12_HEAP_TYPE_DEFAULT;
hp.CreationNodeMask = hp.VisibleNodeMask = 1;
ComPtr<ID3D12Resource> texture;
dx(device->CreateCommittedResource(&hp, D3D12_HEAP_FLAG_NONE, &td, D3D12_RESOURCE_STATE_COPY_DEST,
nullptr, IID_PPV_ARGS(texture.GetAddressOf())), "CreateCommittedResource(texture)");
D3D12_PLACED_SUBRESOURCE_FOOTPRINT footprint{};
UINT rows; UINT64 row_bytes, total_bytes;
device->GetCopyableFootprints(&td, 0, 1, 0, &footprint, &rows, &row_bytes, &total_bytes);
if (total_bytes != bytes || footprint.Footprint.RowPitch != width * 4)
throw std::runtime_error("Unexpected footprint; test requires no padding");
auto texture_copy = D3D12_TEXTURE_COPY_LOCATION{};
texture_copy.pResource = texture.Get();
texture_copy.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
auto linear_copy = D3D12_TEXTURE_COPY_LOCATION{};
linear_copy.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
linear_copy.PlacedFootprint = footprint;
hipModule_t module = nullptr;
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad");
struct ModuleCleanup { HipApi& hip; hipModule_t m; ~ModuleCleanup() { (void)hip.hipModuleUnload(m); } } mc{hip, module};
hipFunction_t kernel = nullptr;
hip.check(hip.hipModuleGetFunction(&kernel, module, "d4r_probe_interop"), "hipModuleGetFunction");
struct StreamCleanup {
HipApi& hip;
hipStream_t stream = nullptr;
~StreamCleanup() { if (stream) {
(void)hip.hipStreamSynchronize(stream);
(void)hip.hipStreamDestroy(stream);
} }
} persistent{hip};
if (args.stream_lifetime == "persistent")
hip.check(hip.hipStreamCreateWithFlags(&persistent.stream, hipStreamNonBlocking), "hipStreamCreateWithFlags(persistent)");
loaded_modules();
DWORD handles_before = 0; GetProcessHandleCount(GetCurrentProcess(), &handles_before);
DWORD warmed_handles = 0;
DWORD final_cycle_handles = 0;
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
const uint32_t seed = 0x12345678u + iteration * 31u;
auto shared = make_buffer(device.Get(), bytes, D3D12_HEAP_TYPE_DEFAULT, D3D12_HEAP_FLAG_SHARED,
D3D12_RESOURCE_STATE_COPY_DEST);
const auto shared_desc = buffer_desc(bytes);
D3D12_RESOURCE_ALLOCATION_INFO allocation_info{};
#if defined(__MINGW32__)
device->GetResourceAllocationInfo(&allocation_info, 0, 1, &shared_desc);
#else
allocation_info = device->GetResourceAllocationInfo(0, 1, &shared_desc);
#endif
NativeHandle memory_handle;
dx(device->CreateSharedHandle(shared.Get(), nullptr, GENERIC_ALL, nullptr,
&memory_handle.handle), "CreateSharedHandle(resource)");
Imported imported{external};
handle_checkpoint("created_resource");
hipExternalMemoryHandleDesc memory_desc{};
memory_desc.type = hipExternalMemoryHandleTypeD3D12Resource;
memory_desc.handle.win32.handle = memory_handle.handle;
memory_desc.size = allocation_info.SizeInBytes;
memory_desc.flags = hipExternalMemoryDedicated;
std::printf("STAGE hipImportExternalMemory(D3D12Resource) size=%llu\n", memory_desc.size);
hip.check(external.hipImportExternalMemory(&imported.memory, &memory_desc), "hipImportExternalMemory(D3D12Resource)");
handle_checkpoint("imported_memory");
hipExternalMemoryBufferDesc map_desc{}; map_desc.size = bytes;
hip.check(external.hipExternalMemoryGetMappedBuffer(&imported.mapped, imported.memory, &map_desc), "hipExternalMemoryGetMappedBuffer");
handle_checkpoint("mapped_memory");
hipExternalSemaphoreHandleDesc sem_desc{};
sem_desc.type = hipExternalSemaphoreHandleTypeD3D12Fence;
sem_desc.handle.win32.handle = fence_handle.handle;
std::printf("STAGE hipImportExternalSemaphore(D3D12Fence)\n");
hip.check(external.hipImportExternalSemaphore(&imported.semaphore, &sem_desc), "hipImportExternalSemaphore(D3D12Fence)");
handle_checkpoint("imported_semaphore");
if (persistent.stream) { imported.stream = persistent.stream; imported.owns_stream = false; }
else hip.check(hip.hipStreamCreateWithFlags(&imported.stream, hipStreamNonBlocking), "hipStreamCreateWithFlags");
handle_checkpoint("created_stream");
hip.check(hip.hipMalloc(&imported.errors, sizeof(uint32_t)), "hipMalloc(errors)");
const uint32_t zero = 0;
hip.check(hip.hipMemcpy(imported.errors, &zero, sizeof(zero), hipMemcpyHostToDevice), "hipMemcpy(error initialization)");
uint32_t* host_upload = nullptr;
D3D12_RANGE no_read{0, 0};
dx(upload->Map(0, &no_read, reinterpret_cast<void**>(&host_upload)), "Map(upload initialization)");
for (uint32_t i = 0; i < count; ++i) host_upload[i] = pattern(i, seed);
D3D12_RANGE written{0, static_cast<SIZE_T>(bytes)}; upload->Unmap(0, &written);
ComPtr<ID3D12CommandAllocator> allocator;
ComPtr<ID3D12GraphicsCommandList> first, second;
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(first.GetAddressOf())), "CreateCommandList(input)");
if (iteration) transition(first.Get(), texture.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COPY_DEST);
linear_copy.pResource = upload.Get();
first->CopyTextureRegion(&texture_copy, 0, 0, 0, &linear_copy, nullptr);
transition(first.Get(), texture.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_COPY_SOURCE);
linear_copy.pResource = shared.Get();
first->CopyTextureRegion(&linear_copy, 0, 0, 0, &texture_copy, nullptr);
transition(first.Get(), shared.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_COMMON);
dx(first->Close(), "Close(input)");
ID3D12CommandList* submit[] = {first.Get()}; queue->ExecuteCommandLists(1, submit);
const UINT64 ready = 2ull * iteration + 1, done = ready + 1;
dx(queue->Signal(shared_fence.Get(), ready), "Signal(D3D12 input ready)");
hipExternalSemaphoreWaitParams wait{}; wait.params.fence.value = ready;
hip.check(external.hipWaitExternalSemaphoresAsync(&imported.semaphore, &wait, 1, imported.stream), "hipWaitExternalSemaphoresAsync");
uint32_t n = count, mutable_seed = seed;
void* parameters[] = {&imported.mapped, &n, &mutable_seed, &imported.errors};
hip.check(hip.hipModuleLaunchKernel(kernel, (n + 127) / 128, 1, 1, 128, 1, 1, 0,
imported.stream, parameters, nullptr), "hipModuleLaunchKernel(shared VRAM)");
hipExternalSemaphoreSignalParams signal{}; signal.params.fence.value = done;
hip.check(external.hipSignalExternalSemaphoresAsync(&imported.semaphore, &signal, 1, imported.stream), "hipSignalExternalSemaphoresAsync");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(second.GetAddressOf())), "CreateCommandList(output)");
transition(second.Get(), shared.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_SOURCE);
transition(second.Get(), texture.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COPY_DEST);
second->CopyTextureRegion(&texture_copy, 0, 0, 0, &linear_copy, nullptr);
transition(second.Get(), texture.Get(), D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_COPY_SOURCE);
linear_copy.pResource = readback.Get();
second->CopyTextureRegion(&linear_copy, 0, 0, 0, &texture_copy, nullptr);
dx(second->Close(), "Close(output)");
dx(queue->Wait(shared_fence.Get(), done), "Wait(HIP output ready)");
submit[0] = second.Get(); queue->ExecuteCommandLists(1, submit);
dx(queue->Signal(complete.Get(), iteration + 1), "Signal(completion)");
dx(complete->SetEventOnCompletion(iteration + 1, event.handle), "SetEventOnCompletion");
if (WaitForSingleObject(event.handle, 30000) != WAIT_OBJECT_0) throw std::runtime_error("D3D12 fence timeout");
hip.check(hip.hipStreamSynchronize(imported.stream), "hipStreamSynchronize");
uint32_t errors = ~0u;
hip.check(hip.hipMemcpy(&errors, imported.errors, sizeof(errors), hipMemcpyDeviceToHost), "hipMemcpy(error verification)");
if (errors) throw std::runtime_error("HIP input mismatch count=" + std::to_string(errors));
uint32_t* host_result = nullptr;
D3D12_RANGE read{0, static_cast<SIZE_T>(bytes)};
dx(readback->Map(0, &read, reinterpret_cast<void**>(&host_result)), "Map(final verification)");
bool good = true;
for (uint32_t i = 0; i < count; ++i) if (host_result[i] != (pattern(i, seed) ^ 0x3579bdf1u)) { good = false; break; }
readback->Unmap(0, &no_read);
if (!good) throw std::runtime_error("D3D12 output mismatch");
// Check each release in the successful path; RAII handles all failures.
if (!persistent.stream) hip.check(hip.hipStreamDestroy(imported.stream), "hipStreamDestroy");
imported.stream = nullptr;
handle_checkpoint("released_stream_reference");
hip.check(hip.hipFree(imported.errors), "hipFree(errors)"); imported.errors = nullptr;
handle_checkpoint("freed_errors");
hip.check(hip.hipFree(imported.mapped), "hipFree(external mapping)"); imported.mapped = nullptr;
handle_checkpoint("freed_mapping");
hip.check(external.hipDestroyExternalMemory(imported.memory), "hipDestroyExternalMemory"); imported.memory = nullptr;
handle_checkpoint("destroyed_memory");
hip.check(external.hipDestroyExternalSemaphore(imported.semaphore), "hipDestroyExternalSemaphore"); imported.semaphore = nullptr;
handle_checkpoint("destroyed_semaphore");
shared.Reset();
if (!CloseHandle(memory_handle.handle)) throw std::runtime_error("CloseHandle(shared memory) failed");
memory_handle.handle = nullptr;
first.Reset(); second.Reset(); allocator.Reset();
DWORD cycle_handles = 0; GetProcessHandleCount(GetCurrentProcess(), &cycle_handles);
if (iteration == 0) warmed_handles = cycle_handles;
final_cycle_handles = cycle_handles;
std::printf("CLEANUP iteration=%u handles=%lu warmed_baseline=%lu\n", iteration, cycle_handles, warmed_handles);
std::printf("ROUNDTRIP iteration=%u verified=%u vram_transfer=1 cpu_staging_between_apis=0\n", iteration, count);
}
if (persistent.stream) {
hip.check(hip.hipStreamDestroy(persistent.stream), "hipStreamDestroy(persistent)");
persistent.stream = nullptr;
}
DWORD handles_after = 0; GetProcessHandleCount(GetCurrentProcess(), &handles_after);
std::printf("HANDLES before=%lu after=%lu\n", handles_before, handles_after);
if (final_cycle_handles > warmed_handles + 4) throw std::runtime_error("Handle count grew after first-cycle warmup");
std::printf("PASS INTEROP architecture=%s iterations=%u texture=R32_UINT external_fence=1\n", d4r::diag::HipApi::target_arch(), args.iterations);
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL INTEROP %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/d3d12_newer_commands.h
-46
View File
@@ -1,46 +0,0 @@
#pragma once
// Copyright (c) Microsoft Corporation.
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
// ABI subset of Microsoft's MIT-licensed DirectX-Headers, commit adbd6f3b.
// No pointers to CPU-owned nested structures exist in SetProgram's descriptor.
// The generic/raytracing variants use only the ProgramIdentifier field.
namespace d4r::win::commands {
struct NewWorkGraphDescription {
UINT64 program_identifier[4];
UINT flags;
D3D12_GPU_VIRTUAL_ADDRESS_RANGE backing_memory;
D3D12_GPU_VIRTUAL_ADDRESS_RANGE_AND_STRIDE root_arguments;
};
struct NewProgramDescription {
UINT type;
union {
UINT64 generic_program_identifier[4];
UINT64 raytracing_program_identifier[4];
NewWorkGraphDescription work_graph;
};
};
static_assert(sizeof(NewWorkGraphDescription) == 80 && sizeof(NewProgramDescription) == 88,
"Public Windows x64 SetProgram ABI");
inline constexpr GUID command_list8_iid = {0xee936ef9,0x599d,0x4d28,{0x93,0x8e,0x23,0xc4,0xad,0x05,0xce,0x51}};
inline constexpr GUID command_list9_iid = {0x34ed2808,0xffe6,0x4c2b,{0xb1,0x1a,0xca,0xbd,0x2b,0x0c,0x59,0xe1}};
inline constexpr GUID command_list10_iid = {0x7013c015,0xd161,0x4b63,{0xa0,0x8c,0x23,0x85,0x52,0xdd,0x8a,0xcc}};
}
tools/windows/d3d12_pixel_probe.cpp
-264
View File
@@ -1,264 +0,0 @@
// Independent reference: D3D12 typed SRV loads and typed UAV stores.
#include "pixel_conversion.h"
#include <dxgi1_6.h>
#include <d3dcompiler.h>
#include <array>
#include <cmath>
using namespace d4r::diag;
using namespace d4r::win;
namespace {
constexpr unsigned width = 259, height = 17, count = width * height;
struct Case { DXGI_FORMAT resource, view; unsigned plane; const char* name; };
const Case cases[] = {
{DXGI_FORMAT_R16G16B16A16_FLOAT, DXGI_FORMAT_R16G16B16A16_FLOAT, 0, "rgba16f"},
{DXGI_FORMAT_R32G32B32A32_FLOAT, DXGI_FORMAT_R32G32B32A32_FLOAT, 0, "rgba32f"},
{DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, 0, "rgba8"},
{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_B8G8R8A8_UNORM, 0, "bgra8"},
{DXGI_FORMAT_R11G11B10_FLOAT, DXGI_FORMAT_R11G11B10_FLOAT, 0, "r11g11b10f"},
{DXGI_FORMAT_R10G10B10A2_UNORM, DXGI_FORMAT_R10G10B10A2_UNORM, 0, "rgb10a2"},
{DXGI_FORMAT_R16G16_FLOAT, DXGI_FORMAT_R16G16_FLOAT, 2, "rg16f"},
{DXGI_FORMAT_R32G32_FLOAT, DXGI_FORMAT_R32G32_FLOAT, 2, "rg32f"},
{DXGI_FORMAT_R16G16_SNORM, DXGI_FORMAT_R16G16_SNORM, 2, "rg16snorm"},
{DXGI_FORMAT_R16G16_UNORM, DXGI_FORMAT_R16G16_UNORM, 2, "rg16unorm"},
{DXGI_FORMAT_R16G16B16A16_FLOAT, DXGI_FORMAT_R16G16B16A16_FLOAT, 2, "motion-rgba16f"},
{DXGI_FORMAT_R32G32B32A32_FLOAT, DXGI_FORMAT_R32G32B32A32_FLOAT, 2, "motion-rgba32f"},
{DXGI_FORMAT_R32_FLOAT, DXGI_FORMAT_R32_FLOAT, 1, "depth-r32f"},
{DXGI_FORMAT_R16_TYPELESS, DXGI_FORMAT_R16_UNORM, 1, "depth-r16unorm"},
{DXGI_FORMAT_R24G8_TYPELESS, DXGI_FORMAT_R24_UNORM_X8_TYPELESS, 1, "depth24"},
{DXGI_FORMAT_R32G8X24_TYPELESS, DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS, 1, "depth32-stencil8-plane0"},
{DXGI_FORMAT_R16_TYPELESS, DXGI_FORMAT_R16_FLOAT, 4, "exposure-r16f"},
{DXGI_FORMAT_R32_FLOAT, DXGI_FORMAT_R32_FLOAT, 4, "exposure-r32f"},
{DXGI_FORMAT_R16G16B16A16_FLOAT, DXGI_FORMAT_R16G16B16A16_FLOAT, 4, "exposure-rgba16f"},
{DXGI_FORMAT_R32G32B32A32_FLOAT, DXGI_FORMAT_R32G32B32A32_FLOAT, 4, "exposure-rgba32f"},
};
struct List {
ComPtr<ID3D12CommandAllocator> alloc;
ComPtr<ID3D12GraphicsCommandList> list;
explicit List(ID3D12Device* device) {
dx(device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(alloc.GetAddressOf())), "Pixel allocator");
dx(device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, alloc.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Pixel command list");
}
void submit(ID3D12CommandQueue* queue, SharedTimeline& timeline) {
dx(list->Close(), "Close pixel list"); ID3D12CommandList* lists[] = {list.Get()};
queue->ExecuteCommandLists(1, lists); timeline.drain(queue);
}
};
ComPtr<ID3D12Resource> make_texture(ID3D12Device* device, DXGI_FORMAT fmt, bool uav = false) {
D3D12_RESOURCE_DESC desc{}; desc.Dimension = D3D12_RESOURCE_DIMENSION_TEXTURE2D;
desc.Width = width; desc.Height = height; desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
desc.Format = fmt; desc.Flags = uav ? D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS : D3D12_RESOURCE_FLAG_NONE;
if (fmt == DXGI_FORMAT_R32G8X24_TYPELESS) desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_DEPTH_STENCIL;
D3D12_HEAP_PROPERTIES heap{}; heap.Type = D3D12_HEAP_TYPE_DEFAULT; heap.CreationNodeMask = heap.VisibleNodeMask = 1;
ComPtr<ID3D12Resource> result;
dx(device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_COMMON,
nullptr, IID_PPV_ARGS(result.GetAddressOf())), "Pixel texture"); return result;
}
ComPtr<ID3D12Resource> uav_buffer(ID3D12Device* device) {
D3D12_RESOURCE_DESC desc{}; desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; desc.Width = count * 16;
desc.Height = desc.DepthOrArraySize = desc.MipLevels = desc.SampleDesc.Count = 1;
desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; desc.Flags = D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS;
D3D12_HEAP_PROPERTIES heap{}; heap.Type = D3D12_HEAP_TYPE_DEFAULT; heap.CreationNodeMask = heap.VisibleNodeMask = 1;
ComPtr<ID3D12Resource> result;
dx(device->CreateCommittedResource(&heap, D3D12_HEAP_FLAG_NONE, &desc, D3D12_RESOURCE_STATE_UNORDERED_ACCESS,
nullptr, IID_PPV_ARGS(result.GetAddressOf())), "Pixel reference buffer"); return result;
}
ResourceAccess shader_access() {
ResourceAccess result; result.enhanced = true; result.layout = D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_SHADER_RESOURCE;
result.access = D3D12_BARRIER_ACCESS_SHADER_RESOURCE; return result;
}
void native_layout(ID3D12GraphicsCommandList* list, ID3D12Resource* texture, D3D12_BARRIER_LAYOUT before, D3D12_BARRIER_LAYOUT after,
D3D12_BARRIER_ACCESS accessBefore, D3D12_BARRIER_ACCESS accessAfter) {
ComPtr<ID3D12GraphicsCommandList7> newer; dx(list->QueryInterface(IID_PPV_ARGS(newer.GetAddressOf())), "Probe enhanced list");
D3D12_TEXTURE_BARRIER barrier{}; barrier.pResource = texture; barrier.LayoutBefore = before; barrier.LayoutAfter = after;
barrier.AccessBefore = accessBefore; barrier.AccessAfter = accessAfter; barrier.SyncBefore = barrier.SyncAfter = D3D12_BARRIER_SYNC_ALL;
D3D12_BARRIER_GROUP group{}; group.Type = D3D12_BARRIER_TYPE_TEXTURE; group.NumBarriers = 1; group.pTextureBarriers = &barrier; newer->Barrier(1,&group);
}
void fixture(uint8_t* row, unsigned y, PixelSpec spec) {
for (unsigned x = 0; x < width; ++x) {
unsigned i = x + y * width; auto* p = row + x * spec.bytes;
const uint16_t half[] = {0, 1, 0x3ff, 0x400, 0x1400, 0x3555, 0x3800, 0x3bff, 0x3c00, 0x3c01, 0x5c07, 0x7bff, 0x8001, 0xbc00};
switch (spec.storage) {
case d4r::pixel::rgba16f: case d4r::pixel::rg16f: case d4r::pixel::r16f:
for (unsigned c = 0; c < spec.bytes / 2; ++c) reinterpret_cast<uint16_t*>(p)[c] = half[(i + c * 3) % std::size(half)]; break;
case d4r::pixel::rgba32f: case d4r::pixel::rg32f: case d4r::pixel::r32f:
for (unsigned c = 0; c < spec.bytes / 4; ++c) reinterpret_cast<float*>(p)[c] = float(int((i * 23 + c * 13) % 4097) - 2048) / 4096.f; break;
case d4r::pixel::r11g11b10f:
*reinterpret_cast<uint32_t*>(p) = (i % 1984) | (((i * 13) % 1984) << 11) | (((i * 7) % 992) << 22); break;
case d4r::pixel::rgb10a2:
*reinterpret_cast<uint32_t*>(p) = (i % 1024) | (((i * 13) % 1024) << 10) | (((i * 7) % 1024) << 20) | ((i % 4) << 30); break;
case d4r::pixel::rgba8: case d4r::pixel::bgra8:
for (unsigned c = 0; c < 4; ++c) p[c] = uint8_t(i * (2 * c + 1) + c * 43); break;
case d4r::pixel::rg16snorm: case d4r::pixel::rg16unorm: case d4r::pixel::r16unorm:
for (unsigned c = 0; c < spec.bytes / 2; ++c) reinterpret_cast<uint16_t*>(p)[c] = uint16_t(i < 4 ? i * 32767 : i * 193 + c * 37); break;
case d4r::pixel::depth24: *reinterpret_cast<uint32_t*>(p) = (i * 3791) | ((i % 256) << 24); break;
default: throw std::runtime_error("Missing diagnostic fixture");
}
}
}
uint16_t reference_half(uint32_t bits) {
// Independent nearest-value search over the binary16 number line. HLSL
// f32tof16 may truncate; it is not the CUDA RNE conversion being tested.
static const std::vector<double> values = [] {
std::vector<double> v(31744);
for (unsigned h = 0; h < v.size(); ++h) v[h] = h < 1024 ? std::ldexp(double(h), -24) :
std::ldexp(1. + double(h & 1023) / 1024., int(h >> 10) - 15);
return v;
}();
float value = 0; std::memcpy(&value, &bits, 4);
double magnitude = std::abs(double(value));
if (!std::isfinite(magnitude) || magnitude > values.back()) throw std::runtime_error("Unexpected non-finite decode reference");
auto upper = std::lower_bound(values.begin(), values.end(), magnitude);
unsigned h = unsigned(upper - values.begin());
if (h && (magnitude - values[h - 1] < values[h] - magnitude ||
(magnitude - values[h - 1] == values[h] - magnitude && (h & 1)))) --h;
return uint16_t(h | ((bits >> 16) & 0x8000));
}
struct Shaders {
ComPtr<ID3D12RootSignature> root;
ComPtr<ID3D12PipelineState> decode, encode;
explicit Shaders(ID3D12Device* device) {
wchar_t system[32768]{}; if (!GetSystemDirectoryW(system, 32768)) throw std::runtime_error("System path");
Library compiler(std::filesystem::path(system) / L"d3dcompiler_47.dll");
auto compile = compiler.symbol<decltype(&D3DCompile)>("D3DCompile");
const char* source = R"(
Texture2D<float4> Input : register(t0);
StructuredBuffer<uint4> Canonical : register(t1);
RWStructuredBuffer<uint4> Output : register(u0);
RWTexture2D<float4> Expected : register(u1);
cbuffer Constants : register(b0) { uint Width, Height, Plane; };
[numthreads(64,1,1)] void read(uint3 t:SV_DispatchThreadID) {
if (t.x >= Width*Height) return;
float4 v = Input.Load(int3(t.x % Width,t.x / Width,0));
Output[t.x] = asuint(v);
}
[numthreads(64,1,1)] void write(uint3 t:SV_DispatchThreadID) {
if (t.x < Width*Height) Expected[uint2(t.x % Width,t.x / Width)] = f16tof32(Canonical[t.x]);
})";
D3D12_DESCRIPTOR_RANGE range[3]{};
for (unsigned i = 0; i < 3; ++i) { range[i].RangeType = i == 2 ? D3D12_DESCRIPTOR_RANGE_TYPE_UAV : D3D12_DESCRIPTOR_RANGE_TYPE_SRV; range[i].NumDescriptors = 1; range[i].BaseShaderRegister = i == 2 ? 1 : i; }
D3D12_ROOT_PARAMETER parameters[5]{};
for (unsigned i = 0; i < 3; ++i) { parameters[i].ParameterType = D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE; parameters[i].DescriptorTable.NumDescriptorRanges = 1; parameters[i].DescriptorTable.pDescriptorRanges = &range[i]; }
parameters[3].ParameterType = D3D12_ROOT_PARAMETER_TYPE_UAV;
parameters[4].ParameterType = D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS; parameters[4].Constants.Num32BitValues = 3;
D3D12_ROOT_SIGNATURE_DESC sd{}; sd.NumParameters = 5; sd.pParameters = parameters;
ComPtr<ID3DBlob> blob, errors; dx(D3D12SerializeRootSignature(&sd, D3D_ROOT_SIGNATURE_VERSION_1, blob.GetAddressOf(), errors.GetAddressOf()), "Pixel root serialize");
dx(device->CreateRootSignature(0, blob->GetBufferPointer(), blob->GetBufferSize(), IID_PPV_ARGS(root.GetAddressOf())), "Pixel root");
for (unsigned i = 0; i < 2; ++i) {
if (FAILED(compile(source, std::strlen(source), nullptr, nullptr, nullptr, i ? "write" : "read", "cs_5_1", D3DCOMPILE_OPTIMIZATION_LEVEL3,
0, blob.ReleaseAndGetAddressOf(), errors.ReleaseAndGetAddressOf()))) throw std::runtime_error(errors ? std::string(static_cast<char*>(errors->GetBufferPointer())) : "Pixel HLSL compilation");
D3D12_COMPUTE_PIPELINE_STATE_DESC pd{}; pd.pRootSignature = root.Get(); pd.CS = {blob->GetBufferPointer(), blob->GetBufferSize()};
dx(device->CreateComputePipelineState(&pd, IID_PPV_ARGS((i ? encode : decode).GetAddressOf())), "Pixel PSO");
}
}
};
std::vector<uint32_t> read_reference(ID3D12Device* device, ID3D12CommandQueue* queue, SharedTimeline& timeline,
Shaders& shader, ID3D12DescriptorHeap* heap, D3D12_GPU_DESCRIPTOR_HANDLE srv, ID3D12Resource* reference, unsigned plane) {
auto readback = make_buffer(device, count * 16, D3D12_HEAP_TYPE_READBACK, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST);
List list(device); list.list->SetPipelineState(shader.decode.Get()); list.list->SetComputeRootSignature(shader.root.Get());
list.list->SetDescriptorHeaps(1, &heap); list.list->SetComputeRootDescriptorTable(0, srv);
list.list->SetComputeRootUnorderedAccessView(3, reference->GetGPUVirtualAddress());
const unsigned constants[] = {width, height, plane}; list.list->SetComputeRoot32BitConstants(4, 3, constants, 0);
list.list->Dispatch((count + 63) / 64, 1, 1);
transition(list.list.Get(), reference, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COPY_SOURCE);
list.list->CopyResource(readback.Get(), reference);
transition(list.list.Get(), reference, D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_UNORDERED_ACCESS);
list.submit(queue, timeline);
void* data = nullptr; D3D12_RANGE range{0, count * 16}; dx(readback->Map(0, &range, &data), "Pixel readback");
std::vector<uint32_t> result(count * 4); std::memcpy(result.data(), data, count * 16); D3D12_RANGE noWrite{}; readback->Unmap(0, &noWrite); return result;
}
}
int main(int argc, char** argv) {
start();
try {
Args args(argc, argv); HipApi hip(args.hip_root); hipDeviceProp_t props{}; hip.select_architecture(args.device, props);
ExternalApi external{hip}; PixelProgram program(hip, wide(args.module));
{
PixelAllocation diagnostic(hip,3,1,8);
const uint16_t bad[] = {0x7e00,0x7c00,0xfc00,0x3c00,0,1,0x3ff,0x7bff,0x8000,0x8001,0xbc00,0x400};
hip.check(hip.hipMemcpy(diagnostic.data,bad,sizeof(bad),hipMemcpyHostToDevice), "Upload nonfinite validation fixture");
const auto counts = program.validate(diagnostic.data,diagnostic.pitch,3,1);
if (counts[0] != 1 || counts[1] != 2) throw std::runtime_error("GPU NaN/Inf validation counters mismatch");
std::printf("PASS OUTPUT_VALIDATION_FIXTURE nan=1 inf=2 finite=9\n");
}
ComPtr<IDXGIFactory4> factory; dx(CreateDXGIFactory1(IID_PPV_ARGS(factory.GetAddressOf())), "Pixel DXGI");
ComPtr<IDXGIAdapter1> adapter;
for (UINT i = 0; ; ++i) { dx(factory->EnumAdapters1(i, adapter.ReleaseAndGetAddressOf()), "Pixel adapter"); DXGI_ADAPTER_DESC1 d{}; dx(adapter->GetDesc1(&d), "Pixel adapter desc"); if (!std::memcmp(&d.AdapterLuid, props.luid, sizeof(LUID))) break; }
ComPtr<ID3D12Device> device; dx(D3D12CreateDevice(adapter.Get(), D3D_FEATURE_LEVEL_12_0, IID_PPV_ARGS(device.GetAddressOf())), "Pixel D3D12");
const bool enhanced = args.interop_mode == "enhanced";
if (enhanced) { D3D12_FEATURE_DATA_D3D12_OPTIONS12 support{}; dx(device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS12, &support, sizeof(support)), "Enhanced barrier support"); if (!support.EnhancedBarriersSupported) throw std::runtime_error("Enhanced barriers unsupported"); }
ComPtr<ID3D12CommandQueue> queue; D3D12_COMMAND_QUEUE_DESC qd{}; qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; dx(device->CreateCommandQueue(&qd, IID_PPV_ARGS(queue.GetAddressOf())), "Pixel queue");
SharedTimeline timeline(external, device.Get()); Shaders shader(device.Get()); auto reference = uav_buffer(device.Get());
unsigned verified = 0;
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) for (const auto& item : cases) {
PixelSpec spec = pixel_spec(item.plane, item.resource); if (!pixel_supported(item.plane, spec.storage)) throw std::runtime_error("Unsupported fixture plane");
auto input = make_texture(device.Get(), item.resource); SharedPlane shared(external, device.Get(), resource_desc(input.Get()));
std::printf("PIXEL_FOOTPRINT format=%s plane0_format=%u pitch=%u storage_bytes=%u\n", item.name, shared.footprint.Footprint.Format, shared.footprint.Footprint.RowPitch, spec.bytes);
auto upload = make_buffer(device.Get(), shared.bytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
void* data = nullptr; D3D12_RANGE noRead{}; dx(upload->Map(0, &noRead, &data), "Pixel fixture upload");
std::memset(data, 0x5a, size_t(shared.bytes)); for (unsigned y = 0; y < height; ++y) fixture(static_cast<uint8_t*>(data) + y * shared.footprint.Footprint.RowPitch, y, spec); upload->Unmap(0, nullptr);
{ List list(device.Get()); D3D12_TEXTURE_COPY_LOCATION src{}, dst{}; src.pResource = upload.Get(); src.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; src.PlacedFootprint = shared.footprint; dst.pResource = input.Get(); dst.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
transition(list.list.Get(), input.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_DEST); list.list->CopyTextureRegion(&dst, 0, 0, 0, &src, nullptr);
transition(list.list.Get(), input.Get(), D3D12_RESOURCE_STATE_COPY_DEST, enhanced ? D3D12_RESOURCE_STATE_COMMON : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
if (enhanced) native_layout(list.list.Get(),input.Get(),D3D12_BARRIER_LAYOUT_COMMON,D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_SHADER_RESOURCE,D3D12_BARRIER_ACCESS_COMMON,D3D12_BARRIER_ACCESS_SHADER_RESOURCE);
shared.copy_input(list.list.Get(), input.Get(), enhanced ? shader_access() : ResourceAccess(D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE)); list.submit(queue.Get(), timeline); }
D3D12_DESCRIPTOR_HEAP_DESC hd{}; hd.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV; hd.NumDescriptors = 4; hd.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE;
ComPtr<ID3D12DescriptorHeap> heap; dx(device->CreateDescriptorHeap(&hd, IID_PPV_ARGS(heap.GetAddressOf())), "Pixel descriptors");
D3D12_CPU_DESCRIPTOR_HANDLE cpu{}, initial{}; D3D12_GPU_DESCRIPTOR_HANDLE gpu{};
#if defined(__MINGW32__)
heap->GetCPUDescriptorHandleForHeapStart(&cpu); heap->GetGPUDescriptorHandleForHeapStart(&gpu);
#else
cpu = heap->GetCPUDescriptorHandleForHeapStart(); gpu = heap->GetGPUDescriptorHandleForHeapStart();
#endif
initial = cpu; unsigned stride = device->GetDescriptorHandleIncrementSize(hd.Type);
D3D12_SHADER_RESOURCE_VIEW_DESC srv{}; srv.Format = item.view; srv.ViewDimension = D3D12_SRV_DIMENSION_TEXTURE2D; srv.Shader4ComponentMapping = D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING; srv.Texture2D.MipLevels = 1; device->CreateShaderResourceView(input.Get(), &srv, cpu);
auto expected = read_reference(device.Get(), queue.Get(), timeline, shader, heap.Get(), gpu, reference.Get(), item.plane);
unsigned channels = item.plane == 2 ? 2 : item.plane == 0 ? 4 : 1; unsigned element = item.plane == 0 || item.plane == 2 ? 2 : 4;
PixelAllocation canonical(hip, width, height, channels * element);
std::vector<uint8_t> guard(size_t(canonical.pitch * height), 0xcd);
hip.check(hip.hipMemcpy(canonical.data, guard.data(), guard.size(), hipMemcpyHostToDevice), "Diagnostic pitch guard initialization");
program.convert(false, shared.mapped, shared.footprint.Footprint.RowPitch, canonical.data, canonical.pitch, width, height, spec.storage, item.plane);
std::vector<uint8_t> actual(size_t(canonical.pitch * height)); hip.check(hip.hipMemcpy(actual.data(), canonical.data, actual.size(), hipMemcpyDeviceToHost), "Diagnostic canonical readback");
unsigned differs = 0; double maxAbs = 0;
for (unsigned y = 0; y < height; ++y) for (uint64_t offset = width * channels * element; offset < canonical.pitch; ++offset)
if (actual[size_t(y * canonical.pitch + offset)] != 0xcd) throw std::runtime_error("GPU decode wrote past the row/pixel range");
for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < width; ++x) for (unsigned c = 0; c < channels; ++c) {
unsigned i = (y * width + x) * 4 + c; const auto* p = actual.data() + y * canonical.pitch + (x * channels + c) * element; uint32_t got = 0; std::memcpy(&got, p, element);
if (element == 2) expected[i] = reference_half(expected[i]);
bool mismatch = got != expected[i];
if (element == 4) { float a = 0, b = 0; std::memcpy(&a, &got, 4); std::memcpy(&b, &expected[i], 4); double error = std::abs(double(a) - b); maxAbs = std::max(maxAbs, error); mismatch = !std::isfinite(a) || !std::isfinite(b) || error > 1.2e-7 * std::max(1., std::abs(double(b))); }
if (mismatch && differs++ < 5) std::fprintf(stderr, "PIXEL_DIFFERENT format=%s pixel=%u channel=%u expected=%08x actual=%08x\n", item.name, y * width + x, c, expected[i], got);
}
if (differs) throw std::runtime_error(std::string("D3D12 typed decode mismatch: ") + item.name + " count=" + std::to_string(differs));
if (item.plane == 0) {
// BGRA uses the identical UNORM converter through an RGBA UAV;
// the consumer SRV independently tests the channel permutation.
DXGI_FORMAT encodedFormat = spec.storage == d4r::pixel::bgra8 ? DXGI_FORMAT_R8G8B8A8_UNORM : item.view;
auto encoded = make_texture(device.Get(), encodedFormat, true); auto actualTexture = make_texture(device.Get(), item.resource);
auto canonicalUpload = make_buffer(device.Get(), count * 16, D3D12_HEAP_TYPE_UPLOAD, D3D12_HEAP_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ);
dx(canonicalUpload->Map(0, &noRead, &data), "Encode fixture upload");
auto* values = static_cast<uint32_t*>(data); std::vector<uint16_t> halves(count * 4);
// Cover finite half values, rounding ties, negatives and overflow.
for (unsigned i = 0; i < count; ++i) for (unsigned c = 0; c < 4; ++c) { uint16_t v = uint16_t(((i * 37 + c * 7919) % 31744) | (i % 11 == 0 ? 0x8000 : 0)); values[i * 4 + c] = v; halves[i * 4 + c] = v; }
canonicalUpload->Unmap(0, nullptr);
std::vector<uint8_t> padded(size_t(canonical.pitch * height), 0x5a); for (unsigned y = 0; y < height; ++y) std::memcpy(padded.data() + y * canonical.pitch, halves.data() + y * width * 4, width * 8);
hip.check(hip.hipMemcpy(canonical.data, padded.data(), padded.size(), hipMemcpyHostToDevice), "Diagnostic encode input");
program.convert(true, canonical.data, canonical.pitch, shared.mapped, shared.footprint.Footprint.RowPitch, width, height, spec.storage, 3);
cpu.ptr = initial.ptr + stride; srv = {}; srv.Shader4ComponentMapping = D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING; srv.ViewDimension = D3D12_SRV_DIMENSION_BUFFER; srv.Buffer.NumElements = count; srv.Buffer.StructureByteStride = 16; device->CreateShaderResourceView(canonicalUpload.Get(), &srv, cpu);
cpu.ptr += stride; D3D12_UNORDERED_ACCESS_VIEW_DESC uv{}; uv.Format = encodedFormat; uv.ViewDimension = D3D12_UAV_DIMENSION_TEXTURE2D; device->CreateUnorderedAccessView(encoded.Get(), nullptr, &uv, cpu);
{ List list(device.Get()); transition(list.list.Get(), encoded.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); list.list->SetPipelineState(shader.encode.Get()); list.list->SetComputeRootSignature(shader.root.Get()); ID3D12DescriptorHeap* heaps[] = {heap.Get()}; list.list->SetDescriptorHeaps(1, heaps); auto handle = gpu; handle.ptr += stride; list.list->SetComputeRootDescriptorTable(1, handle); handle.ptr += stride; list.list->SetComputeRootDescriptorTable(2, handle); unsigned constants[] = {width,height,0}; list.list->SetComputeRoot32BitConstants(4,3,constants,0); list.list->Dispatch((count+63)/64,1,1); transition(list.list.Get(), encoded.Get(), D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
ResourceAccess state; state.enhanced = enhanced; shared.copy_output(list.list.Get(), actualTexture.Get(), state);
if (enhanced) native_layout(list.list.Get(),actualTexture.Get(),D3D12_BARRIER_LAYOUT_COMMON,D3D12_BARRIER_LAYOUT_DIRECT_QUEUE_SHADER_RESOURCE,D3D12_BARRIER_ACCESS_COMMON,D3D12_BARRIER_ACCESS_SHADER_RESOURCE);
else transition(list.list.Get(), actualTexture.Get(), D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE);
list.submit(queue.Get(),timeline); }
srv = {}; srv.Format = item.view; srv.ViewDimension = D3D12_SRV_DIMENSION_TEXTURE2D; srv.Shader4ComponentMapping = D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING; srv.Texture2D.MipLevels = 1; device->CreateShaderResourceView(actualTexture.Get(), &srv, initial);
auto encodedActual = read_reference(device.Get(), queue.Get(), timeline, shader, heap.Get(), gpu, reference.Get(), 4);
srv.Format = encodedFormat; device->CreateShaderResourceView(encoded.Get(), &srv, initial);
auto encodedExpected = read_reference(device.Get(), queue.Get(), timeline, shader, heap.Get(), gpu, reference.Get(), 4);
for (unsigned i = 0; i < count * 4; ++i) if (encodedActual[i] != encodedExpected[i]) { if (differs++ < 5) std::fprintf(stderr, "ENCODE_DIFFERENT format=%s pixel=%u channel=%u expected=%08x actual=%08x\n", item.name, i/4,i%4,encodedExpected[i],encodedActual[i]); }
if (differs) throw std::runtime_error(std::string("D3D12 typed encode mismatch: ") + item.name + " count=" + std::to_string(differs));
}
++verified; std::printf("PASS PIXEL_FORMAT name=%s plane=%u pixels=%u decode_max_abs=%.9g encode=%u\n", item.name,item.plane,count,maxAbs,item.plane==0);
}
std::printf("PASS D3D12_GPU_FORMATS architecture=%s cases=%u iterations=%u width=%u height=%u enhanced=%u typed_srv_reference=1 typed_uav_reference=1\n", d4r::diag::HipApi::target_arch(), verified,args.iterations,width,height,enhanced); return 0;
} catch (const std::exception& error) { std::fprintf(stderr,"FAIL D3D12_GPU_FORMATS %s\n",error.what()); loaded_modules(); return 1; }
}
tools/windows/d3d12_retirement.h
-46
View File
@@ -1,46 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include <condition_variable>
#include <deque>
#include <memory>
#include <mutex>
#include <thread>
namespace d4r::win {
// GPU-completion ownership, independent of command-list Reset and queue COM
// lifetime. The command backend pins its DLL. One process-lifetime thread
// releases completed batches outside its lock, including the final batch;
// queue -> recording -> feature -> queue cannot become a reference cycle.
class GpuRetirement {
struct Batch { ComPtr<ID3D12Fence> fence; uint64_t value; std::shared_ptr<void> owner; };
std::mutex mutex_;
std::condition_variable changed_;
std::deque<Batch> pending_;
GpuRetirement() {
std::thread([this] {
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
for (;;) {
Batch batch;
{ std::unique_lock<std::mutex> lock(mutex_);
changed_.wait(lock, [&] { return !pending_.empty(); });
batch = std::move(pending_.front()); pending_.pop_front(); }
if (batch.fence->GetCompletedValue() < batch.value) {
if (!event.value || FAILED(batch.fence->SetEventOnCompletion(batch.value, event.value)) ||
WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) {
std::fprintf(stderr, "D4R_RETIREMENT_FAILURE GPU completion timeout; retaining resources\n");
// A hung GPU must not make us free live command memory.
auto* retained = new Batch(std::move(batch)); (void)retained;
}
}
batch.owner.reset();
}
}).detach();
}
public:
static GpuRetirement& instance() { static auto* service = new GpuRetirement; return *service; }
void submit(ID3D12Fence* fence, uint64_t value, std::shared_ptr<void> owner) {
{ std::lock_guard<std::mutex> lock(mutex_); pending_.push_back({fence, value, std::move(owner)}); }
changed_.notify_one();
}
};
}
tools/windows/d4r_nvngx_loader.cpp
-176
View File
@@ -1,176 +0,0 @@
// Drag-in releases ship one d4r folder for every supported GPU. This small
// _nvngx.dll is what OptiScaler loads: on first use it reads the real HIP
// architecture of the game's adapter (matched by LUID) and loads that
// target's shim, d4r\_nvngx_<arch>.dll, from the same folder. The per-target
// shim still enforces its own compiled target, so a mismatch cannot run.
// Keeping every file in one folder preserves NGX's search for
// nvngx_dlss.dll beside the calling module.
#include "d3d12_external.h"
#include "ngx_public_abi.h"
#include <dxgi.h>
#include <fstream>
#include <mutex>
namespace {
constexpr unsigned failure = 0xBAD00002, unsupported = 4;
std::mutex selectMutex;
HMODULE shim = nullptr;
bool selectionFailed = false;
std::filesystem::path module_directory() {
HMODULE module = nullptr;
GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
reinterpret_cast<LPCWSTR>(&module_directory), &module);
wchar_t path[32768]{}; GetModuleFileNameW(module, path, 32768);
return std::filesystem::path(path).parent_path();
}
// The shim never ran, so the loader writes the player's log itself.
// Fresh at each launch, then appended: OptiScaler can query several adapters.
void report(const std::string& message) {
static bool fresh = true;
std::ofstream(module_directory() / L"d4r_nvngx.log", fresh ? std::ios::trunc : std::ios::app)
<< "d4r: DLSS disabled: " << message << "\n";
fresh = false;
std::fprintf(stderr, "D4R_WINDOWS_FAILURE %s\n", message.c_str());
}
std::string supported_list(const std::filesystem::path& dir) {
std::string list;
for (const auto& target : d4r::diag::gpu_targets)
if (std::filesystem::exists(dir / (L"_nvngx_" + d4r::diag::wide(target.architecture) + L".dll")))
list += (list.empty() ? "" : ", ") + std::string(target.architecture);
return list.empty() ? "none (the d4r folder is incomplete; extract the ZIP again)" : list;
}
// Loads the matching shim once. A null LUID (calls made before NGX init)
// accepts the only HIP device with a shim in this folder.
HMODULE select(const LUID* luid) {
std::lock_guard<std::mutex> lock(selectMutex);
if (shim || selectionFailed) return shim;
const auto dir = module_directory();
try {
// Spoofed architectures invalidate the target guarantee; see hip_api.h.
for (const char* name : {"HSA_OVERRIDE_GFX_VERSION", "HSA_OVERRIDE_GFX_VERSION_0"})
if (std::getenv(name)) throw std::runtime_error(std::string("unset ") + name + "; d4r does not support architecture overrides");
d4r::diag::HipApi hip((dir / L"hip").string());
if (hip.hipInit(0) != hipSuccess) throw std::runtime_error("the AMD HIP runtime found no usable GPU; update the AMD graphics driver");
int count = 0; (void)hip.hipGetDeviceCount(&count);
std::string found, detected;
for (int i = 0; i < count; ++i) {
hipDeviceProp_t props{};
if (hip.hipGetDeviceProperties(&props, i) != hipSuccess) continue;
const auto arch = d4r::diag::gpu_architecture(props.gcnArchName);
detected += (detected.empty() ? "" : ", ") + arch + " (" + props.name + ")";
if (luid && std::memcmp(props.luid, luid, sizeof(*luid))) continue;
if (!luid && !std::filesystem::exists(dir / (L"_nvngx_" + d4r::diag::wide(arch) + L".dll"))) continue;
if (!found.empty() && found != arch) throw std::runtime_error("several GPUs with different architectures; start the game once DLSS selects one");
found = arch;
}
if (found.empty()) throw std::runtime_error("no supported AMD GPU for this game's adapter. Detected: " +
(detected.empty() ? std::string("none") : detected) + ". This release supports: " + supported_list(dir) + ".");
const auto path = dir / (L"_nvngx_" + d4r::diag::wide(found) + L".dll");
if (!std::filesystem::exists(path))
throw std::runtime_error("this release has no build for your GPU (" + found + "). It supports: " + supported_list(dir) + ".");
shim = LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
if (!shim) {
selectionFailed = true; // A broken install will not fix itself during this session.
throw std::runtime_error("cannot load " + d4r::diag::utf8(path.c_str()) + " (Win32 error " + std::to_string(GetLastError()) + ")");
}
} catch (const std::exception& error) {
// An adapter without a matching GPU (an iGPU queried first) must not
// prevent selecting the discrete GPU on a later call.
report(error.what());
}
return shim;
}
template <typename Fn> Fn resolve(const char* name, const LUID* luid = nullptr) {
HMODULE module = select(luid);
return module ? reinterpret_cast<Fn>(reinterpret_cast<void*>(GetProcAddress(module, name))) : nullptr;
}
LUID device_luid(ID3D12Device* device) { return d4r::win::adapter_luid(device); }
}
#define API extern "C" __declspec(dllexport)
// Discovery marker for OptiScaler: no GPU or runtime initialization.
API unsigned d4r_WindowsBackendVersion() { return 1; }
API unsigned NVSDK_NGX_D3D12_Init_Ext(unsigned long long app, const wchar_t* data, ID3D12Device* device, unsigned sdk, const void* info) {
if (!device) return 0xBAD00005;
const LUID luid = device_luid(device);
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_Init_Ext)>("NVSDK_NGX_D3D12_Init_Ext", &luid);
return fn ? fn(app, data, device, sdk, info) : failure;
}
API unsigned NVSDK_NGX_D3D12_Init(unsigned long long app, const wchar_t* data, ID3D12Device* device, const void* info, unsigned sdk) {
if (!device) return 0xBAD00005;
const LUID luid = device_luid(device);
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_Init)>("NVSDK_NGX_D3D12_Init", &luid);
return fn ? fn(app, data, device, info, sdk) : failure;
}
API unsigned NVSDK_NGX_D3D12_Init_ProjectID(const char* project, int engine, const char* version, const wchar_t* data, ID3D12Device* device, unsigned sdk, const void* info) {
if (!device) return 0xBAD00005;
const LUID luid = device_luid(device);
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_Init_ProjectID)>("NVSDK_NGX_D3D12_Init_ProjectID", &luid);
return fn ? fn(project, engine, version, data, device, sdk, info) : failure;
}
API unsigned NVSDK_NGX_D3D12_GetFeatureRequirements(IDXGIAdapter* adapter, const d4r::ngx::FeatureDiscoveryInfo* info,
d4r::ngx::FeatureRequirement* requirements) {
if (!adapter || !info || !requirements) return 0xBAD00005;
DXGI_ADAPTER_DESC desc{};
if (FAILED(adapter->GetDesc(&desc))) return failure;
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_GetFeatureRequirements)>("NVSDK_NGX_D3D12_GetFeatureRequirements", &desc.AdapterLuid);
if (fn) return fn(adapter, info, requirements);
// Same answer as the shim for an adapter without a matching HIP device.
*requirements = {}; requirements->FeatureSupported = unsupported;
std::strcpy(requirements->MinOSVersion, "10.0.22000.0");
return 1;
}
API unsigned NVSDK_NGX_D3D12_AllocateParameters(void** p) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_AllocateParameters)>("NVSDK_NGX_D3D12_AllocateParameters");
return fn ? fn(p) : failure;
}
API unsigned NVSDK_NGX_D3D12_GetCapabilityParameters(void** p) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_GetCapabilityParameters)>("NVSDK_NGX_D3D12_GetCapabilityParameters");
return fn ? fn(p) : failure;
}
API unsigned NVSDK_NGX_D3D12_GetParameters(void** p) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_GetParameters)>("NVSDK_NGX_D3D12_GetParameters");
return fn ? fn(p) : failure;
}
API unsigned NVSDK_NGX_D3D12_DestroyParameters(void* p) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_DestroyParameters)>("NVSDK_NGX_D3D12_DestroyParameters");
return fn ? fn(p) : failure;
}
API unsigned NVSDK_NGX_D3D12_GetScratchBufferSize(unsigned feature, const void* p, size_t* size) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_GetScratchBufferSize)>("NVSDK_NGX_D3D12_GetScratchBufferSize");
return fn ? fn(feature, p, size) : failure;
}
API unsigned NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList* list, unsigned feature, void* p, void** out) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_CreateFeature)>("NVSDK_NGX_D3D12_CreateFeature");
return fn ? fn(list, feature, p, out) : failure;
}
API unsigned d4r_D3D12_EvaluateAtBoundary(ID3D12CommandQueue* queue, void* handle, void* p) {
auto fn = resolve<decltype(&d4r_D3D12_EvaluateAtBoundary)>("d4r_D3D12_EvaluateAtBoundary");
return fn ? fn(queue, handle, p) : failure;
}
API unsigned NVSDK_NGX_D3D12_EvaluateFeature(ID3D12GraphicsCommandList* list, const void* handle, void* p, void* callback) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_EvaluateFeature)>("NVSDK_NGX_D3D12_EvaluateFeature");
return fn ? fn(list, handle, p, callback) : failure;
}
API unsigned NVSDK_NGX_D3D12_EvaluateFeature_C(ID3D12GraphicsCommandList* list, const void* handle, void* p, void* callback) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_EvaluateFeature_C)>("NVSDK_NGX_D3D12_EvaluateFeature_C");
return fn ? fn(list, handle, p, callback) : failure;
}
API unsigned NVSDK_NGX_D3D12_ReleaseFeature(void* handle) {
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_ReleaseFeature)>("NVSDK_NGX_D3D12_ReleaseFeature");
return fn ? fn(handle) : failure;
}
API unsigned NVSDK_NGX_D3D12_Shutdown() {
// Nothing to shut down if no shim was ever selected.
{ std::lock_guard<std::mutex> lock(selectMutex); if (!shim) return 1; }
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_Shutdown)>("NVSDK_NGX_D3D12_Shutdown");
return fn ? fn() : failure;
}
API unsigned NVSDK_NGX_D3D12_Shutdown1(ID3D12Device* device) {
{ std::lock_guard<std::mutex> lock(selectMutex); if (!shim) return 1; }
auto fn = resolve<decltype(&NVSDK_NGX_D3D12_Shutdown1)>("NVSDK_NGX_D3D12_Shutdown1");
return fn ? fn(device) : failure;
}
tools/windows/d4r_nvngx_windows.cpp
-226
View File
@@ -1,226 +0,0 @@
// Native Windows backend. The Linux/Wine implementation remains separate.
#include "ngx_windows_runtime.h"
#include "player_config.h"
#include <dxgi.h>
#include <unordered_map>
#include <functional>
namespace {
std::mutex apiMutex;
std::shared_ptr<d4r::win::Runtime> runtime;
struct Handle { unsigned Id; };
unsigned nextId = 1;
std::unordered_map<Handle*, std::shared_ptr<d4r::win::Feature>> features;
std::unordered_map<void*, bool> parameters;
constexpr unsigned failure = 0xBAD00002, invalid = 0xBAD00005;
unsigned call(const std::function<unsigned()>& fn) noexcept {
static std::once_flag diagnosticOutput;
std::call_once(diagnosticOutput, [] {
if (std::getenv("D4R_DIAG_DIR")) {
std::setvbuf(stdout, nullptr, _IONBF, 0);
std::setvbuf(stderr, nullptr, _IONBF, 0);
}
});
try { return fn(); }
catch (const std::exception& error) {
std::fprintf(stderr, "D4R_WINDOWS_FAILURE %s\n", error.what()); return failure;
}
}
unsigned allocate(void** out, bool caps) {
if (!runtime || !out) return invalid;
void* value = d4r_ngx_parameters_create();
if (!value) return failure;
*out = nullptr;
struct LocalParameters {
void* p;
~LocalParameters() { if (p) d4r_ngx_parameters_destroy(p); }
} owned{value};
if (caps) {
std::lock_guard<std::mutex> lock(runtime->mutex);
runtime->current(); void* official = nullptr;
d4r::win::ngx_check(runtime->capabilities(&official), "CUDA capabilities");
struct Cleanup { d4r::win::Runtime& rt; void* p; ~Cleanup() { (void)rt.destroy(p); } } cleanup{*runtime, official};
for (const char* name : {"SuperSampling.Available", "SuperSampling.NeedsUpdatedDriver", "SuperSampling.FeatureInitResult"})
d4r_ngx_set_int(value, name, d4r::ngx::int_value(official, name));
for (const char* name : {"SuperSampling.MinDriverVersionMajor", "SuperSampling.MinDriverVersionMinor"})
d4r_ngx_set_uint(value, name, d4r::ngx::uint_value(official, name));
for (const char* name : {"DLSSOptimalSettingsCallback", "DLSSGetStatsCallback"}) {
void* callback = nullptr;
if (d4r_ngx_get_void(official, name, &callback) == 1) d4r_ngx_set_void(value, name, callback);
}
}
parameters.emplace(value, true); *out = value; owned.p = nullptr;
return 1;
}
}
#define API extern "C" __declspec(dllexport)
// Capability marker for an explicit external backend selection. This function
// is safe to call during DLL discovery: no GPU/runtime initialization occurs.
API unsigned d4r_WindowsBackendVersion() { return 1; }
API unsigned NVSDK_NGX_D3D12_Init_Ext(unsigned long long app, const wchar_t* data, ID3D12Device* device, unsigned sdk, const void* info) {
std::lock_guard<std::mutex> lock(apiMutex);
return call([&] {
d4r::win::player::ensure_environment();
if (!runtime) runtime = std::make_shared<d4r::win::Runtime>(device, app, data, sdk, static_cast<const d4r::ngx::FeatureCommonInfo*>(info));
if (const char* backend = std::getenv("D4R_D3D12_COMMAND_BACKEND"); backend && std::string(backend) == "1")
d4r::win::commands::install(device);
return 1u;
});
}
API unsigned NVSDK_NGX_D3D12_Init(unsigned long long app, const wchar_t* data, ID3D12Device* device, const void* info, unsigned sdk) {
return NVSDK_NGX_D3D12_Init_Ext(app, data, device, sdk, info);
}
API unsigned NVSDK_NGX_D3D12_Init_ProjectID(const char* project, int engine, const char* version, const wchar_t* data, ID3D12Device* device, unsigned sdk, const void* info) {
std::lock_guard<std::mutex> lock(apiMutex);
return call([&] {
if (!project || !*project) return invalid;
d4r::win::player::ensure_environment();
d4r::ngx::ProjectIdentity identity{project, engine, version ? version : ""};
if (!runtime) runtime = std::make_shared<d4r::win::Runtime>(device, 0, data, sdk, static_cast<const d4r::ngx::FeatureCommonInfo*>(info), &identity);
if (const char* backend = std::getenv("D4R_D3D12_COMMAND_BACKEND"); backend && std::string(backend) == "1") d4r::win::commands::install(device);
return 1u;
});
}
API unsigned NVSDK_NGX_D3D12_GetFeatureRequirements(IDXGIAdapter* adapter, const d4r::ngx::FeatureDiscoveryInfo* info,
d4r::ngx::FeatureRequirement* requirements) {
if (!adapter || !info || !requirements) return invalid;
return call([&] {
*requirements = {}; requirements->FeatureSupported = 16; // Unsupported feature.
if (info->FeatureID != 1) return 1u;
d4r::win::player::ensure_environment();
DXGI_ADAPTER_DESC desc{}; d4r::win::dx(adapter->GetDesc(&desc), "NGX requirements adapter");
// OptiScaler can override the DXGI vendor ID. The physical HIP
// architecture and full adapter LUID are the authoritative identity.
d4r::diag::HipApi hip(d4r::win::env_path("D4R_HIP_ROOT")); hipDeviceProp_t props{};
const int ordinal = hip.select_architecture(-1,props,&desc.AdapterLuid,true);
requirements->FeatureSupported = ordinal < 0 ? 4u : 0u;
std::strcpy(requirements->MinOSVersion,"10.0.22000.0");
std::printf("D4R_REQUIREMENTS feature=%u support=%u architecture=%s adapter_luid=%08lx:%08lx hip_ordinal=%d\n",
info->FeatureID,requirements->FeatureSupported,d4r::diag::HipApi::target_arch(),
static_cast<unsigned long>(desc.AdapterLuid.HighPart),static_cast<unsigned long>(desc.AdapterLuid.LowPart),ordinal);
return 1u;
});
}
API unsigned NVSDK_NGX_D3D12_AllocateParameters(void** p) {
std::lock_guard<std::mutex> lock(apiMutex); return call([&] { return allocate(p, false); });
}
API unsigned NVSDK_NGX_D3D12_GetCapabilityParameters(void** p) {
std::lock_guard<std::mutex> lock(apiMutex); return call([&] { return allocate(p, true); });
}
API unsigned NVSDK_NGX_D3D12_GetParameters(void** p) { return NVSDK_NGX_D3D12_GetCapabilityParameters(p); }
API unsigned NVSDK_NGX_D3D12_DestroyParameters(void* p) {
std::lock_guard<std::mutex> lock(apiMutex);
if (!parameters.erase(p)) return invalid;
d4r_ngx_parameters_destroy(p); return 1;
}
API unsigned NVSDK_NGX_D3D12_GetScratchBufferSize(unsigned, const void*, size_t* size) {
if (!size) return invalid; *size = 0; return 1;
}
API unsigned NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, unsigned feature, void* p, Handle** out) {
std::lock_guard<std::mutex> lock(apiMutex);
return call([&] {
if (!runtime || feature != 1 || !p || !out) return invalid;
d4r::win::player::apply_preset(p);
auto value = std::make_shared<d4r::win::Feature>(runtime, p);
auto handle = std::make_unique<Handle>(); handle->Id = nextId++;
features.emplace(handle.get(), std::move(value)); *out = handle.release(); return 1u;
});
}
API unsigned d4r_D3D12_EvaluateAtBoundary(ID3D12CommandQueue* queue, Handle* handle, void* p) {
std::shared_ptr<d4r::win::Feature> feature;
{ std::lock_guard<std::mutex> lock(apiMutex);
auto it = features.find(handle); if (it == features.end() || !p) return invalid; feature = it->second; }
return call([&] { feature->evaluate_boundary(queue, p); return 1u; });
}
API unsigned NVSDK_NGX_D3D12_EvaluateFeature(ID3D12GraphicsCommandList* list, const Handle* handle, void* p, void*) {
std::shared_ptr<d4r::win::Feature> feature;
{ std::lock_guard<std::mutex> lock(apiMutex);
auto it = features.find(const_cast<Handle*>(handle));
if (it == features.end() || !p || !list) return invalid; feature = it->second; }
return call([&] {
struct Snapshot {
void* parameters = d4r_ngx_parameters_create();
std::vector<d4r::win::ComPtr<ID3D12Resource>> resources;
d4r::win::ResourceAccess access[5];
~Snapshot() { if (parameters) d4r_ngx_parameters_destroy(parameters); }
};
auto snapshot = std::make_shared<Snapshot>();
if (!snapshot->parameters) return failure;
d4r::win::player::apply_preset(p);
d4r::ngx::copy_create(p, snapshot->parameters); d4r::ngx::copy_frame(p, snapshot->parameters);
// NVIDIA's library stamps "DLSS SDK - DO NOT DISTRIBUTE" over its output unless the SDK's
// Disable.Watermark parameter is set (the Linux shim sets it too). The game's own value is
// not used. [DLSS] ShowWatermark = true (D4R_SHIM_WATERMARK=1) keeps the stamp.
static const bool watermark = [] { const char* v = std::getenv("D4R_SHIM_WATERMARK"); return v && v[0] == '1'; }();
d4r_ngx_set_int(snapshot->parameters, "Disable.Watermark", watermark ? 0 : 1);
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* resource = nullptr; (void)d4r_ngx_get_d3d12_resource(p, names[i], &resource);
snapshot->resources.emplace_back(resource);
d4r_ngx_set_d3d12_resource(snapshot->parameters, names[i], resource);
const auto state = d4r::win::commands::resource_access(list, resource, d4r::win::ngx_resource_access(p, states[i],
i == 3 ? D3D12_RESOURCE_STATE_UNORDERED_ACCESS : D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE));
snapshot->access[i] = state;
if (state.pending_split) throw std::runtime_error("NGX resource has an unfinished split barrier");
d4r_ngx_set_uint(snapshot->parameters, states[i], unsigned(state.legacy));
const std::string prefix(states[i]);
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Enhanced").c_str(), state.enhanced);
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Layout").c_str(), unsigned(state.layout));
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Access").c_str(), unsigned(state.access));
if (!std::getenv("D4R_QUIET_API")) std::printf("D4R_RESOURCE_ACCESS plane=%s enhanced=%u legacy=0x%x layout=%u access=0x%x\n",
names[i], state.enhanced, unsigned(state.legacy), unsigned(state.layout), unsigned(state.access));
}
d4r::win::commands::record_boundary(list, [feature, snapshot](ID3D12CommandQueue* queue) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
for (unsigned i = 0; i < 5; ++i) if (snapshot->access[i].inherited) {
const auto resolved = d4r::win::commands::submitted_resource_access(snapshot->resources[i].Get(), snapshot->access[i]);
if (resolved.pending_split) throw std::runtime_error("Inherited NGX texture has an unfinished split barrier");
const std::string prefix(states[i]);
d4r_ngx_set_uint(snapshot->parameters, states[i], unsigned(resolved.legacy));
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Enhanced").c_str(), resolved.enhanced);
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Layout").c_str(), unsigned(resolved.layout));
d4r_ngx_set_uint(snapshot->parameters, (prefix + ".Access").c_str(), unsigned(resolved.access));
if (!std::getenv("D4R_QUIET_API")) std::printf("D4R_INHERITED_ACCESS plane=%s tracked=%u enhanced=%u layout=%u legacy=0x%x\n",
names[i], !resolved.inherited, resolved.enhanced, unsigned(resolved.layout), unsigned(resolved.legacy));
}
feature->evaluate_boundary(queue, snapshot->parameters);
});
return 1u;
});
}
API unsigned NVSDK_NGX_D3D12_EvaluateFeature_C(ID3D12GraphicsCommandList* list, const Handle* handle, void* p, void* callback) {
return NVSDK_NGX_D3D12_EvaluateFeature(list, handle, p, callback);
}
API unsigned NVSDK_NGX_D3D12_ReleaseFeature(Handle* handle) {
std::shared_ptr<d4r::win::Feature> feature;
{ std::lock_guard<std::mutex> lock(apiMutex);
auto it = features.find(handle); if (it == features.end()) return invalid;
feature = std::move(it->second); features.erase(it); delete handle; }
// OptiScaler frees its root-CBV/descriptor backing after Release returns.
// A retained C++ Feature alone does not keep that caller-owned memory alive.
return call([&] { feature->drain(); feature.reset(); return 1u; });
}
API unsigned NVSDK_NGX_D3D12_Shutdown() {
std::lock_guard<std::mutex> lock(apiMutex);
return call([&] {
if (runtime && std::getenv("D4R_ASYNC_INTEROP")) runtime->drain_async_queue();
for (auto& pair : features) delete pair.first;
features.clear();
for (auto& pair : parameters) d4r_ngx_parameters_destroy(pair.first);
parameters.clear(); runtime.reset(); return 1u;
});
}
API unsigned NVSDK_NGX_D3D12_Shutdown1(ID3D12Device*) { return NVSDK_NGX_D3D12_Shutdown(); }
// Public diagnostics exercise queue/list integration without NVIDIA binaries.
API unsigned d4r_D3D12_InstallCommandBackend(ID3D12Device* device) {
return call([&] { d4r::win::commands::install(device); return 1u; });
}
using DiagnosticBoundary = void(WINAPI*)(ID3D12CommandQueue*, void*);
API unsigned d4r_D3D12_RecordDiagnosticBoundary(ID3D12GraphicsCommandList* list, DiagnosticBoundary callback, void* context) {
if (!callback) return invalid;
return call([&] { d4r::win::commands::record_boundary(list, [=](ID3D12CommandQueue* queue) { callback(queue, context); }); return 1u; });
}
API unsigned long long d4r_D3D12_LiveRecordings() { return d4r::win::commands::live_recordings(); }
tools/windows/debug_launcher.cpp
-170
View File
@@ -1,170 +0,0 @@
// Out-of-process crash capture: unaffected by the frontend's loader hooks.
#include "diagnostic.h"
#include <map>
#include <fstream>
#include <tlhelp32.h>
namespace {
std::wstring quote(const std::wstring& value) {
std::wstring result = L"\""; unsigned slashes = 0;
for (wchar_t c : value) {
if (c == L'\\') { ++slashes; continue; }
result.append(slashes * (c == L'\"' ? 2 : 1), L'\\'); slashes = 0;
if (c == L'\"') result += L'\\';
result += c;
}
result.append(slashes * 2, L'\\'); return result + L'\"';
}
std::string file_path(HANDLE file) {
wchar_t path[32768]{};
const auto length = GetFinalPathNameByHandleW(file, path, 32768, FILE_NAME_NORMALIZED);
return length && length < 32768 ? d4r::diag::utf8(path) : "unknown";
}
struct Module { std::string path; uint32_t size; };
uint32_t module_size(HANDLE process, uintptr_t base) {
IMAGE_DOS_HEADER dos{}; IMAGE_NT_HEADERS64 nt{}; SIZE_T read = 0;
if (!ReadProcessMemory(process, reinterpret_cast<void*>(base), &dos, sizeof(dos), &read) ||
dos.e_magic != IMAGE_DOS_SIGNATURE || dos.e_lfanew < 0 || dos.e_lfanew > 65536 ||
!ReadProcessMemory(process, reinterpret_cast<void*>(base + dos.e_lfanew), &nt, sizeof(nt), &read) ||
nt.Signature != IMAGE_NT_SIGNATURE) return 0;
return nt.OptionalHeader.SizeOfImage;
}
}
int main(int argc, char** argv) {
try {
const bool debug = argc < 4 || std::strcmp(argv[3], "--no-debug");
const int separator = debug ? 3 : 4;
if (argc <= separator + 1 || std::strcmp(argv[1], "--output-directory") || std::strcmp(argv[separator], "--"))
throw std::runtime_error("Usage: d4r_debug_launcher --output-directory PATH [--no-debug] -- EXE [arguments]");
const std::filesystem::path directory(d4r::diag::wide(argv[2]));
std::filesystem::create_directories(directory);
std::ofstream log(directory / L"debugger.log", std::ios::app);
const auto executable = d4r::diag::wide(argv[separator + 1]);
std::wstring command;
for (int i = separator + 1; i < argc; ++i) { if (i > separator + 1) command += L' '; command += quote(d4r::diag::wide(argv[i])); }
STARTUPINFOW startup{sizeof(startup)}; PROCESS_INFORMATION child{};
startup.dwFlags = STARTF_USESTDHANDLES;
startup.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
startup.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
startup.hStdError = GetStdHandle(STD_ERROR_HANDLE);
HANDLE job = CreateJobObjectW(nullptr, nullptr);
JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits{};
limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
if (!job || !SetInformationJobObject(job, JobObjectExtendedLimitInformation, &limits, sizeof(limits)))
throw std::runtime_error("Cannot create diagnostic process lifetime job");
if (!CreateProcessW(executable.c_str(), command.data(), nullptr, nullptr, TRUE,
(debug ? DEBUG_ONLY_THIS_PROCESS : 0) | CREATE_SUSPENDED | CREATE_NO_WINDOW, nullptr, nullptr, &startup, &child))
throw std::runtime_error("CreateProcess debugger error=" + std::to_string(GetLastError()));
if (!AssignProcessToJobObject(job, child.hProcess)) {
TerminateProcess(child.hProcess, 1); CloseHandle(job);
throw std::runtime_error("AssignProcessToJobObject debugger failed");
}
log << "LAUNCH pid=" << child.dwProcessId << " attached_debugger=" << debug << std::endl;
if (ResumeThread(child.hThread) == DWORD(-1)) {
CloseHandle(job); throw std::runtime_error("Cannot resume diagnostic child");
}
if (!debug) {
// Retain bounded process ownership and raw inherited output without
// debugger stops on DLL/thread/exception/debug-string events.
std::map<uintptr_t, Module> inventory;
while (WaitForSingleObject(child.hProcess, 1000) == WAIT_TIMEOUT) {
HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE | TH32CS_SNAPMODULE32, child.dwProcessId);
if (snapshot == INVALID_HANDLE_VALUE) continue;
MODULEENTRY32W module{sizeof(module)};
if (Module32FirstW(snapshot, &module)) do {
const auto base = reinterpret_cast<uintptr_t>(module.modBaseAddr);
if (inventory.count(base)) continue;
inventory[base] = {d4r::diag::utf8(module.szExePath), module.modBaseSize};
log << "LOAD base=0x" << std::hex << base << " size=0x" << module.modBaseSize
<< " path=" << inventory[base].path << '\n';
} while (Module32NextW(snapshot, &module));
CloseHandle(snapshot); log.flush();
}
DWORD result = 1;
GetExitCodeProcess(child.hProcess, &result);
log << "EXIT code=0x" << std::hex << result << std::endl;
CloseHandle(child.hThread); CloseHandle(child.hProcess); CloseHandle(job);
return static_cast<int>(result);
}
std::map<uintptr_t, Module> modules;
bool initialBreakpoint = true, dumped = false; DWORD exitCode = 1;
auto add_module = [&](void* base, HANDLE file) {
const auto address = reinterpret_cast<uintptr_t>(base);
modules[address] = {file_path(file), module_size(child.hProcess, address)};
log << "LOAD base=0x" << std::hex << address << " size=0x" << modules[address].size << " path=" << modules[address].path << std::endl;
if (file && file != INVALID_HANDLE_VALUE) CloseHandle(file);
};
auto address_info = [&](uintptr_t address) {
auto it = modules.upper_bound(address);
if (it != modules.begin()) {
--it;
if (address - it->first < it->second.size)
log << " module=" << it->second.path << "+0x" << std::hex << address - it->first;
}
};
DEBUG_EVENT event{};
uint64_t debugEvents = 0, debugStrings = 0, exceptions = 0;
ULONGLONG lastSample = GetTickCount64();
while (WaitForDebugEvent(&event, INFINITE)) {
++debugEvents;
if (event.dwDebugEventCode == OUTPUT_DEBUG_STRING_EVENT) ++debugStrings;
if (event.dwDebugEventCode == EXCEPTION_DEBUG_EVENT) ++exceptions;
const auto now = GetTickCount64();
if (now - lastSample >= 5000) {
log << "DEBUG_EVENTS events=" << std::dec << debugEvents << " debug_strings=" << debugStrings << " exceptions=" << exceptions << std::endl;
lastSample = now;
}
DWORD disposition = DBG_CONTINUE;
if (event.dwDebugEventCode == CREATE_PROCESS_DEBUG_EVENT) {
add_module(event.u.CreateProcessInfo.lpBaseOfImage, event.u.CreateProcessInfo.hFile);
CloseHandle(event.u.CreateProcessInfo.hProcess); CloseHandle(event.u.CreateProcessInfo.hThread);
} else if (event.dwDebugEventCode == CREATE_THREAD_DEBUG_EVENT) CloseHandle(event.u.CreateThread.hThread);
else if (event.dwDebugEventCode == LOAD_DLL_DEBUG_EVENT) add_module(event.u.LoadDll.lpBaseOfDll, event.u.LoadDll.hFile);
else if (event.dwDebugEventCode == UNLOAD_DLL_DEBUG_EVENT) {
const auto base = reinterpret_cast<uintptr_t>(event.u.UnloadDll.lpBaseOfDll);
log << "UNLOAD base=0x" << std::hex << base << std::endl; modules.erase(base);
} else if (event.dwDebugEventCode == EXCEPTION_DEBUG_EVENT) {
const auto& exception = event.u.Exception;
const DWORD code = exception.ExceptionRecord.ExceptionCode;
log << "EXCEPTION code=0x" << std::hex << code << " firstChance=" << exception.dwFirstChance
<< " address=0x" << reinterpret_cast<uintptr_t>(exception.ExceptionRecord.ExceptionAddress) << std::endl;
disposition = DBG_EXCEPTION_NOT_HANDLED;
if (initialBreakpoint && code == EXCEPTION_BREAKPOINT) { initialBreakpoint = false; disposition = DBG_CONTINUE; }
if (!dumped && (!exception.dwFirstChance || code == EXCEPTION_ACCESS_VIOLATION || code == EXCEPTION_ILLEGAL_INSTRUCTION || code == EXCEPTION_STACK_OVERFLOW || code == EXCEPTION_IN_PAGE_ERROR)) {
dumped = true;
log << "FAULT code=0x" << std::hex << code << " address=0x" << reinterpret_cast<uintptr_t>(exception.ExceptionRecord.ExceptionAddress)
<< " thread=" << std::dec << event.dwThreadId << " firstChance=" << exception.dwFirstChance;
address_info(reinterpret_cast<uintptr_t>(exception.ExceptionRecord.ExceptionAddress)); log << std::endl;
HANDLE thread = OpenThread(THREAD_GET_CONTEXT | THREAD_QUERY_INFORMATION, FALSE, event.dwThreadId);
CONTEXT context{}; context.ContextFlags = CONTEXT_ALL;
if (thread && GetThreadContext(thread, &context)) {
log << "CONTEXT rip=0x" << std::hex << context.Rip << " rsp=0x" << context.Rsp << " rcx=0x" << context.Rcx << " rdx=0x" << context.Rdx << std::endl;
EXCEPTION_RECORD record = exception.ExceptionRecord;
EXCEPTION_POINTERS pointers{&record, &context};
MINIDUMP_EXCEPTION_INFORMATION info{event.dwThreadId, &pointers, FALSE};
const auto path = directory / (L"crash-" + std::to_wstring(child.dwProcessId) + L".dmp");
HANDLE file = CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, 0, nullptr);
if (file != INVALID_HANDLE_VALUE) {
const BOOL ok = MiniDumpWriteDump(child.hProcess, child.dwProcessId, file,
static_cast<MINIDUMP_TYPE>(MiniDumpNormal | MiniDumpWithThreadInfo), &info, nullptr, nullptr);
log << "MINIDUMP success=" << ok << " error=" << (ok ? 0 : GetLastError()) << std::endl; CloseHandle(file);
}
uintptr_t stack[256]{}; SIZE_T read = 0;
if (ReadProcessMemory(child.hProcess, reinterpret_cast<void*>(context.Rsp), stack, sizeof(stack), &read))
for (unsigned i = 0; i < read / sizeof(uintptr_t); ++i) {
log << "STACK slot=" << std::dec << i << " address=0x" << std::hex << stack[i]; address_info(stack[i]); log << '\n';
}
}
if (thread) CloseHandle(thread); log.flush();
}
} else if (event.dwDebugEventCode == EXIT_PROCESS_DEBUG_EVENT) {
exitCode = event.u.ExitProcess.dwExitCode;
log << "EXIT code=0x" << std::hex << exitCode << std::endl;
ContinueDebugEvent(event.dwProcessId, event.dwThreadId, disposition); break;
}
ContinueDebugEvent(event.dwProcessId, event.dwThreadId, disposition);
}
CloseHandle(child.hThread); CloseHandle(child.hProcess); CloseHandle(job);
return static_cast<int>(exitCode);
} catch (const std::exception& error) { std::fprintf(stderr, "DEBUG_LAUNCH_FAILURE %s\n", error.what()); return 1; }
}
tools/windows/diagnostic.h
-286
View File
@@ -1,286 +0,0 @@
#pragma once
#include <windows.h>
#include <tlhelp32.h>
#include <dbghelp.h>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <stdexcept>
#include <string>
#include <vector>
#include <algorithm>
namespace d4r::diag {
inline std::wstring wide(const std::string& s)
{
const int n = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), -1, nullptr, 0);
if (!n) throw std::runtime_error("Invalid UTF-8 path");
std::wstring out(n, L'\0');
MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s.c_str(), -1, out.data(), n);
out.pop_back();
return out;
}
inline std::string utf8(const wchar_t* s)
{
const int n = WideCharToMultiByte(CP_UTF8, 0, s, -1, nullptr, 0, nullptr, nullptr);
std::string out(n, '\0');
WideCharToMultiByte(CP_UTF8, 0, s, -1, out.data(), n, nullptr, nullptr);
if (n) out.pop_back();
return out;
}
inline void loaded_modules()
{
HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, GetCurrentProcessId());
if (snapshot == INVALID_HANDLE_VALUE) return;
MODULEENTRY32W entry{};
entry.dwSize = sizeof(entry);
if (Module32FirstW(snapshot, &entry)) do {
DWORD unused = 0;
DWORD size = GetFileVersionInfoSizeW(entry.szExePath, &unused);
std::vector<unsigned char> bytes(size);
VS_FIXEDFILEINFO* version = nullptr;
UINT length = 0;
std::printf("DLL path=%s", utf8(entry.szExePath).c_str());
if (size && GetFileVersionInfoW(entry.szExePath, 0, size, bytes.data()) &&
VerQueryValueW(bytes.data(), L"\\", reinterpret_cast<void**>(&version), &length) &&
length >= sizeof(*version))
std::printf(" version=%u.%u.%u.%u", HIWORD(version->dwFileVersionMS),
LOWORD(version->dwFileVersionMS), HIWORD(version->dwFileVersionLS),
LOWORD(version->dwFileVersionLS));
std::printf("\n");
} while (Module32NextW(snapshot, &entry));
CloseHandle(snapshot);
}
inline LONG WINAPI unhandled(EXCEPTION_POINTERS* info)
{
// Loader hooks can fault while holding the loader lock. Do not enumerate
// modules or read their resources here; the dump contains the module list.
// Guard the final filter as well as the first-chance handler against a
// second exception inside DbgHelp.
static LONG dumping = 0;
if (InterlockedCompareExchange(&dumping, 1, 0) != 0)
return EXCEPTION_EXECUTE_HANDLER;
std::fprintf(stderr, "EXCEPTION code=0x%08lx address=%p\n",
info->ExceptionRecord->ExceptionCode, info->ExceptionRecord->ExceptionAddress);
if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION &&
info->ExceptionRecord->NumberParameters >= 2)
std::fprintf(stderr, "EXCEPTION_ACCESS operation=%llu target=0x%llx\n",
static_cast<unsigned long long>(info->ExceptionRecord->ExceptionInformation[0]),
static_cast<unsigned long long>(info->ExceptionRecord->ExceptionInformation[1]));
std::fprintf(stderr, "CONTEXT rip=0x%llx rsp=0x%llx rcx=0x%llx rdx=0x%llx r8=0x%llx r9=0x%llx\n",
static_cast<unsigned long long>(info->ContextRecord->Rip),
static_cast<unsigned long long>(info->ContextRecord->Rsp),
static_cast<unsigned long long>(info->ContextRecord->Rcx),
static_cast<unsigned long long>(info->ContextRecord->Rdx),
static_cast<unsigned long long>(info->ContextRecord->R8),
static_cast<unsigned long long>(info->ContextRecord->R9));
wchar_t directory[32768]{};
const DWORD n = GetEnvironmentVariableW(L"D4R_DIAG_DIR", directory, 32768);
if (n && n < 32768) {
const std::wstring path = std::wstring(directory) + L"\\crash-" +
std::to_wstring(GetCurrentProcessId()) + L".dmp";
HANDLE file = CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, 0, nullptr);
if (file != INVALID_HANDLE_VALUE) {
MINIDUMP_EXCEPTION_INFORMATION exception{GetCurrentThreadId(), info, FALSE};
const BOOL ok = MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), file,
MiniDumpNormal, &exception, nullptr, nullptr);
std::fprintf(stderr, "MINIDUMP success=%d error=%lu\n", ok, ok ? 0 : GetLastError());
CloseHandle(file);
}
}
return EXCEPTION_EXECUTE_HANDLER;
}
inline LONG WINAPI first_chance(EXCEPTION_POINTERS* info)
{
// NGX installs its own final exception filter. Capture a diagnostic dump
// before that filter can replace ours; do not swallow or retry the fault.
const auto code = info->ExceptionRecord->ExceptionCode;
if (code == EXCEPTION_ACCESS_VIOLATION || code == EXCEPTION_ILLEGAL_INSTRUCTION) {
static LONG captured = 0;
if (InterlockedCompareExchange(&captured, 1, 0) == 0) (void)unhandled(info);
}
return EXCEPTION_CONTINUE_SEARCH;
}
inline void start()
{
std::setvbuf(stdout, nullptr, _IONBF, 0);
std::setvbuf(stderr, nullptr, _IONBF, 0);
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
SetUnhandledExceptionFilter(unhandled);
AddVectoredExceptionHandler(1, first_chance);
std::printf("d4r native Windows diagnostic x64 pid=%lu\n", GetCurrentProcessId());
}
struct Args {
std::string hip_root, cuda_dll, module, context = "primary";
std::string stream_lifetime = "persistent";
std::string interop_mode = "roundtrip";
std::string pixel_profile = "baseline";
std::string barrier_mode = "legacy";
bool early_indirect = false;
std::string ngx_core, dlss_dll;
std::string nvapi_dll;
std::string ngx_abi = "driver";
unsigned long long ngx_app_id = 241534723ull;
std::string ngx_frontend = "d4r";
std::string ngx_mode = "init";
unsigned preset = 11;
unsigned ngx_create_flags = 0;
unsigned output_width = 512, output_height = 288;
unsigned input_width = 0, input_height = 0;
std::string fixture_dir;
std::string benchmark_module;
std::string benchmark_timing = "events";
unsigned benchmark_batch = 1;
std::string output_dir;
std::string kernel_name = "enc1";
unsigned iterations = 32;
int device = -1;
Args(int argc, char** argv) {
for (int i = 1; i < argc; ++i) {
const std::string key = argv[i];
if (i + 1 == argc) throw std::runtime_error("Missing value for " + key);
const std::string value = argv[++i];
if (key == "--hip-root") hip_root = value;
else if (key == "--cuda-dll") cuda_dll = value;
else if (key == "--module") module = value;
else if (key == "--context") context = value;
else if (key == "--stream-lifetime") stream_lifetime = value;
else if (key == "--interop-mode") interop_mode = value;
else if (key == "--pixel-profile") {
if (value != "baseline" && value != "packed" && value != "unorm" && value != "depth-stencil") throw std::runtime_error("Invalid pixel-profile");
pixel_profile = value;
}
else if (key == "--barrier-mode") {
if (value != "legacy" && value != "enhanced" && value != "inherited-legacy" && value != "inherited-enhanced") throw std::runtime_error("Invalid barrier-mode");
barrier_mode = value;
}
else if (key == "--early-indirect") early_indirect = value == "1";
else if (key == "--ngx-core") ngx_core = value;
else if (key == "--dlss-dll") dlss_dll = value;
else if (key == "--nvapi-dll") nvapi_dll = value;
else if (key == "--ngx-abi") ngx_abi = value;
else if (key == "--ngx-app-id") {
size_t end = 0;
if (value.empty() || value[0] == '-') throw std::runtime_error("Invalid NGX application ID");
ngx_app_id = std::stoull(value, &end);
if (end != value.size()) throw std::runtime_error("Invalid NGX application ID");
}
else if (key == "--ngx-frontend") {
if (value != "d4r" && value != "optiscaler") throw std::runtime_error("ngx-frontend must be d4r or optiscaler");
ngx_frontend = value;
}
else if (key == "--ngx-mode") ngx_mode = value;
else if (key == "--ngx-create-flags") {
size_t end = 0;
ngx_create_flags = static_cast<unsigned>(std::stoul(value, &end));
if (end != value.size() || (ngx_create_flags != 0 && ngx_create_flags != 11))
throw std::runtime_error("ngx-create-flags must be 0 or 11 (HDR, low-resolution MV, inverted depth)");
}
else if (key == "--ngx-output-resolution") {
const auto separator = value.find('x');
if (separator == std::string::npos) throw std::runtime_error("Output resolution must be WIDTHxHEIGHT");
size_t wend = 0, hend = 0;
output_width = static_cast<unsigned>(std::stoul(value.substr(0, separator), &wend));
output_height = static_cast<unsigned>(std::stoul(value.substr(separator + 1), &hend));
if (wend != separator || hend != value.size() - separator - 1 || output_width < 128 || output_width > 4096 ||
output_height < 128 || output_height > 2160 || output_width % 2 || output_height % 2)
throw std::runtime_error("Output resolution must be even and within 128x128..4096x2160");
}
else if (key == "--ngx-input-resolution") {
const auto separator = value.find('x');
if (separator == std::string::npos) throw std::runtime_error("Input resolution must be WIDTHxHEIGHT");
size_t wend = 0, hend = 0;
const auto width = std::stoul(value.substr(0, separator), &wend);
const auto height = std::stoul(value.substr(separator + 1), &hend);
if (wend != separator || hend != value.size() - separator - 1 || width < 64 || width > 4096 ||
height < 64 || height > 2160)
throw std::runtime_error("Input resolution must be within 64x64..4096x2160");
input_width = static_cast<unsigned>(width); input_height = static_cast<unsigned>(height);
}
else if (key == "--preset") {
size_t end = 0;
preset = static_cast<unsigned>(std::stoul(value, &end));
if (end != value.size() || (preset != 5 && preset != 11 && preset != 12 && preset != 13))
throw std::runtime_error("preset must be 5 (E), 11 (K), 12 (L) or 13 (M)");
}
else if (key == "--fixture-dir") fixture_dir = value;
else if (key == "--benchmark-module") benchmark_module = value;
else if (key == "--benchmark-timing") {
if (value != "events" && value != "dispatch")
throw std::runtime_error("benchmark-timing must be events or dispatch");
benchmark_timing = value;
}
else if (key == "--benchmark-batch") {
size_t end = 0;
const auto parsed = std::stoul(value, &end);
if (end != value.size() || parsed < 1 || parsed > 256)
throw std::runtime_error("benchmark-batch must be 1..256");
benchmark_batch = static_cast<unsigned>(parsed);
}
else if (key == "--output-dir") output_dir = value;
else if (key == "--kernel-name") kernel_name = value;
else if (key == "--iterations") {
size_t end = 0;
iterations = static_cast<unsigned>(std::stoul(value, &end));
if (end != value.size() || iterations < 1 || iterations > 10000)
throw std::runtime_error("iterations must be 1..10000");
} else if (key == "--device") {
size_t end = 0;
device = std::stoi(value, &end);
if (end != value.size() || device < 0) throw std::runtime_error("Invalid device ordinal");
} else throw std::runtime_error("Unknown option " + key);
}
if (hip_root.empty()) throw std::runtime_error("--hip-root is required");
if (input_width > output_width || input_height > output_height)
throw std::runtime_error("Input resolution cannot exceed output resolution");
}
};
class Library {
HMODULE handle_ = nullptr;
public:
explicit Library(const std::filesystem::path& path) {
if (!path.is_absolute()) throw std::runtime_error("DLL path must be absolute");
handle_ = LoadLibraryExW(path.c_str(), nullptr,
LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | LOAD_LIBRARY_SEARCH_USER_DIRS);
if (!handle_) throw std::runtime_error("LoadLibrary " + utf8(path.c_str()) +
" Win32=" + std::to_string(GetLastError()));
wchar_t actual[32768]{};
GetModuleFileNameW(handle_, actual, 32768);
std::printf("LOAD requested=%s actual=%s\n", utf8(path.c_str()).c_str(), utf8(actual).c_str());
}
Library(const Library&) = delete;
Library& operator=(const Library&) = delete;
~Library() { if (handle_) FreeLibrary(handle_); }
template<class T> T symbol(const char* name) const {
FARPROC raw = GetProcAddress(handle_, name);
if (!raw) throw std::runtime_error(std::string("Missing export ") + name);
static_assert(sizeof(T) == sizeof(raw));
T out;
std::memcpy(&out, &raw, sizeof(out));
return out;
}
};
class SearchDirectory {
DLL_DIRECTORY_COOKIE cookie_ = nullptr;
public:
explicit SearchDirectory(const std::filesystem::path& path) {
if (!path.is_absolute()) throw std::runtime_error("HIP root must be absolute");
cookie_ = AddDllDirectory(path.c_str());
if (!cookie_) throw std::runtime_error("AddDllDirectory failed");
}
~SearchDirectory() { if (cookie_) RemoveDllDirectory(cookie_); }
};
inline uint32_t pattern(uint32_t i, uint32_t seed) { return (i * 1664525u + seed) ^ 0xa5a55a5au; }
inline bool verify(const std::vector<uint32_t>& result, uint32_t count, uint32_t seed) {
for (size_t i = 0; i < result.size(); ++i) {
const uint32_t expected = i < count ? pattern(static_cast<uint32_t>(i), seed) : 0xdeadbeefu;
if (result[i] != expected) {
std::fprintf(stderr, "MISMATCH index=%zu expected=0x%08x actual=0x%08x\n", i, expected, result[i]);
return false;
}
}
return true;
}
}
tools/windows/dll-pins.txt
-14
View File
@@ -1,14 +0,0 @@
# NVIDIA DLLs the Windows backend accepts, by exact SHA256. NVIDIA DLLs are
# never shipped or committed; players copy their own.
# dlss = nvngx_dlss.dll (d4r\nvngx_dlss.dll)
# ngx = _nvngx.dll, NVIDIA's NGX core from a display driver (d4r\ngx\_nvngx.dll)
# status: validated = tested on Windows hardware by the project; anything else is
# accepted but reported as having had less testing in d4r\d4r_nvngx.log and d4r-check.ps1.
# Native kernels only replace DLSS code whose PTX identity matches; anything else
# falls back to ZLUDA's translated kernels, so an accepted DLL is never run with a
# wrong native kernel.
# kind sha256 version status
dlss 3975567B8943C53ACCE397F2B72380092F84F162D00B0D2C7D08A1025C563983 310.9.1 validated
dlss 1CDFDC957CB7FC9805500CA6793F607EF2D4DBD7C967FEAC70ACA9CAF382D0C0 310.7.0 unvalidated
ngx 66767018C36B3BAB46398DADE3ADF173DAA3730FDA75965689EA848C9BC4E79B 32.0.16.1714 validated
ngx 0765FB7FB2B8BE28EE1F0226679D7B2636A69CACCE5A23891984978F690D81A4 32.0.15.9636 unvalidated
tools/windows/fp8_pack_probe.hip
-9
View File
@@ -1,9 +0,0 @@
// Diagnostic only: all finite OCP e4m3 encodings, including signed zero.
#include "../../kernels/m/swin_common.h"
struct PackParams { const half_t* input; uint32_t* output; };
extern "C" __global__ void fp8_pack_probe(PackParams p) {
const unsigned lane = blockIdx.x * 32 + threadIdx.x;
if (lane >= 32) return;
const auto packed = to_fp8(*(const sop*)(p.input + lane * 8));
p.output[lane * 2] = packed[0]; p.output[lane * 2 + 1] = packed[1];
}
tools/windows/fp8_pack_validate.py
-46
View File
@@ -1,46 +0,0 @@
"""Check gfx1201 hardware packing of every finite OCP e4m3 value and signed zero."""
import argparse
import hashlib
import json
import os
import pathlib
import struct
import subprocess
import numpy as np
from m_replay_validate import me
def main(args):
root = args.output_dir.resolve()
root.mkdir(parents=True, exist_ok=True)
codes = np.arange(256, dtype=np.uint8)
codes[(codes & 127) == 127] = 0 # NaN inputs are outside this finite-value packing contract.
inputs = me.E4M3[codes].astype(np.float16).tobytes()
(root / 'alloc-0.bin').write_bytes(inputs)
(root / 'alloc-1.bin').write_bytes(bytes(256))
(root / 'args.bin').write_bytes(struct.pack('<QQ', 0x100000000, 0x200000000))
(root / 'manifest.txt').write_text('kernel fp8_pack_probe\nlaunch 1 1 1 32 1 1 0\nargs 16\n'
'alloc 0 0x100000000 512\npointer 0 0 0\nalloc 1 0x200000000 256\npointer 8 1 0\n')
with (root / 'gpu.stdout.log').open('wb') as stdout, (root / 'gpu.stderr.log').open('wb') as stderr:
result = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
'--module', str(args.module.resolve()), '--fixture-dir', str(root), '--output-dir', str(root / 'output'),
'--iterations', '1'], stdout=stdout, stderr=stderr, timeout=90,
env=dict(os.environ, D4R_DIAG_DIR=str(root)))
summary = dict(passed=False, architecture='gfx1201', finiteEncodings=254, signedZero=True,
moduleSha256=hashlib.sha256(args.module.read_bytes()).hexdigest(), exitCode=f'0x{result.returncode & 0xffffffff:08x}')
if result.returncode == 0:
actual = np.fromfile(root / 'output/alloc-1.bin', np.uint8)
summary['passed'] = actual.size == 256 and bool(np.array_equal(actual, codes))
if actual.size == 256:
summary['differingCodes'] = int(np.count_nonzero(actual != codes))
(root / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(('PASS' if summary['passed'] else 'FAIL') + ' FP8_HARDWARE_PACK finite_encodings=254 signed_zero=1 architecture=gfx1201', flush=True)
return 0 if summary['passed'] else 1
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
for name in ('probe', 'module', 'hip-root', 'output-dir'):
parser.add_argument('--' + name, type=pathlib.Path, required=True)
raise SystemExit(main(parser.parse_args()))
tools/windows/frame_compare.py
-62
View File
@@ -1,62 +0,0 @@
"""Validate every RGBA16F diagnostic frame against another complete pipeline."""
import argparse
import math
import pathlib
import sys
import numpy as np
def main(args):
frames = sorted(args.reference.glob('output-*.rgba16f'))
if not frames:
raise RuntimeError('Reference contains no frames')
for path in frames:
actual = args.actual / path.name
if not actual.is_file():
raise RuntimeError(f'Missing output frame {path.name}')
reference = np.fromfile(path, np.float16)
output = np.fromfile(actual, np.float16)
if reference.size != output.size or reference.size != args.width * args.height * 4:
raise RuntimeError(f'Output size mismatch for {path.name}')
if not np.isfinite(reference).all() or not np.isfinite(output).all():
raise RuntimeError(f'NaN/Inf in {path.name}')
if args.output_storage == 'r11g11b10f':
# D3D float narrowing truncates. Compare after the independent
# resource-format quantisation, rather than relaxing the PSNR gate.
bits = reference.reshape(-1, 4).view(np.uint16).copy()
bits[bits & 0x8000 != 0] = 0
bits[:, :2] &= 0x7ff0
bits[:, 2] &= 0x7fe0
reference = bits.view(np.float16).reshape(-1)
elif args.output_storage == 'bgra8':
reference = (np.rint(np.clip(reference.astype(np.float32), 0, 1) * 255) / 255).astype(np.float16)
x = reference.reshape(-1, 4)[:, :3].astype(np.float64)
y = output.reshape(-1, 4)[:, :3].astype(np.float64)
error = np.abs(x - y)
rms = float(np.sqrt(np.mean(error ** 2)))
peak = max(1., float(np.abs(x).max()))
psnr = math.inf if rms == 0 else 20 * math.log10(peak / rms)
outliers = float(np.mean(error > .01 * peak))
print(f'FRAME_REFERENCE name={path.name} max_abs={error.max():.9g} '
f'max_rel={(error / np.maximum(np.abs(x), .001)).max():.9g} rms={rms:.9g} psnr_db={psnr:.6g} '
f'fraction_over_1pct={outliers:.9g}')
if (args.exact and np.any(error != 0)) or psnr < args.min_psnr or error.max() > .025 * peak or outliers > .0001:
raise RuntimeError(f'Complete pipeline mismatch in {path.name}')
print(f'PASS FRAME_REFERENCE frames={len(frames)} finite=1 min_psnr_db={args.min_psnr}')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--reference', type=pathlib.Path, required=True)
parser.add_argument('--actual', type=pathlib.Path, required=True)
parser.add_argument('--width', type=int, default=512)
parser.add_argument('--height', type=int, default=288)
parser.add_argument('--min-psnr', type=float, default=60.)
parser.add_argument('--output-storage', choices=('rgba16f', 'r11g11b10f', 'bgra8'), default='rgba16f')
parser.add_argument('--exact', action='store_true')
args = parser.parse_args()
try:
main(args)
except Exception as error:
print(f'FAIL FRAME_REFERENCE {error}', file=sys.stderr)
raise SystemExit(5)
tools/windows/generate_d3d12_hooks.py
-137
View File
@@ -1,137 +0,0 @@
"""Generate typed command forwarding from public MinGW D3D12 SDK declarations.
Run with --header <SDK/include/d3d12.h> --output d3d12_command_hooks.generated.h.
Every recording method through CommandList10 is forwarded. State setters copy
their host arguments; COM object arguments retain a reference. SDK slot numbers
are derived from the public interface inheritance, never a driver binary.
"""
import argparse
import pathlib
import re
# Public additions verified against Microsoft's DirectX-Headers adbd6f3b.
# The old MinGW SDK has the needed scalar types but lacks the work-graph
# descriptor names; NewProgramDescription is a typed public-ABI mirror.
NEWER = [
(8, 'OMSetFrontAndBackStencilRef', 'UINT front_stencil, UINT back_stencil'),
(9, 'RSSetDepthBias', 'FLOAT depth_bias, FLOAT clamp, FLOAT slope'),
(9, 'IASetIndexBufferStripCutValue', 'D3D12_INDEX_BUFFER_STRIP_CUT_VALUE value'),
(10, 'SetProgram', 'const NewProgramDescription* desc'),
(10, 'DispatchGraph', 'const void* desc'),
]
POINTER_COUNTS = {
'RSSetViewports': {'viewports': 'viewport_count'},
'RSSetScissorRects': {'rects': 'rect_count'},
'OMSetBlendFactor': {'blend_factor': '4'},
'SetDescriptorHeaps': {'heaps': 'heap_count'},
'SetComputeRoot32BitConstants': {'data': 'constant_count'},
'SetGraphicsRoot32BitConstants': {'data': 'constant_count'},
'IASetIndexBuffer': {'view': '1'},
'IASetVertexBuffers': {'views': 'view_count'},
'SOSetTargets': {'views': 'view_count'},
'OMSetRenderTargets': {'render_target_descriptors': '(single_descriptor_handle ? 1 : render_target_descriptor_count)', 'depth_stencil_descriptor': '1'},
'SetSamplePositions': {'sample_positions': 'sample_count * pixel_count'},
'RSSetShadingRate': {'combiners': '2'},
}
SPECIAL = {'Close', 'Reset', 'ClearState', 'ResourceBarrier', 'Barrier',
'BeginEvent', 'EndEvent', 'BeginRenderPass', 'EndRenderPass',
'BeginQuery', 'EndQuery', 'ExecuteBundle', 'ExecuteIndirect', 'SetPredication'}
def generate(source):
classes = []
for match in re.finditer(r'\n(ID3D12GraphicsCommandList\d*) : public \w+\s*\{(.*?)\n\};', source, re.S):
name, body = match.groups()
methods = re.findall(r'virtual\s+(.*?)\s+STDMETHODCALLTYPE\s+(\w+)\s*\((.*?)\)\s*= 0;', body, re.S)
classes.append((int(name[len('ID3D12GraphicsCommandList'):] or '0'), name, methods))
if [version for version, _, _ in sorted(classes)] != list(range(8)):
raise RuntimeError('Expected exactly the public CommandList0..7 interfaces; review a changed SDK')
lines = ['// Generated by generate_d3d12_hooks.py from the public SDK. Do not edit.', '']
registrations = []
slot = 9 # IUnknown(3), ID3D12Object(4), DeviceChild(1), CommandList(1).
for version, interface, methods in sorted(classes):
for result, name, arguments in methods:
# MinGW WINBOOL and Microsoft SDK BOOL have the same ABI.
result = ' '.join(result.split()).replace('WINBOOL', 'BOOL')
arguments = ' '.join(arguments.split()).replace('WINBOOL', 'BOOL')
declarations = [arg.strip() for arg in arguments.split(',')] if arguments else []
names = []
for declaration in declarations:
match = re.search(r'(\w+)\s*(?:\[\d+\])?$', declaration)
if not match:
raise RuntimeError(f'Unparsed SDK argument: {name}: {declaration}')
names.append(match[1])
signature = ', '.join(['ID3D12GraphicsCommandList* self'] + declarations)
call = ', '.join(['access.target()'] + names)
lines.append(f'using Fn_{name} = {result}(STDMETHODCALLTYPE*)({signature});')
lines.append(f'static Fn_{name} original_{name} = nullptr;')
if name in SPECIAL:
lines.append(f'static {result} STDMETHODCALLTYPE hook_{name}({signature});')
else:
lines += [f'static {result} STDMETHODCALLTYPE hook_{name}({signature}) {{', ' Access access(self);']
stateful = name.startswith(('Set', 'IASet', 'RSSet', 'OMSet', 'SOSet')) and name != 'SetMarker'
if stateful:
captured = []
for declaration, argument in zip(declarations, names):
pointer = '*' in declaration or '[' in declaration
if pointer and 'ID3D12' in declaration:
if declaration.count('*') > 1:
count = POINTER_COUNTS.get(name, {}).get(argument)
if count is None:
raise RuntimeError(f'No COM array copy rule: {name}: {declaration}')
lines.append(f' auto saved_{argument} = retain_values({argument}, {count});')
captured.append(f'saved_{argument}.raw.data()')
else:
lines.append(f' auto saved_{argument} = retain({argument});')
captured.append(f'saved_{argument}.Get()')
elif pointer:
count = POINTER_COUNTS.get(name, {}).get(argument)
if count is None:
raise RuntimeError(f'No deep-copy rule for {name}: {declaration}')
value = f'static_cast<const UINT*>({argument})' if argument == 'data' else argument
lines.append(f' auto saved_{argument} = copy_values({value}, {count});')
captured.append(f'values_pointer(saved_{argument})')
else:
captured.append(argument)
lines += [' if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) mutable {',
f' original_{name}(' + ', '.join(['target'] + captured) + ');', ' });']
lines += [' access.driver_call();', f' return original_{name}({call});', '}']
registrations.append(f' attach_method(table[{slot}], reinterpret_cast<void*>(hook_{name}), reinterpret_cast<void**>(&original_{name}), "{name}");')
registrations[-1] = f' if (version >= {version})' + registrations[-1][3:]
slot += 1
lines.append('')
for version, name, arguments in NEWER:
declarations = arguments.split(', ')
names = [re.search(r'(\w+)$', argument)[1] for argument in declarations]
signature = ', '.join(['ID3D12GraphicsCommandList* self'] + declarations)
lines += [f'using Fn_{name} = void(STDMETHODCALLTYPE*)({signature});', f'static Fn_{name} original_{name} = nullptr;']
if name == 'SetProgram':
lines.append(f'static void STDMETHODCALLTYPE hook_{name}({signature});')
else:
lines += [f'static void STDMETHODCALLTYPE hook_{name}({signature}) {{', ' Access access(self);']
if name != 'DispatchGraph':
lines += [' if (access.recording) access.recording->state.emplace_back([=](ID3D12GraphicsCommandList* target) {',
f' original_{name}(' + ', '.join(['target'] + names) + ');', ' });']
lines += [' access.driver_call();', f' original_{name}(' + ', '.join(['access.target()'] + names) + ');', '}']
registrations.append(f' if (version >= {version}) attach_method(table[{slot}], reinterpret_cast<void*>(hook_{name}), reinterpret_cast<void**>(&original_{name}), "{name}");')
slot += 1
lines.append('')
lines += ['static void attach_recording_methods(void** table, unsigned version) {'] + registrations + ['}', '']
device = re.search(r'typedef struct ID3D12DeviceVtbl \{(.*?)\n\} ID3D12DeviceVtbl;', source, re.S)[1]
fields = list(dict.fromkeys(re.findall(r'\(STDMETHODCALLTYPE \*(\w+)\)', device)))
lines.append(f'static constexpr unsigned device_signature_slot = {fields.index("CreateCommandSignature")};')
lines.append(f'static constexpr unsigned device_command_list_slot = {fields.index("CreateCommandList")};')
queue = re.search(r'typedef struct ID3D12CommandQueueVtbl \{(.*?)\n\} ID3D12CommandQueueVtbl;', source, re.S)[1]
queue_fields = list(dict.fromkeys(re.findall(r'\(STDMETHODCALLTYPE \*(\w+)\)', queue)))
for name in ('Signal', 'Wait'):
lines.append(f'static constexpr unsigned queue_{name.lower()}_slot = {queue_fields.index(name)};')
return '\n'.join(lines)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--header', type=pathlib.Path, required=True)
parser.add_argument('--output', type=pathlib.Path, required=True)
args = parser.parse_args()
args.output.write_text(generate(args.header.read_text()), encoding='utf-8')
tools/windows/gfx12_wmma_probe.cpp
-131
View File
@@ -1,131 +0,0 @@
#include "hip_api.h"
#include <array>
#include <cmath>
#include <limits>
namespace {
uint16_t half_integer(int value)
{
const unsigned n = static_cast<unsigned>(std::abs(value));
if (n > 3u) throw std::runtime_error("test half integer out of range");
const uint16_t bits[] = {0u, 0x3c00u, 0x4000u, 0x4200u};
return static_cast<uint16_t>(bits[n] | (value < 0 ? 0x8000u : 0u));
}
struct Allocation {
d4r::diag::HipApi& hip;
void* pointer = nullptr;
Allocation(d4r::diag::HipApi& api, size_t bytes) : hip(api) {
hip.check(hip.hipMalloc(&pointer, bytes), "hipMalloc(WMMA)");
}
~Allocation() { if (pointer) (void)hip.hipFree(pointer); }
};
}
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.module.empty()) throw std::runtime_error("--module is required");
HipApi hip(args.hip_root);
hipDeviceProp_t properties{};
hip.select_architecture(args.device, properties);
hipModule_t module = nullptr;
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(WMMA)");
struct ModuleCleanup {
HipApi& hip; hipModule_t module;
~ModuleCleanup() { if (module) (void)hip.hipModuleUnload(module); }
} cleanup{hip, module};
hipFunction_t raw_kernel = nullptr, adapter_kernel = nullptr, upstream_kernel = nullptr, swap_kernel = nullptr;
hip.check(hip.hipModuleGetFunction(&raw_kernel, module, "d4r_wmma_gfx1201"),
"hipModuleGetFunction(WMMA)");
hip.check(hip.hipModuleGetFunction(&adapter_kernel, module, "d4r_wmma_legacy_contract"),
"hipModuleGetFunction(WMMA adapter)");
hip.check(hip.hipModuleGetFunction(&upstream_kernel, module, "d4r_wmma_upstream_contract"),
"hipModuleGetFunction(WMMA upstream)");
hip.check(hip.hipModuleGetFunction(&swap_kernel, module, "d4r_wmma_swap_test"),
"hipModuleGetFunction(WMMA swap)");
constexpr size_t elements = 16u * 16u;
std::array<uint16_t, elements> a{}, b{};
std::array<float, elements> initial{}, output{};
Allocation da(hip, sizeof(a)), db(hip, sizeof(b));
Allocation dc(hip, sizeof(initial)), dout(hip, sizeof(output));
void* swap_out = dout.pointer;
void* swap_params[] = {&swap_out};
hip.check(hip.hipModuleLaunchKernel(swap_kernel, 1, 1, 1, 32, 1, 1, 0,
nullptr, swap_params, nullptr), "hipModuleLaunchKernel(WMMA swap)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(WMMA swap)");
hip.check(hip.hipMemcpy(output.data(), dout.pointer, sizeof(output), hipMemcpyDeviceToHost), "hipMemcpy(WMMA swap)");
for (unsigned lane = 0; lane < 32u; ++lane) {
if (output[lane] != static_cast<float>(lane ^ 16u)) {
std::fprintf(stderr, "FAIL WMMA swap lane=%u expected=%u actual=%g\n",
lane, lane ^ 16u, output[lane]);
return 5;
}
if (output[32u + lane] != static_cast<float>(lane)) {
std::fprintf(stderr, "FAIL WMMA lane-id lane=%u mbcnt=%g\n", lane, output[32u + lane]);
return 5;
}
}
float max_absolute_error = 0.0f;
float max_relative_error = 0.0f;
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
const unsigned steps = 1u + (iteration & 1u);
for (unsigned row = 0; row < 16u; ++row) {
for (unsigned k = 0; k < 16u; ++k) {
a[row * 16u + k] = half_integer(static_cast<int>((row * 3u + k * 5u + iteration) % 7u) - 3);
b[row * 16u + k] = half_integer(static_cast<int>((row * 5u + k * 2u + iteration * 3u) % 7u) - 3);
initial[row * 16u + k] = static_cast<float>(static_cast<int>((row * 11u + k * 3u + iteration) % 17u) - 8) * 0.125f;
}
}
hip.check(hip.hipMemcpy(da.pointer, a.data(), sizeof(a), hipMemcpyHostToDevice), "hipMemcpy(A)");
hip.check(hip.hipMemcpy(db.pointer, b.data(), sizeof(b), hipMemcpyHostToDevice), "hipMemcpy(B)");
hip.check(hip.hipMemcpy(dc.pointer, initial.data(), sizeof(initial), hipMemcpyHostToDevice), "hipMemcpy(C)");
void* a_arg = da.pointer;
void* b_arg = db.pointer;
void* c_arg = dc.pointer;
void* output_arg = dout.pointer;
unsigned steps_arg = steps;
void* params[] = {&a_arg, &b_arg, &c_arg, &output_arg, &steps_arg};
const hipFunction_t kernels[] = {raw_kernel, adapter_kernel, upstream_kernel};
const char* modes[] = {"raw", "legacy_adapter", "upstream_layout"};
for (unsigned mode = 0; mode < 3u; ++mode) {
output.fill(std::numeric_limits<float>::quiet_NaN());
hip.check(hip.hipMemcpy(dout.pointer, output.data(), sizeof(output), hipMemcpyHostToDevice), "hipMemcpy(output sentinel)");
hip.check(hip.hipModuleLaunchKernel(kernels[mode], 1, 1, 1, 32, 1, 1, 0, nullptr,
params, nullptr), modes[mode]);
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(WMMA)");
hip.check(hip.hipMemcpy(output.data(), dout.pointer, sizeof(output), hipMemcpyDeviceToHost), "hipMemcpy(output)");
for (unsigned row = 0; row < 16u; ++row) {
for (unsigned col = 0; col < 16u; ++col) {
int dot = 0;
for (unsigned k = 0; k < 16u; ++k) {
const int av = static_cast<int>((row * 3u + k * 5u + iteration) % 7u) - 3;
const int bv = static_cast<int>((col * 5u + k * 2u + iteration * 3u) % 7u) - 3;
dot += av * bv;
}
const float expected = initial[row * 16u + col] + static_cast<float>(static_cast<int>(steps) * dot);
const float got = output[row * 16u + col];
const float absolute = std::fabs(got - expected);
const float relative = absolute / std::max(1.0f, std::fabs(expected));
if (!std::isfinite(got) || absolute > 0.0001f) {
std::fprintf(stderr, "FAIL WMMA mode=%s iteration=%u row=%u col=%u steps=%u expected=%.9g actual=%.9g abs=%.9g\n",
modes[mode], iteration, row, col, steps, expected, got, absolute);
return 5;
}
max_absolute_error = std::max(max_absolute_error, absolute);
max_relative_error = std::max(max_relative_error, relative);
}
}
}
}
std::printf("PASS GFX12_WMMA architecture=%s modes=raw,legacy_adapter,upstream_layout iterations=%u max_abs=%.9g max_rel=%.9g\n", d4r::diag::HipApi::target_arch(),
args.iterations, max_absolute_error, max_relative_error);
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "FAIL GFX12_WMMA %s\n", error.what());
loaded_modules();
return 4;
}
}
tools/windows/gpu-target.json.in
-6
View File
@@ -1,6 +0,0 @@
{
"schema": 1,
"architecture": "@D4R_GPU_ARCH@",
"wavefrontSize": 32,
"validation": "Compilation is not hardware validation; run the GPU diagnostics."
}
tools/windows/gpu-targets.json
-16
View File
@@ -1,16 +0,0 @@
{
"schema": 1,
"targets": [
{"architecture":"gfx1100","elfMachine":65,"isaMajor":11,"family":"RDNA3"},
{"architecture":"gfx1101","elfMachine":70,"isaMajor":11,"family":"RDNA3"},
{"architecture":"gfx1102","elfMachine":71,"isaMajor":11,"family":"RDNA3"},
{"architecture":"gfx1103","elfMachine":68,"isaMajor":11,"family":"RDNA3"},
{"architecture":"gfx1150","elfMachine":67,"isaMajor":11,"family":"RDNA3.5"},
{"architecture":"gfx1151","elfMachine":74,"isaMajor":11,"family":"RDNA3.5"},
{"architecture":"gfx1152","elfMachine":85,"isaMajor":11,"family":"RDNA3.5"},
{"architecture":"gfx1153","elfMachine":88,"isaMajor":11,"family":"RDNA3.5"},
{"architecture":"gfx1154","elfMachine":87,"isaMajor":11,"family":"RDNA3.5"},
{"architecture":"gfx1200","elfMachine":72,"isaMajor":12,"family":"RDNA4"},
{"architecture":"gfx1201","elfMachine":78,"isaMajor":12,"family":"RDNA4"}
]
}
tools/windows/gpu_inventory.cpp
-57
View File
@@ -1,57 +0,0 @@
#include "hip_api.h"
#include <fstream>
#include <sstream>
#include <iomanip>
static std::string json_string(const std::string& value) {
std::ostringstream out; out << '"';
for (unsigned char c : value) {
if (c == '"' || c == '\\') out << '\\' << char(c);
else if (c < 32) out << "\\u" << std::hex << std::setw(4) << std::setfill('0') << unsigned(c);
else out << char(c);
}
out << '"'; return out.str();
}
int main(int argc, char** argv) {
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.output_dir.empty()) throw std::runtime_error("--output-dir is required");
for (const char* name : {"HSA_OVERRIDE_GFX_VERSION", "HSA_OVERRIDE_GFX_VERSION_0"})
if (std::getenv(name)) throw std::runtime_error(std::string("Unset architecture override ") + name);
HipApi hip(args.hip_root);
hip.check(hip.hipInit(0), "hipInit(inventory)");
int count=0, version=0, driver=0;
hip.check(hip.hipRuntimeGetVersion(&version), "hipRuntimeGetVersion");
hip.check(hip.hipDriverGetVersion(&driver), "hipDriverGetVersion");
std::printf("GPU_INVENTORY_RUNTIME runtime=%d driver_api=%d headers=%d.%d.%d sizeof(properties)=%zu\n",
version,driver,HIP_VERSION_MAJOR,HIP_VERSION_MINOR,HIP_VERSION_PATCH,sizeof(hipDeviceProp_t));
hip.check(hip.hipGetDeviceCount(&count), "hipGetDeviceCount");
std::ostringstream json;
json << "{\"schema\":1,\"runtimeVersion\":" << version << ",\"driverApiVersion\":" << driver << ",\"compiledTarget\":"
<< json_string(D4R_TARGET_ARCH) << ",\"gpuWorkExecuted\":false,\"devices\":[";
for (int i=0; i<count; ++i) {
hipDeviceProp_t p{}; hip.check(hip.hipGetDeviceProperties(&p,i), "hipGetDevicePropertiesR0600");
const auto arch=gpu_architecture(p.gcnArchName);
std::printf("GPU_INVENTORY ordinal=%d name=%s gcnArchName=%s wave=%d build_eligible=%d\n",
i,p.name,p.gcnArchName,p.warpSize,supported_gpu_target(arch));
if (i) json << ',';
json << "{\"ordinal\":" << i << ",\"name\":" << json_string(p.name)
<< ",\"gcnArchName\":" << json_string(p.gcnArchName) << ",\"architecture\":" << json_string(arch)
<< ",\"wavefrontSize\":" << p.warpSize << ",\"vramBytes\":" << p.totalGlobalMem
<< ",\"buildEligible\":" << (supported_gpu_target(arch) ? "true" : "false") << '}';
}
json << "]}\n";
std::filesystem::create_directories(wide(args.output_dir));
std::ofstream file(std::filesystem::path(wide(args.output_dir))/L"gpu-inventory.json",std::ios::binary);
file << json.str(); file.close();
if (!file) throw std::runtime_error("Cannot write GPU inventory");
if (!count) throw std::runtime_error("HIP reports no devices");
loaded_modules();
std::printf("PASS GPU_INVENTORY devices=%d gpu_work_executed=0\n",count);
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr,"FAIL GPU_INVENTORY %s\n",error.what()); loaded_modules(); return 4;
}
}
tools/windows/gpu_target.h
-40
View File
@@ -1,40 +0,0 @@
#pragma once
#include <array>
#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <string>
#include "gpu_targets.generated.h"
#ifndef D4R_TARGET_ARCH
#define D4R_TARGET_ARCH "gfx1201"
#endif
namespace d4r::diag {
inline bool supported_gpu_target(const std::string& target) {
for (const auto& row : gpu_targets) if (target == row.architecture) return true;
return false;
}
inline std::string gpu_architecture(const std::string& name) {
return name.substr(0, name.find(':'));
}
inline void require_gpu_target(const std::string& actual, const std::string& expected) {
if (!supported_gpu_target(expected) || gpu_architecture(actual) != expected)
throw std::runtime_error("GPU architecture mismatch: detected " + actual + ", package requires " + expected);
}
inline std::string code_object_target(const std::filesystem::path& path) {
// AMDGPU ELF e_flags machine IDs from LLVM's documented code-object ABI.
// Never infer compatibility from a filename or rewrite the ELF target.
std::ifstream file(path, std::ios::binary);
std::array<unsigned char, 64> header{};
if (!file.read(reinterpret_cast<char*>(header.data()), header.size()) ||
header[0] != 0x7f || header[1] != 'E' || header[2] != 'L' || header[3] != 'F' ||
header[4] != 2 || header[5] != 1 || header[18] != 0xe0 || header[19] != 0)
throw std::runtime_error("Invalid AMDGPU ELF64 code object: " + path.string());
for (const auto& row : gpu_targets) if (header[48] == row.elf_machine) return row.architecture;
throw std::runtime_error("Unsupported AMDGPU code-object target: " + path.string());
}
inline void require_code_object_target(const std::filesystem::path& path, const std::string& expected) {
require_gpu_target(code_object_target(path), expected);
}
}
tools/windows/gpu_target_probe.cpp
-32
View File
@@ -1,32 +0,0 @@
#include "gpu_target.h"
#include <cstdio>
int main(int argc, char** argv) {
using namespace d4r::diag;
try {
unsigned rejected = 0, objects = 0;
for (const auto& info : gpu_targets) {
const char* target = info.architecture;
require_gpu_target(target, target);
require_gpu_target(std::string(target) + ":xnack-:sramecc-", target);
for (const char* invalid : {"gfx1030", "gfx1104", "gfx1155", "gfx1170", "gfx12000", "gfx1202", "gfx1250", ""}) {
try { require_gpu_target(invalid, target); }
catch (const std::exception&) { ++rejected; continue; }
throw std::runtime_error("Invalid GPU target accepted");
}
}
for (int i = 1; i < argc; ++i) {
require_code_object_target(argv[i], D4R_TARGET_ARCH);
const auto other = std::string(D4R_TARGET_ARCH) == "gfx1201" ? "gfx1200" : "gfx1201";
try { require_code_object_target(argv[i], other); }
catch (const std::exception&) { ++rejected; ++objects; continue; }
throw std::runtime_error("Wrong code-object target accepted");
}
if (!objects) throw std::runtime_error("Supply actual compiled code objects");
std::printf("PASS GPU_TARGET_CONTRACT target=%s objects=%u rejected=%u hardware_executed=0\n",
D4R_TARGET_ARCH, objects, rejected);
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL GPU_TARGET_CONTRACT %s\n", e.what()); return 5;
}
}
tools/windows/gpu_targets.generated.h.in
-7
View File
@@ -1,7 +0,0 @@
#pragma once
#include <array>
namespace d4r::diag {
struct GpuTarget { const char* architecture; unsigned elf_machine; unsigned isa_major; const char* family; };
inline constexpr std::array<GpuTarget, @d4r_target_count@> gpu_targets = {{
@d4r_target_cpp_entries@}};
}
tools/windows/gpu_targets.py
-10
View File
@@ -1,10 +0,0 @@
"""Canonical Windows build target registry, shared with CMake and PowerShell."""
import json
import pathlib
data = json.loads(pathlib.Path(__file__).with_name('gpu-targets.json').read_text())
if data['schema'] != 1:
raise RuntimeError('Unsupported GPU target registry schema')
TARGETS = {row['architecture']: row for row in data['targets']}
if len(TARGETS) != len(data['targets']):
raise RuntimeError('Duplicate GPU target')
tools/windows/gpu_telemetry.c
-149
View File
@@ -1,149 +0,0 @@
// Read-only AMD driver telemetry, separate from the game and HIP runtime.
// Uses the ADLX SDK's C interface; no SDK/sample implementation is vendored.
#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <errno.h>
#include <ADLX.h>
#include <IPerformanceMonitoring.h>
static int number(const char* text, unsigned* value) {
char* end = NULL;
errno = 0;
unsigned long parsed = strtoul(text, &end, 10);
if (!text[0] || *end || text[0] == '-' || errno == ERANGE) return 0;
*value = (unsigned)parsed;
return 1;
}
static void real_value(ADLX_RESULT status, double value) {
if (ADLX_SUCCEEDED(status) && isfinite(value) && value >= 0) printf("%.6f", value);
}
static void int_value(ADLX_RESULT status, int value) {
if (ADLX_SUCCEEDED(status) && value >= 0) printf("%d", value);
}
int main(int argc, char** argv) {
unsigned seconds = 30, interval = 500, target_pid = 0, delay = 0;
const char* gpu_name = "RX 9070 XT";
for (int i = 1; i < argc; ++i) {
if (i + 1 == argc) return 2;
const char* key = argv[i];
const char* value = argv[++i];
unsigned* dest = !strcmp(key, "--seconds") ? &seconds :
!strcmp(key, "--interval-ms") ? &interval :
!strcmp(key, "--target-pid") ? &target_pid :
!strcmp(key, "--delay-ms") ? &delay : NULL;
if (dest) { if (!number(value, dest)) return 2; }
else if (!strcmp(key, "--gpu-name")) gpu_name = value;
else return 2;
}
if (seconds < 1 || seconds > 300 || interval < 100 || interval > 5000 || delay > 300000) return 2;
int exit_code = 1, initialized = 0;
HMODULE dll = NULL;
HANDLE target = NULL;
IADLXSystem* system = NULL; // Owned by ADLXInitialize/ADLXTerminate.
IADLXGPUList* gpus = NULL;
IADLXGPU* selected = NULL;
IADLXPerformanceMonitoringServices* service = NULL;
ADLXTerminate_Fn terminate = NULL;
if (target_pid) {
target = OpenProcess(SYNCHRONIZE, FALSE, target_pid);
if (!target) { fprintf(stderr, "OpenProcess error=%lu\n", GetLastError()); goto done; }
}
// Load the installed driver library, never a DLL from the game/current dir.
dll = LoadLibraryExW(L"amdadlx64.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32);
if (!dll) { fprintf(stderr, "ADLX driver DLL unavailable error=%lu\n", GetLastError()); goto done; }
ADLXQueryFullVersion_Fn query_version = (ADLXQueryFullVersion_Fn)GetProcAddress(dll, ADLX_QUERY_FULL_VERSION_FUNCTION_NAME);
ADLXInitialize_Fn initialize = (ADLXInitialize_Fn)GetProcAddress(dll, ADLX_INIT_FUNCTION_NAME);
terminate = (ADLXTerminate_Fn)GetProcAddress(dll, ADLX_TERMINATE_FUNCTION_NAME);
adlx_uint64 runtime_version = 0;
if (!query_version || !initialize || !terminate || ADLX_FAILED(query_version(&runtime_version))) goto done;
// Only use the base interfaces. Request no version newer than either the
// installed runtime or these headers (e.g. SDK 2.0 with driver ADLX 1.5).
adlx_uint64 requested = runtime_version < ADLX_FULL_VERSION ? runtime_version : ADLX_FULL_VERSION;
ADLX_RESULT result = initialize(requested, &system);
fprintf(stderr, "ADLX runtime=%llu requested=%llu initialize=%d interval_ms=%u target_pid=%u\n",
(unsigned long long)runtime_version, (unsigned long long)requested, result, interval, target_pid);
if (ADLX_FAILED(result) || !system) goto done;
initialized = 1;
result = system->pVtbl->GetGPUs(system, &gpus);
if (ADLX_FAILED(result) || !gpus) goto done;
unsigned matches = 0;
for (adlx_uint i = gpus->pVtbl->Begin(gpus); i < gpus->pVtbl->End(gpus); ++i) {
IADLXGPU* gpu = NULL;
if (ADLX_FAILED(gpus->pVtbl->At_GPUList(gpus, i, &gpu)) || !gpu) goto done;
const char* name = NULL; const char* pnp = NULL; adlx_int id = -1;
gpu->pVtbl->Name(gpu, &name);
gpu->pVtbl->PNPString(gpu, &pnp);
gpu->pVtbl->UniqueId(gpu, &id);
fprintf(stderr, "ADLX GPU index=%u id=%d name=%s pnp=%s\n", i, id, name ? name : "unknown", pnp ? pnp : "unknown");
if (name && strstr(name, gpu_name)) {
++matches;
if (!selected) { selected = gpu; gpu = NULL; }
}
if (gpu) gpu->pVtbl->Release(gpu);
}
if (matches != 1) { fprintf(stderr, "Expected one GPU matching '%s', found %u\n", gpu_name, matches); goto done; }
result = system->pVtbl->GetPerformanceMonitoringServices(system, &service);
if (ADLX_FAILED(result) || !service) goto done;
LARGE_INTEGER frequency, qpc;
QueryPerformanceFrequency(&frequency);
// Preserve the driver's timestamp verbatim: the installed ADLX 1.5 reports
// uptime here despite SDK documentation describing epoch time. Use our
// independent QPC/UTC stamps for correlation with PresentMon.
printf("qpc_ms,host_unix_ms,sensor_timestamp_ms,gpu_usage_pct,gpu_clock_mhz,vram_clock_mhz,gpu_power_w,board_power_w,temperature_c,hotspot_c,poll_ms\n");
fflush(stdout);
if (delay) Sleep(delay);
ULONGLONG start = GetTickCount64();
unsigned samples = 0;
while (GetTickCount64() - start < (ULONGLONG)seconds * 1000) {
if (target && WaitForSingleObject(target, 0) == WAIT_OBJECT_0) break;
LARGE_INTEGER before, after;
QueryPerformanceCounter(&before);
IADLXGPUMetrics* metrics = NULL;
result = service->pVtbl->GetCurrentGPUMetrics(service, selected, &metrics);
QueryPerformanceCounter(&after);
if (ADLX_FAILED(result) || !metrics) { fprintf(stderr, "GetCurrentGPUMetrics result=%d\n", result); goto done; }
FILETIME time; ULARGE_INTEGER unix_time;
GetSystemTimePreciseAsFileTime(&time);
unix_time.LowPart = time.dwLowDateTime; unix_time.HighPart = time.dwHighDateTime;
QueryPerformanceCounter(&qpc);
adlx_int64 sensor_time = 0;
printf("%.6f,%llu,", qpc.QuadPart * 1000. / frequency.QuadPart,
(unsigned long long)((unix_time.QuadPart - 116444736000000000ULL) / 10000));
if (ADLX_SUCCEEDED(metrics->pVtbl->TimeStamp(metrics, &sensor_time))) printf("%lld", (long long)sensor_time);
double real = -1; int integer = -1;
printf(","); result = metrics->pVtbl->GPUUsage(metrics, &real); real_value(result, real);
printf(","); result = metrics->pVtbl->GPUClockSpeed(metrics, &integer); int_value(result, integer);
printf(","); result = metrics->pVtbl->GPUVRAMClockSpeed(metrics, &integer); int_value(result, integer);
printf(","); result = metrics->pVtbl->GPUPower(metrics, &real); real_value(result, real);
printf(","); result = metrics->pVtbl->GPUTotalBoardPower(metrics, &real); real_value(result, real);
printf(","); result = metrics->pVtbl->GPUTemperature(metrics, &real); real_value(result, real);
printf(","); result = metrics->pVtbl->GPUHotspotTemperature(metrics, &real); real_value(result, real);
printf(",%.6f\n", (after.QuadPart - before.QuadPart) * 1000. / frequency.QuadPart);
metrics->pVtbl->Release(metrics);
++samples; fflush(stdout);
ULONGLONG elapsed = GetTickCount64() - start;
ULONGLONG next = (ULONGLONG)samples * interval;
if (next > elapsed) Sleep((DWORD)(next - elapsed));
}
fprintf(stderr, "ADLX collected=%u read_only=1\n", samples);
exit_code = samples ? 0 : 1;
done:
// All acquired interfaces must be released before unloading ADLX.
if (service) service->pVtbl->Release(service);
if (selected) selected->pVtbl->Release(selected);
if (gpus) gpus->pVtbl->Release(gpus);
if (initialized) {
ADLX_RESULT status = terminate();
fprintf(stderr, "ADLX terminate=%d\n", status);
if (ADLX_FAILED(status)) exit_code = 1;
}
if (dll) FreeLibrary(dll);
if (target) CloseHandle(target);
return exit_code;
}
tools/windows/hip_api.h
-81
View File
@@ -1,81 +0,0 @@
#pragma once
#include "diagnostic.h"
#include "gpu_target.h"
#include <hip/hip_runtime_api.h>
namespace d4r::diag {
struct HipApi {
bool verbose = true; // Probes log every call; the game can suppress successful API calls.
SearchDirectory search;
Library library;
#define D4R_HIP_FUNCTION(name) decltype(&::name) name = library.symbol<decltype(&::name)>(#name)
D4R_HIP_FUNCTION(hipInit);
D4R_HIP_FUNCTION(hipGetDeviceCount);
// hipGetDeviceProperties is an SDK macro; the ABI-versioned export is intentional.
decltype(&::hipGetDeviceProperties) hipGetDeviceProperties =
library.symbol<decltype(&::hipGetDeviceProperties)>("hipGetDevicePropertiesR0600");
D4R_HIP_FUNCTION(hipSetDevice);
D4R_HIP_FUNCTION(hipRuntimeGetVersion);
D4R_HIP_FUNCTION(hipDriverGetVersion);
D4R_HIP_FUNCTION(hipGetErrorName);
D4R_HIP_FUNCTION(hipGetErrorString);
using MallocFn = hipError_t (*)(void**, size_t);
MallocFn hipMalloc = library.symbol<MallocFn>("hipMalloc");
D4R_HIP_FUNCTION(hipFree);
D4R_HIP_FUNCTION(hipMemcpy);
D4R_HIP_FUNCTION(hipMemGetInfo);
D4R_HIP_FUNCTION(hipModuleLoad);
D4R_HIP_FUNCTION(hipModuleUnload);
D4R_HIP_FUNCTION(hipModuleGetFunction);
D4R_HIP_FUNCTION(hipModuleGetGlobal);
D4R_HIP_FUNCTION(hipModuleLaunchKernel);
D4R_HIP_FUNCTION(hipDeviceSynchronize);
D4R_HIP_FUNCTION(hipStreamCreateWithFlags);
D4R_HIP_FUNCTION(hipStreamSynchronize);
D4R_HIP_FUNCTION(hipStreamDestroy);
#undef D4R_HIP_FUNCTION
explicit HipApi(const std::string& root) :
search(std::filesystem::path(wide(root)) / L"bin"),
library(std::filesystem::path(wide(root)) / L"bin" / L"amdhip64_7.dll") {}
void check(hipError_t result, const char* call) const {
if (verbose || result != hipSuccess) std::printf("HIP %s -> %d (%s)\n", call, static_cast<int>(result), hipGetErrorName(result));
if (result != hipSuccess) throw std::runtime_error(std::string(call) + ": " + hipGetErrorString(result));
}
static const char* target_arch() { return D4R_TARGET_ARCH; }
int select_architecture(int requested, hipDeviceProp_t& selected, const void* required_luid = nullptr,
bool allow_no_match = false) const {
// Architecture spoofing invalidates a gfx12 correctness test.
for (const char* name : {"HSA_OVERRIDE_GFX_VERSION", "HSA_OVERRIDE_GFX_VERSION_0"})
if (std::getenv(name)) throw std::runtime_error(std::string("Unset architecture override ") + name);
check(hipInit(0), "hipInit");
int version = 0, driver = 0, count = 0;
check(hipRuntimeGetVersion(&version), "hipRuntimeGetVersion");
check(hipDriverGetVersion(&driver), "hipDriverGetVersion");
std::printf("HIP runtime=%d driver_api=%d headers=%d.%d.%d sizeof(properties)=%zu\n",
version, driver, HIP_VERSION_MAJOR, HIP_VERSION_MINOR, HIP_VERSION_PATCH, sizeof(hipDeviceProp_t));
check(hipGetDeviceCount(&count), "hipGetDeviceCount");
int found = -1;
for (int i = 0; i < count; ++i) {
hipDeviceProp_t props{};
check(hipGetDeviceProperties(&props, i), "hipGetDevicePropertiesR0600");
std::printf("GPU ordinal=%d name=%s gcnArchName=%s wave=%d vram=%zu pci=%04x:%02x:%02x\n",
i, props.name, props.gcnArchName, props.warpSize, props.totalGlobalMem,
props.pciDomainID, props.pciBusID, props.pciDeviceID);
const std::string arch = props.gcnArchName;
if (gpu_architecture(arch) == target_arch() && (requested < 0 || requested == i) && found < 0 &&
(!required_luid || std::memcmp(props.luid, required_luid, sizeof(props.luid)) == 0)) {
found = i;
selected = props;
}
}
if (found < 0 && allow_no_match) return -1; // Capability enumeration can include Intel/WARP adapters.
if (found < 0) throw std::runtime_error(std::string("No selected ") + target_arch() +
" device for this adapter; use the package matching the actual HIP architecture (see GPU log)");
require_gpu_target(selected.gcnArchName, target_arch());
check(hipSetDevice(found), "hipSetDevice");
std::printf("SELECTED HIP ordinal=%d architecture=%s validation=%s\n", found, target_arch(),
std::string(target_arch()) == "gfx1201" ? "previously_tested_target" : "hardware_unverified_target");
return found;
}
};
}
tools/windows/hip_gfx1201_probe.cpp
-51
View File
@@ -1,51 +0,0 @@
#include "hip_api.h"
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.module.empty()) throw std::runtime_error("--module is required");
HipApi api(args.hip_root);
hipDeviceProp_t props{};
api.select_architecture(args.device, props);
loaded_modules();
size_t free = 0, total = 0;
api.check(api.hipMemGetInfo(&free, &total), "hipMemGetInfo");
std::printf("VRAM free=%zu total=%zu\n", free, total);
hipModule_t module = nullptr;
api.check(api.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(target)");
struct ModuleCleanup { HipApi& api; hipModule_t m; ~ModuleCleanup() { if (m) (void)api.hipModuleUnload(m); } } cleanup{api, module};
hipFunction_t kernel = nullptr;
api.check(api.hipModuleGetFunction(&kernel, module, "d4r_probe_pattern"), "hipModuleGetFunction");
constexpr uint32_t count = 4099, guard = 61;
std::vector<uint32_t> result(count + guard, 0xdeadbeefu);
const size_t bytes = result.size() * sizeof(uint32_t);
void* allocation = nullptr;
api.check(api.hipMalloc(&allocation, bytes), "hipMalloc(VRAM)");
struct AllocCleanup { HipApi& api; void* p; ~AllocCleanup() { if (p) (void)api.hipFree(p); } } allocation_cleanup{api, allocation};
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
uint32_t n = count - iteration % 129u, seed = 0x12345678u + iteration * 31u;
std::fill(result.begin(), result.end(), 0xdeadbeefu);
api.check(api.hipMemcpy(allocation, result.data(), bytes, hipMemcpyHostToDevice), "hipMemcpy(HtoD sentinel)");
void* parameters[] = {&allocation, &n, &seed};
api.check(api.hipModuleLaunchKernel(kernel, (n + 127u) / 128u, 1, 1, 128, 1, 1,
0, nullptr, parameters, nullptr), "hipModuleLaunchKernel");
api.check(api.hipDeviceSynchronize(), "hipDeviceSynchronize");
api.check(api.hipMemcpy(result.data(), allocation, bytes, hipMemcpyDeviceToHost), "hipMemcpy(DtoH)");
if (!verify(result, n, seed)) return 5;
}
// Explicitly validate teardown too; disable RAII cleanup after success.
api.check(api.hipFree(allocation), "hipFree");
allocation_cleanup.p = nullptr;
api.check(api.hipModuleUnload(module), "hipModuleUnload");
cleanup.m = nullptr;
std::printf("PASS HIP architecture=%s iterations=%u guard_verified=1\n", d4r::diag::HipApi::target_arch(), args.iterations);
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL HIP %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/hip_stream_lifecycle_probe.cpp
-60
View File
@@ -1,60 +0,0 @@
#include "hip_api.h"
// Isolate Windows HIP stream lifetime from D3D12 and external-resource imports.
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
hipModule_t module = nullptr;
hipFunction_t kernel = nullptr;
void* output = nullptr;
struct ResourceCleanup {
HipApi& hip; hipModule_t& module; void*& output;
~ResourceCleanup() {
if (output) (void)hip.hipFree(output);
if (module) (void)hip.hipModuleUnload(module);
}
} resources{hip, module, output};
if (!args.module.empty()) {
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad");
hip.check(hip.hipModuleGetFunction(&kernel, module, "d4r_probe_pattern"), "hipModuleGetFunction");
hip.check(hip.hipMalloc(&output, 128 * sizeof(uint32_t)), "hipMalloc");
}
DWORD baseline = 0, count = 0;
for (unsigned i = 0; i < args.iterations; ++i) {
hipStream_t stream = nullptr;
hip.check(hip.hipStreamCreateWithFlags(&stream, hipStreamNonBlocking), "hipStreamCreateWithFlags");
struct StreamCleanup {
HipApi& hip; hipStream_t& stream;
~StreamCleanup() { if (stream) { (void)hip.hipStreamSynchronize(stream); (void)hip.hipStreamDestroy(stream); } }
} cleanup{hip, stream};
if (kernel) {
uint32_t n = 128, seed = i;
void* parameters[] = {&output, &n, &seed};
hip.check(hip.hipModuleLaunchKernel(kernel, 1, 1, 1, 128, 1, 1, 0,
stream, parameters, nullptr), "hipModuleLaunchKernel");
}
hip.check(hip.hipStreamSynchronize(stream), "hipStreamSynchronize");
hip.check(hip.hipStreamDestroy(stream), "hipStreamDestroy");
stream = nullptr;
if (!GetProcessHandleCount(GetCurrentProcess(), &count))
throw std::runtime_error("GetProcessHandleCount failed");
if (i == 1 || (args.iterations == 1 && i == 0)) baseline = count;
std::printf("STREAM_CLEANUP iteration=%u handles=%lu baseline=%lu\n", i, count, baseline);
}
if (output) { hip.check(hip.hipFree(output), "hipFree"); output = nullptr; }
if (module) { hip.check(hip.hipModuleUnload(module), "hipModuleUnload"); module = nullptr; }
if (count > baseline + 4) throw std::runtime_error("HIP stream lifecycle handle growth");
std::printf("PASS HIP_STREAM iterations=%u handles=%lu\n", args.iterations, count);
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL HIP_STREAM %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/k_layer_validate.py
-85
View File
@@ -1,85 +0,0 @@
#!/usr/bin/env python3
"""Compare native K enc1/enc2 launches with the existing numpy reference."""
import argparse
import math
import pathlib
import sys
import numpy as np
SCRIPT = pathlib.Path(__file__).resolve()
REFERENCE = SCRIPT.parent / "reference"
if not (REFERENCE / "pwin_model.py").exists():
REFERENCE = SCRIPT.parents[2] / "kernels" / "tools"
sys.path.insert(0, str(REFERENCE))
from pwin_model import pwin_encoder # noqa: E402
class SyntheticParams:
def __init__(self, folder):
self.input = np.fromfile(folder / "input.bin", dtype=np.uint8)
self.weights = np.fromfile(folder / "weights.bin", dtype=np.uint8)
def i32x2(self, offset):
return (8, 8) if offset == 0 else (0, 0)
def ptr(self, offset):
if offset == 8:
return self.input, 0
if offset == 64:
return self.weights, 0
raise ValueError(f"unsupported synthetic pointer offset {offset}")
def f16(self, offset, byte, count):
if offset != 64:
raise ValueError(f"unsupported synthetic half pointer offset {offset}")
return self.weights[byte:byte + 2 * count].view(np.float16).copy()
def compare(name, actual, expected):
if actual.shape != expected.shape:
raise RuntimeError(f"{name} shape mismatch: {actual.shape} != {expected.shape}")
if not np.isfinite(actual).all() or not np.isfinite(expected).all():
raise RuntimeError(f"{name} contains NaN or Inf")
diff = np.abs(actual.astype(np.float32) - expected.astype(np.float32))
maximum = float(diff.max())
relative = float((diff / np.maximum(np.abs(expected.astype(np.float32)), 1e-3)).max())
rms = float(np.sqrt(np.mean(np.square(diff.astype(np.float64)))))
peak = max(1.0, float(np.abs(expected.astype(np.float32)).max()))
psnr = math.inf if rms == 0 else 20.0 * math.log10(peak / rms)
mismatch = np.unravel_index(int(diff.argmax()), diff.shape)
print(f"K_REFERENCE {name} max_abs={maximum:.9g} max_rel={relative:.9g} "
f"rms={rms:.9g} psnr_db={psnr:.5g} worst={mismatch} "
f"expected={float(expected[mismatch]):.9g} actual={float(actual[mismatch]):.9g}")
return maximum, psnr
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--fixture-dir", required=True, type=pathlib.Path)
parser.add_argument("--kernel-name", choices=("enc1", "enc2"), default="enc1")
args = parser.parse_args()
folder = args.fixture_dir
params = SyntheticParams(folder)
output_channels = 96 if args.kernel_name == "enc2" else 64
reference_full, reference_merged = pwin_encoder(
params, 0, 0, H=2, C=64, COUT=output_channels,
posattn=args.kernel_name == "enc1")
actual_full = np.fromfile(folder / "full.bin", np.float16).reshape(64, 64)
actual_merged = np.fromfile(folder / "merged.bin", np.float16).reshape(16, output_channels)
full_abs, full_psnr = compare("full", actual_full, reference_full)
merged_abs, merged_psnr = compare("merged", actual_merged, reference_merged)
# Initial acceptance for the synthetic nonzero FP16 fixture. Tighten this
# after each K stage has been compared on captured DLSS weights/activations.
if full_abs > 0.001 or full_psnr < 60.0 or merged_abs > 0.001 or merged_psnr < 60.0:
raise RuntimeError(f"K {args.kernel_name} numerical mismatch against pwin_model.py")
print(f"PASS K_REFERENCE reference=logical kernel={args.kernel_name} "
"nonzero_weights=1 transformer_executed=1 nan_inf=0")
if __name__ == "__main__":
try:
main()
except Exception as error:
print(f"FAIL K_REFERENCE {error}", file=sys.stderr)
raise SystemExit(5)
tools/windows/k_module_probe.cpp
-224
View File
@@ -1,224 +0,0 @@
#include "hip_api.h"
#include <fstream>
namespace {
struct PwinParams {
int width, height;
const uint8_t* input;
const uint8_t* skip;
uint8_t* merged;
uint8_t* full;
uint64_t pad40, pad48;
int sx, sy;
const uint8_t* weights;
uint8_t rest[104];
};
static_assert(sizeof(PwinParams) == 176, "K parameter ABI");
static_assert(offsetof(PwinParams, weights) == 64, "K weight pointer ABI");
uint16_t half_quarter(int value)
{
const uint16_t positive[] = {0x0000u, 0x3400u, 0x3800u, 0x3a00u, 0x3c00u};
if (value < -4 || value > 4) throw std::runtime_error("bad half fixture");
return static_cast<uint16_t>(positive[std::abs(value)] | (value < 0 ? 0x8000u : 0u));
}
struct Allocation {
d4r::diag::HipApi& hip;
void* pointer = nullptr;
Allocation(d4r::diag::HipApi& api, size_t bytes) : hip(api) {
hip.check(hip.hipMalloc(&pointer, bytes), "hipMalloc(K fixture)");
}
~Allocation() { if (pointer) (void)hip.hipFree(pointer); }
};
}
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.module.empty()) throw std::runtime_error("--module is required");
HipApi hip(args.hip_root);
hipDeviceProp_t properties{};
hip.select_architecture(args.device, properties);
if (args.kernel_name != "enc1" && args.kernel_name != "enc2")
throw std::runtime_error("--kernel-name must be enc1 or enc2");
const bool enc2 = args.kernel_name == "enc2";
const std::string entry = "dltss_pwin_" + args.kernel_name + "_layer";
hipModule_t module = nullptr;
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(K)");
struct Cleanup {
HipApi& hip; hipModule_t module;
~Cleanup() { if (module) (void)hip.hipModuleUnload(module); }
} cleanup{hip, module};
hipFunction_t prep = nullptr, transformer = nullptr;
hip.check(hip.hipModuleGetFunction(&prep, module, (entry + "_prep").c_str()), "K prep");
hip.check(hip.hipModuleGetFunction(&transformer, module, entry.c_str()), "K transformer");
if (!prep || !transformer) throw std::runtime_error("Null K entry point");
// Identity fixture: all projection, MLP and patch-merge weights are zero.
// G1/G2 are one. The full transformer must return its input through its
// residual path and the merged output must be zero. Every stage launches.
constexpr size_t full_elements = 8u * 8u * 64u;
const size_t output_channels = enc2 ? 96u : 64u;
const size_t merged_elements = 4u * 4u * output_channels;
const size_t weight_bytes = enc2 ? 166080u : 149504u;
const size_t prep_items = (enc2 ? 288u : 256u) * 16u + 1024u;
std::vector<uint8_t> weights(weight_bytes, 0);
auto set_half = [&](size_t byte, uint16_t bits) {
weights.at(byte) = static_cast<uint8_t>(bits);
weights.at(byte + 1) = static_cast<uint8_t>(bits >> 8);
};
for (size_t c = 0; c < 64u; ++c) {
set_half(2u * c, 0x3c00u); // G1
set_half(49408u + 2u * c, 0x3c00u); // G2
}
std::vector<uint16_t> input(full_elements), full(full_elements, 0x7e00u);
std::vector<uint16_t> merged(merged_elements, 0x7e00u);
for (size_t i = 0; i < input.size(); ++i)
input[i] = half_quarter(static_cast<int>(i % 9u) - 4);
Allocation dev_input(hip, input.size() * sizeof(uint16_t));
Allocation dev_weights(hip, weights.size());
Allocation dev_full(hip, full.size() * sizeof(uint16_t));
Allocation dev_merged(hip, merged.size() * sizeof(uint16_t));
hip.check(hip.hipMemcpy(dev_input.pointer, input.data(), input.size() * sizeof(uint16_t),
hipMemcpyHostToDevice), "hipMemcpy(K input)");
hip.check(hip.hipMemcpy(dev_weights.pointer, weights.data(), weights.size(),
hipMemcpyHostToDevice), "hipMemcpy(K weights)");
PwinParams params{};
params.width = params.height = 8;
params.input = static_cast<const uint8_t*>(dev_input.pointer);
params.full = static_cast<uint8_t*>(dev_full.pointer);
params.merged = static_cast<uint8_t*>(dev_merged.pointer);
params.weights = static_cast<const uint8_t*>(dev_weights.pointer);
void* arguments[] = {&params};
hip.check(hip.hipModuleLaunchKernel(prep, static_cast<unsigned>((prep_items + 127u) / 128u), 1, 1,
128, 1, 1, 0, nullptr, arguments, nullptr), "hipModuleLaunchKernel(K prep)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(K prep)");
for (unsigned iteration = 0; iteration < args.iterations; ++iteration) {
std::fill(full.begin(), full.end(), 0x7e00u);
std::fill(merged.begin(), merged.end(), 0x7e00u);
hip.check(hip.hipMemcpy(dev_full.pointer, full.data(), full.size() * sizeof(uint16_t),
hipMemcpyHostToDevice), "hipMemcpy(K full sentinel)");
hip.check(hip.hipMemcpy(dev_merged.pointer, merged.data(), merged.size() * sizeof(uint16_t),
hipMemcpyHostToDevice), "hipMemcpy(K merged sentinel)");
hip.check(hip.hipModuleLaunchKernel(transformer, 1, 1, 1, 32, 1, 4, 0,
nullptr, arguments, nullptr), "hipModuleLaunchKernel(K transformer)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(K transformer)");
hip.check(hip.hipMemcpy(full.data(), dev_full.pointer, full.size() * sizeof(uint16_t),
hipMemcpyDeviceToHost), "hipMemcpy(K full result)");
hip.check(hip.hipMemcpy(merged.data(), dev_merged.pointer, merged.size() * sizeof(uint16_t),
hipMemcpyDeviceToHost), "hipMemcpy(K merged result)");
for (size_t i = 0; i < full.size(); ++i)
if (full[i] != input[i]) {
std::fprintf(stderr, "FAIL K_IDENTITY iteration=%u full_index=%zu expected=0x%04x actual=0x%04x\n",
iteration, i, input[i], full[i]);
return 5;
}
for (size_t i = 0; i < merged.size(); ++i)
if (merged[i] != 0u) {
std::fprintf(stderr, "FAIL K_IDENTITY iteration=%u merged_index=%zu expected=0x0000 actual=0x%04x\n",
iteration, i, merged[i]);
return 5;
}
}
std::printf("PASS K_IDENTITY architecture=%s kernel=%s iterations=%u full_elements=%zu merged_elements=%zu transformer_executed=1 nonzero_weights=0\n", d4r::diag::HipApi::target_arch(),
args.kernel_name.c_str(), args.iterations, full_elements, merged_elements);
// Exercise nonzero V projection, position-only attention and Wo GEMMs.
// The generated data is saved for comparison with pwin_model.py.
auto fragment_offset = [](unsigned k, unsigned n) {
return 64u * (n & 7u) + 16u * ((k & 7u) >> 1) + 8u * (n >> 3) +
4u * (k >> 3) + 2u * (k & 1u);
};
auto set_matrix = [&](size_t base, unsigned k, unsigned n, unsigned nt_stride) {
set_half(base + 512u * (k >> 4) + nt_stride * (n >> 4) +
fragment_offset(k & 15u, n & 15u), 0x3c00u);
};
constexpr size_t qkv = 128u, head = 12288u, bias = 24704u, wo = 41216u;
constexpr size_t w1 = 49664u, w2 = 82944u, pm = 115712u;
for (unsigned k = 0; k < 32u; ++k) {
if (enc2) {
set_matrix(qkv, k, k, 1024u); // head 0 Q
set_matrix(qkv + 2048u, k, k, 1024u); // head 0 K
set_matrix(qkv + head + 2048u * 3u, k, k, 1024u); // head 1 Q
set_matrix(qkv + head + 2048u * 4u, k, k, 1024u); // head 1 K
}
set_matrix(qkv + 2048u * 2u, k, k, 1024u); // head 0 V <- input 0:32
set_matrix(qkv + head + 2048u * 5u, k, k, 1024u); // head 1 V <- input 32:64
set_matrix(wo, k, k, 1024u); // head 0 Wo -> output 0:32
set_matrix(wo + 64u * 64u, k, k + 32u, 1024u); // head 1 Wo -> output 32:64
set_matrix(w1, k, k, 2048u); // first MLP group input -> hidden
set_matrix(w2, k, k, 1024u); // first MLP group hidden -> output
}
for (unsigned k = 0; k < 64u; ++k)
set_matrix(pm, k, k, 8192u); // patch merge selects top-left token
for (unsigned h = 0; h < 2u; ++h)
for (unsigned row = 0; row < 64u; ++row)
for (unsigned col = 0; col < 64u; ++col) {
const unsigned r = row & 15u, c = col & 15u;
const size_t offset = bias + h * 8192u +
512u * (4u * (col >> 4) + (row >> 4)) +
64u * (r & 7u) + 16u * ((c & 7u) >> 1) +
2u * (c & 1u) + 4u * (r >> 3) + 8u * (c >> 3);
set_half(offset, 0x2400u); // 1/64 position average
}
hip.check(hip.hipMemcpy(dev_weights.pointer, weights.data(), weights.size(),
hipMemcpyHostToDevice), "hipMemcpy(K nonzero weights)");
hip.check(hip.hipModuleLaunchKernel(prep, static_cast<unsigned>((prep_items + 127u) / 128u), 1, 1,
128, 1, 1, 0, nullptr, arguments, nullptr), "hipModuleLaunchKernel(K nonzero prep)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(K nonzero prep)");
std::fill(full.begin(), full.end(), 0x7e00u);
std::fill(merged.begin(), merged.end(), 0x7e00u);
hip.check(hip.hipMemcpy(dev_full.pointer, full.data(), full.size() * sizeof(uint16_t),
hipMemcpyHostToDevice), "hipMemcpy(K nonzero full sentinel)");
hip.check(hip.hipMemcpy(dev_merged.pointer, merged.data(), merged.size() * sizeof(uint16_t),
hipMemcpyHostToDevice), "hipMemcpy(K nonzero merged sentinel)");
hip.check(hip.hipModuleLaunchKernel(transformer, 1, 1, 1, 32, 1, 4, 0,
nullptr, arguments, nullptr), "hipModuleLaunchKernel(K nonzero transformer)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(K nonzero transformer)");
hip.check(hip.hipMemcpy(full.data(), dev_full.pointer, full.size() * sizeof(uint16_t),
hipMemcpyDeviceToHost), "hipMemcpy(K nonzero full result)");
hip.check(hip.hipMemcpy(merged.data(), dev_merged.pointer, merged.size() * sizeof(uint16_t),
hipMemcpyDeviceToHost), "hipMemcpy(K nonzero merged result)");
size_t changed = 0;
for (size_t i = 0; i < full.size(); ++i) {
if ((full[i] & 0x7c00u) == 0x7c00u)
throw std::runtime_error("K nonzero full output has NaN/Inf or unwritten sentinel at " + std::to_string(i));
changed += full[i] != input[i];
}
for (unsigned mt = 0; mt < 16u; ++mt) {
const unsigned my = mt >> 2, mx = mt & 3u;
const size_t full_base = (8u * (2u * my) + 2u * mx) * 64u;
for (unsigned channel = 0; channel < output_channels; ++channel) {
const size_t i = mt * output_channels + channel;
const uint16_t expected = channel < 64u ? full[full_base + channel] : 0u;
if (merged[i] != expected)
throw std::runtime_error("K patch merge identity mismatch at " + std::to_string(i));
}
}
if (!changed) throw std::runtime_error("K nonzero weights had no visible transformer effect");
if (!args.fixture_dir.empty()) {
std::filesystem::create_directories(wide(args.fixture_dir));
auto save = [&](const char* name, const void* data, size_t bytes) {
const auto path = std::filesystem::path(wide(args.fixture_dir)) / name;
std::ofstream file(path, std::ios::binary);
if (!file.write(static_cast<const char*>(data), static_cast<std::streamsize>(bytes)))
throw std::runtime_error(std::string("Failed writing K fixture ") + name);
};
save("input.bin", input.data(), input.size() * sizeof(uint16_t));
save("weights.bin", weights.data(), weights.size());
save("full.bin", full.data(), full.size() * sizeof(uint16_t));
save("merged.bin", merged.data(), merged.size() * sizeof(uint16_t));
}
std::printf("PASS K_NONZERO architecture=%s kernel=%s changed_elements=%zu full_elements=%zu merged_elements=%zu fixture_saved=%d numerical_reference_pending=1\n", d4r::diag::HipApi::target_arch(),
args.kernel_name.c_str(), changed, full.size(), merged.size(), !args.fixture_dir.empty());
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "FAIL K_MODULE %s\n", error.what());
loaded_modules();
return 4;
}
}
tools/windows/k_ptx_validate.py
-95
View File
@@ -1,95 +0,0 @@
"""Compare real K captures with the upstream CPU PTX interpreter.
The local DLL and captures remain private. Only numerical metrics are public.
Use this independent reference to audit the NumPy model's reduction uncertainty.
"""
import argparse
import math
import pathlib
import re
import sys
import numpy as np
ROOT = pathlib.Path(__file__).resolve().parents[2]
REFERENCE = pathlib.Path(__file__).resolve().parent / 'reference'
if not (REFERENCE / 'ptxsim.py').exists():
REFERENCE = ROOT / 'kernels/tools'
sys.path.insert(0, str(REFERENCE))
import pwin_model as pm
pm.NORM_FRAGMENT_ORDER = True
import pwin_check as pc
import ptxsim
from extract_dlss_ptx import fatbins, ptx_entries
def metrics(label, expected, actual):
expected = expected.astype(np.float64)
actual = actual.astype(np.float64)
if not np.isfinite(expected).all() or not np.isfinite(actual).all():
raise RuntimeError(f'{label} contains NaN/Inf')
error = np.abs(expected - actual)
peak = max(1., float(np.abs(expected).max()))
maximum = float(error.max())
relative = float((error / np.maximum(np.abs(expected), .001)).max())
rms = float(np.sqrt(np.mean(error ** 2)))
psnr = math.inf if rms == 0 else 20 * math.log10(peak / rms)
print(f'{label} max_abs={maximum:.9g} max_rel={relative:.9g} rms={rms:.9g} '
f'peak={peak:.9g} psnr_db={psnr:.6g}', flush=True)
return maximum, peak, psnr
def validate(args):
name = f'dltss_pwin_{args.layer}_layer'
dump = pm.Dump(str(args.capture_dir))
if dump.kernel != name:
raise RuntimeError('Capture kernel does not match requested layer')
native = pc.Outputs(dump, str(args.native_dir))
candidates = []
for bx, by in pc.windows(dump.grid, 12):
model = pc.model_window(dump, args.layer, bx, by)
expected, actual = pc.stored(dump, native, args.layer, bx, by, *model)
difference = float(np.abs(expected.astype(np.float64) - actual.astype(np.float64)).max())
candidates.append((difference, bx, by, model, expected, actual))
selected = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))[:args.windows]
pattern = re.compile(rb'\.entry\s+' + name.encode() + rb'\s*\(')
sources = [source.rstrip(b'\0') for _, fatbin in fatbins(args.dlss_dll.read_bytes())
for source in ptx_entries(fatbin) if pattern.search(source)]
if not sources:
raise RuntimeError('Local DLSS DLL does not contain the captured kernel')
args.output_dir.mkdir(parents=True, exist_ok=True)
path = args.output_dir / (name + '.ptx')
path.write_bytes(sources[-1])
for _, bx, by, model, numpy_expected, actual in selected:
_, memory, steps = ptxsim.run_block(str(path), name, str(args.capture_dir), (bx, by, 0))
reference_output = pc.Outputs.__new__(pc.Outputs)
reference_output.d = dump
reference_output.allocs = {identifier: (base, data) for identifier, base, data
in zip(memory.ids, memory.bases, memory.data)}
_, expected = pc.stored(dump, reference_output, args.layer, bx, by, *model)
print(f'PTX_REFERENCE layer={args.layer} window={bx},{by} statements={steps}', flush=True)
maximum, peak, psnr = metrics('PTX_VS_NATIVE', expected, actual)
metrics('PTX_VS_NUMPY', expected, numpy_expected)
# Half reductions and approximate reciprocal/rsqrt are not bit-exact
# across ISAs. Bound both the worst element and total error explicitly.
if maximum > max(.001, .005 * peak) or psnr < 60:
raise RuntimeError('Native layer differs from independent PTX reference')
print(f'PASS K_PTX_REFERENCE layer={args.layer} windows={len(selected)} nan_inf=0', flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--layer', choices=list(pc.LAYERS), required=True)
parser.add_argument('--capture-dir', type=pathlib.Path, required=True)
parser.add_argument('--native-dir', type=pathlib.Path, required=True)
parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
parser.add_argument('--windows', type=int, default=2)
args = parser.parse_args()
if args.windows < 1:
parser.error('--windows must be positive')
try:
validate(args)
except Exception as error:
print(f'FAIL K_PTX_REFERENCE {error}', file=sys.stderr)
raise SystemExit(5)
tools/windows/k_replay_validate.py
-165
View File
@@ -1,165 +0,0 @@
#!/usr/bin/env python3
"""Generate public synthetic K replay fixtures and validate native outputs.
Uses the upstream replay format and NumPy model for all eleven K layers.
No NVIDIA weights, PTX or DLLs are used by this fixture generator.
"""
import argparse
import math
import pathlib
import struct
import sys
import numpy as np
ROOT = pathlib.Path(__file__).resolve().parents[2]
REFERENCE = pathlib.Path(__file__).resolve().parent / 'reference'
if not (REFERENCE / 'pwin_check.py').exists():
REFERENCE = ROOT / 'kernels/tools'
sys.path.insert(0, str(REFERENCE))
import pwin_model as pm
pm.NORM_FRAGMENT_ORDER = True
from pwin_check import LAYERS, model_window, Outputs, stored, windows
def generate(layer, directory):
kind, heads, channels, other, extra = LAYERS[layer]
cin = 16 if kind == 'enc0' else channels
low = kind in ('dec', 'dec0')
cout = other if kind in ('enc', 'enc0') else channels
layout = pm.Layout(heads, channels, cout)
core_base = 2 * channels + 2 * 16 * channels if kind == 'enc0' else 0
if low:
core_base = 2 * other * 4 * channels + 8 * channels
weight_bytes = core_base + layout.PMB + 2 * layout.NPA * heads
weights = np.zeros(weight_bytes, np.uint8)
def half(offset, value):
weights[offset:offset + 2] = np.array([value], np.float16).view(np.uint8)
def matrix(base, k, n, stride, value=1.0):
half(base + 512 * (k // 16) + stride * (n // 16) + pm.frag_offset(k % 16, n % 16), value)
if kind == 'enc0':
for k in range(16):
matrix(2 * channels, k, k, 512)
matrix(2 * channels, k, k + 16, 512, .5)
if low:
for q in range(4):
for k in range(min(other, channels)):
matrix(0, k, q * channels + k, 512 * (other // 16), .25)
for c in range(channels):
half(core_base + layout.G1 + 2 * c, 1)
half(core_base + layout.G2 + 2 * c, 1)
half(core_base + layout.BO + 2 * c, 1 / 64)
half(core_base + layout.B2 + 2 * c, 1 / 64)
position_only = kind in ('enc0', 'dec0') or bool(extra)
for head in range(heads):
source_group = head % (channels // 32)
for j in range(3):
for k in range(32):
matrix(core_base + layout.QKV + layout.HEAD * head + 2048 * (3 * source_group + j),
k, k, 1024, .5 if j == 2 else .25)
for k in range(32):
matrix(core_base + layout.WO + 64 * channels * head, k,
32 * source_group + k, 1024, .5)
if position_only:
for row in range(64):
for col in range(64):
r, c = row % 16, col % 16
offset = core_base + layout.BIAS + head * 8192 + 512 * (4 * (col // 16) + row // 16)
offset += 64 * (r & 7) + 16 * ((c & 7) >> 1) + 2 * (c & 1) + 4 * (r >> 3) + 8 * (c >> 3)
half(offset, 1 / 64)
for group in range(channels // 32):
for k in range(32):
matrix(core_base + layout.W1 + 64 * channels * group, 32 * group + k, k, 32 * channels, .5)
matrix(core_base + layout.W2 + 64 * channels * group, k, 32 * group + k, 1024, .5)
if kind in ('enc', 'enc0'):
for n in range(cout):
matrix(core_base + layout.PM, n % channels + channels * ((n // channels) % 4),
n, 128 * channels, .5)
if kind == 'dec0':
for n in range(40):
matrix(core_base + layout.PM + 2 * 48, n % channels, n, 1024, .5)
half(core_base + layout.PM + 2 * n, 1 / 64)
input_elements = 16 * other if low else 64 * cin
x = ((np.arange(input_elements) % 9 - 4) / 4).astype(np.float16)
skip = ((np.arange(64 * channels) % 7 - 3) / 8).astype(np.float16)
output_channels = 40 if kind == 'dec0' else channels
allocations = {0: x.view(np.uint8), 1: skip.view(np.uint8), 2: weights}
if kind in ('enc0', 'enc'):
allocations[3] = np.full(16 * cout, np.nan, np.float16).view(np.uint8)
allocations[4] = np.full(64 * channels, np.nan, np.float16).view(np.uint8)
else:
allocations[3] = np.full(64 * output_channels, np.nan, np.float16).view(np.uint8)
parameters = bytearray(176)
struct.pack_into('<ii', parameters, 0, 8, 8)
pointers = {8: 0, 64: 2, 24: 3}
if low:
pointers[16] = 1
if kind in ('enc0', 'enc'):
pointers[32] = 4
bases = {i: 0x100000000 + i * 0x1000000 for i in allocations}
for offset, identifier in pointers.items():
struct.pack_into('<Q', parameters, offset, bases[identifier])
directory.mkdir(parents=True, exist_ok=True)
for identifier, data in allocations.items():
data.tofile(directory / f'alloc-{identifier}.bin')
(directory / 'args.bin').write_bytes(parameters)
lines = [f'kernel dltss_pwin_{layer}_layer', 'launch 1 1 1 32 1 4 0']
lines += [f'alloc {i} {bases[i]:016x} {data.size}' for i, data in allocations.items()]
lines += [f'pointer {offset} {i} 0' for offset, i in pointers.items()]
(directory / 'manifest.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
print(f'PASS K_FIXTURE kernel={layer} weights={weight_bytes} public_synthetic=1')
def validate(layer, directory, output, window_count=1, real_capture=False):
dump = pm.Dump(str(directory))
outputs = Outputs(dump, str(output))
pairs = []
for bx, by in windows(dump.grid, window_count):
model, extra = model_window(dump, layer, bx, by)
pairs.append(stored(dump, outputs, layer, bx, by, model, extra))
expected, actual = (np.concatenate(values) for values in zip(*pairs))
if not np.isfinite(expected).all() or not np.isfinite(actual).all():
raise RuntimeError('K replay contains NaN/Inf or unwritten output')
difference = np.abs(actual.astype(np.float64) - expected.astype(np.float64))
maximum = float(difference.max())
relative = float((difference / np.maximum(np.abs(expected.astype(np.float64)), 1e-3)).max())
rms = float(np.sqrt(np.mean(difference ** 2)))
peak = max(1., float(np.abs(expected.astype(np.float64)).max()))
psnr = math.inf if not rms else 20 * math.log10(peak / rms)
print(f'K_REPLAY_REFERENCE kernel={layer} elements={actual.size} max_abs={maximum:.9g} '
f'max_rel={relative:.9g} rms={rms:.9g} psnr_db={psnr:.6g}')
# Bound ISA approximate reciprocal/rsqrt and matrix accumulation differences;
# require the independent PTX reference gate for every real-weight layer.
absolute_limit = max(.001, .005 * peak) if real_capture else .001
print(f'K_REPLAY_LIMIT windows={len(pairs)} max_abs_limit={absolute_limit:.9g} min_psnr_db=60 real_capture={int(real_capture)}')
if maximum > absolute_limit or psnr < 60:
worst = int(difference.argmax())
raise RuntimeError(f'K mismatch at {worst}: reference={expected[worst]}, gpu={actual[worst]}')
print(f'PASS K_REPLAY_REFERENCE kernel={layer} reference=logical nan_inf=0')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('mode', choices=['generate', 'validate'])
parser.add_argument('--layer', choices=list(LAYERS), required=True)
parser.add_argument('--fixture-dir', type=pathlib.Path, required=True)
parser.add_argument('--output-dir', type=pathlib.Path)
parser.add_argument('--windows', type=int, default=1)
parser.add_argument('--real-capture', action='store_true')
args = parser.parse_args()
try:
if args.mode == 'generate':
generate(args.layer, args.fixture_dir)
else:
if not args.output_dir:
parser.error('validate requires --output-dir')
if args.windows < 1:
parser.error('--windows must be positive')
validate(args.layer, args.fixture_dir, args.output_dir, args.windows, args.real_capture)
except Exception as error:
print(f'FAIL K_REPLAY {error}', file=sys.stderr)
raise SystemExit(5)
tools/windows/loader_selection_probe.cpp
-33
View File
@@ -1,33 +0,0 @@
// Calls the drag-in loader the way OptiScaler does: the GPU-free discovery
// marker, then NGX feature requirements for every DXGI adapter.
#include <windows.h>
#include <dxgi.h>
#include <cstdio>
struct Info { unsigned SDKVersion, FeatureID; char reserved[48]; };
struct Requirement { unsigned FeatureSupported, MinHWArchitecture; char MinOSVersion[255]; };
using Requirements = unsigned (*)(IDXGIAdapter*, const Info*, Requirement*);
using Version = unsigned (*)();
int wmain(int argc, wchar_t** argv) {
if (argc != 2) { std::printf("usage: loader_selection_probe LOADER_DLL\n"); return 2; }
HMODULE loader = LoadLibraryW(argv[1]);
if (!loader) { std::printf("LOAD_FAILED error=%lu\n", GetLastError()); return 2; }
auto version = reinterpret_cast<Version>(reinterpret_cast<void*>(GetProcAddress(loader, "d4r_WindowsBackendVersion")));
auto requirements = reinterpret_cast<Requirements>(reinterpret_cast<void*>(GetProcAddress(loader, "NVSDK_NGX_D3D12_GetFeatureRequirements")));
if (!version || !requirements) { std::printf("EXPORTS_MISSING\n"); return 2; }
std::printf("MARKER version=%u hip_loaded=%d\n", version(), GetModuleHandleW(L"amdhip64_7.dll") != nullptr);
IDXGIFactory1* factory = nullptr;
if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory1), reinterpret_cast<void**>(&factory)))) return 2;
IDXGIAdapter1* adapter = nullptr;
for (UINT i = 0; factory->EnumAdapters1(i, &adapter) == S_OK; ++i) {
DXGI_ADAPTER_DESC1 desc{}; adapter->GetDesc1(&desc);
Info info{0x15, 1, {}}; Requirement result{};
const unsigned status = requirements(adapter, &info, &result);
std::printf("ADAPTER vendor=0x%04x status=0x%x supported=%u marker=0x%x name=%ls\n", desc.VendorId, status,
result.FeatureSupported, result.MinHWArchitecture, desc.Description);
adapter->Release();
}
factory->Release();
return 0;
}
tools/windows/m_capture_compare.py
-54
View File
@@ -1,54 +0,0 @@
"""Compare logical M inputs/outputs between private temporal capture sequences."""
import argparse
import pathlib
import numpy as np
from m_replay_validate import me, LAYERS
def logical(parameters, pointer, count, output=False, directory=None):
base, allocation = parameters.alloc_of(pointer)
if output:
identifier = next(i for i, (b, _) in parameters.allocs.items() if b == base)
allocation = np.fromfile(directory / 'gpu-output' / f'alloc-{identifier}.bin', np.uint8)
return allocation[pointer - base:pointer - base + count]
def compare(reference, actual):
paths = []
order = ['enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1']
for path in reference.glob('replay-*-rrlite_*_4x4'):
layer = path.name.split('rrlite_', 1)[1][:-4]
ordinal = int(path.name.split('-')[1])
frame = (ordinal - 1) // (6 if layer == 'enc3_tube' else 1)
paths.append((frame, order.index(layer), ordinal, path, layer))
for frame, _, ordinal, path, layer in sorted(paths):
other = actual / path.name
a, b = me.Params(str(path)), me.Params(str(other))
c, _, merged, cin = LAYERS[layer]
if (a.W, a.H, a.sx, a.sy, a.grid) != (b.W, b.H, b.sx, b.sy, b.grid):
raise RuntimeError(f'Launch geometry differs: {path.name}')
fields = [('input', 'inp', (a.W // 2) * (a.H // 2) * cin if cin else a.W * a.H * c, False)]
if cin:
fields.append(('skip', 'p32', a.W * a.H * c, False))
fields.append(('output', 'out', a.W * a.H * c, True))
if merged:
fields.append(('merge', 'p48', (a.W // 2) * (a.H // 2) * merged, True))
for label, field, count, output in fields:
x = logical(a, getattr(a, field), count, output, path)
y = logical(b, getattr(b, field), count, output, other)
if x.size != count or y.size != count:
raise RuntimeError(f'Truncated {path.name} {label}')
differing = np.flatnonzero(x != y)
print(f'M_CAPTURE_COMPARE frame={frame} layer={layer} ordinal={ordinal} field={label} '
f'codes={count} differing={differing.size} '
f'first={int(differing[0]) if differing.size else -1}', flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--reference', type=pathlib.Path, required=True)
parser.add_argument('--actual', type=pathlib.Path, required=True)
args = parser.parse_args()
compare(args.reference, args.actual)
tools/windows/m_native_replay_compare.py
-109
View File
@@ -1,109 +0,0 @@
"""Replay private M captures and require exact logical outputs against a validated baseline."""
import argparse
import hashlib
import json
import os
import pathlib
import subprocess
import traceback
import numpy as np
from m_capture_compare import logical, me, LAYERS
from m_replay_validate import metrics
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def main(args):
args.output_dir.mkdir(parents=True, exist_ok=True)
summary = dict(passed=False, strict=True, architecture='gfx1201', launches=[],
hipRoot=str(args.hip_root), modules={},
baselineValidation=json.loads(args.baseline_validation.read_text(encoding='utf-8-sig')))
try:
validation = summary['baselineValidation']
if not (validation.get('passed') and validation.get('exact') and validation.get('captured_outputs')):
raise RuntimeError('An exact captured-output NumPy validation is required for the baseline')
audited = {row['capture'] for row in validation['launches']}
captures = []
for layer in ('enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1'):
paths = sorted(args.capture_dir.glob(f'replay-*-rrlite_{layer}_4x4'))
if not paths:
raise RuntimeError(f'Baseline has no {layer} captures')
captures.extend((layer, path) for path in paths)
for layer, capture in captures:
if capture.name not in audited:
raise RuntimeError(f'Capture was not included in exact baseline validation: {capture.name}')
module = args.module_dir / f'rrlite_{layer}_4x4.hsaco'
summary['modules'][module.name] = sha(module)
output = args.output_dir / capture.name
output.mkdir(exist_ok=True)
env = dict(os.environ, D4R_DIAG_DIR=str(output.resolve()))
with (output / 'gpu.stdout.log').open('wb') as stdout, (output / 'gpu.stderr.log').open('wb') as stderr:
process = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
'--module', str(module.resolve()), '--fixture-dir', str(capture.resolve()),
'--output-dir', str((output / 'gpu-output').resolve()), '--iterations', '1'],
stdout=stdout, stderr=stderr, env=env, timeout=90)
row = dict(capture=capture.name, layer=layer, exitCode=f'0x{process.returncode & 0xffffffff:08x}', fields=[])
summary['launches'].append(row)
if process.returncode:
raise RuntimeError(f'GPU replay failed: {capture.name} ({row["exitCode"]})')
p = me.Params(str(capture))
channels, _, merged, _ = LAYERS[layer]
fields = [('output', p.out, p.W * p.H * channels)]
if merged:
fields.append(('merge', p.p48, (p.W // 2) * (p.H // 2) * merged))
for name, pointer, count in fields:
reference = logical(p, pointer, count, True, capture)
actual = logical(p, pointer, count, True, output)
if reference.size != count or actual.size != count:
raise RuntimeError(f'Truncated logical {name}: {capture.name}')
different = int(np.count_nonzero(reference != actual))
x, y = me.E4M3[reference], me.E4M3[actual]
maximum, _, psnr, _ = metrics(x, y, f'M_NATIVE_COMPARE capture={capture.name} field={name}')
row['fields'].append(dict(name=name, elements=count, differingCodes=different,
maxAbs=maximum, referenceSha256=hashlib.sha256(reference.tobytes()).hexdigest(),
actualSha256=hashlib.sha256(actual.tobytes()).hexdigest()))
if different:
raise RuntimeError(f'Exact baseline mismatch: {capture.name} {name}, codes={different}, PSNR={psnr}')
row['passed'] = True
if args.benchmark_control_dir:
summary['benchmarks'] = []
for layer in ('enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1'):
capture = next(path for item, path in captures if item == layer)
output = args.output_dir / ('paired-' + layer)
output.mkdir(exist_ok=True)
control = args.benchmark_control_dir / f'rrlite_{layer}_4x4.hsaco'
candidate = args.module_dir / control.name
with (output / 'paired.stdout.log').open('wb') as stdout, (output / 'paired.stderr.log').open('wb') as stderr:
process = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
'--module', str(control.resolve()), '--benchmark-module', str(candidate.resolve()),
'--fixture-dir', str(capture.resolve()), '--output-dir', str((output / 'output').resolve()),
'--iterations', '64'], stdout=stdout, stderr=stderr, timeout=90,
env=dict(os.environ, D4R_DIAG_DIR=str(output.resolve())))
if process.returncode:
raise RuntimeError(f'Paired benchmark failed: {layer}, exit={process.returncode:#x}')
from profile_report import report
measured = report(output)
if len(measured['replayPairs']) != 1 or measured['replayPairs'][0]['controlGpuMs']['samples'] != 64:
raise RuntimeError(f'Incomplete paired measurements: {layer}')
summary['benchmarks'].extend(measured['replayPairs'])
print(f'PAIRED M_NATIVE_COMPARE layer={layer} samples=64 logs={output}', flush=True)
summary['passed'] = True
print(f'PASS M_NATIVE_COMPARE launches={len(captures)} exact=1 finite=1 architecture=gfx1201', flush=True)
return 0
except Exception as error:
summary['error'] = str(error)
traceback.print_exc()
return 1
finally:
(args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
for name in ('capture-dir', 'module-dir', 'probe', 'hip-root', 'baseline-validation', 'output-dir'):
parser.add_argument('--' + name, type=pathlib.Path, required=True)
parser.add_argument('--benchmark-control-dir', type=pathlib.Path)
raise SystemExit(main(parser.parse_args()))
tools/windows/m_ptx_validate.py
-69
View File
@@ -1,69 +0,0 @@
"""Independent PTX interpreter validation of real M captures; keep outputs private."""
import argparse
import pathlib
import re
import sys
import numpy as np
from m_replay_validate import me, LAYERS, blocks, model_block, stored, metrics
import ptxsim
from extract_dlss_ptx import fatbins, ptx_entries
def validate(args):
name = f'rrlite_{args.layer}_4x4'
parameters = me.Params(str(args.capture_dir))
if parameters.kernel != name:
raise RuntimeError('Capture kernel does not match requested M layer')
native = {i: np.fromfile(args.native_dir / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs}
candidates = []
coordinates = args.block or blocks(parameters.grid, args.candidate_blocks)
for bx, by in coordinates:
if not (0 <= bx < parameters.grid[0] and 0 <= by < parameters.grid[1]):
raise RuntimeError(f'Block {bx},{by} is outside the capture grid')
model = model_block(parameters, args.layer, bx, by)
expected, actual = stored(parameters, args.layer, bx, by, native, model)
difference = float(np.abs(expected - actual).max())
candidates.append((difference, bx, by, model, expected, actual))
selected = sorted(candidates, key=lambda c: (-c[0], c[1], c[2]))[:args.blocks]
pattern = re.compile(rb'\.entry\s+' + name.encode() + rb'\s*\(')
sources = [source.rstrip(b'\0') for _, fatbin in fatbins(args.dlss_dll.read_bytes())
for source in ptx_entries(fatbin) if pattern.search(source)]
if not sources:
raise RuntimeError('Local DLSS DLL does not contain this M kernel')
args.output_dir.mkdir(parents=True, exist_ok=True)
path = args.output_dir / (name + '.ptx')
path.write_bytes(sources[-1])
for _, bx, by, model, numpy_expected, actual in selected:
_, memory, steps = ptxsim.run_block(str(path), name, str(args.capture_dir), (bx, by, 0))
allocation_data = dict(zip(memory.ids, memory.data))
_, expected = stored(parameters, args.layer, bx, by, allocation_data, model)
print(f'PTX_REFERENCE layer={args.layer} block={bx},{by} statements={steps}', flush=True)
_, _, psnr, fp8_steps = metrics(expected, actual, 'PTX_VS_NATIVE')
metrics(expected, numpy_expected, 'PTX_VS_NUMPY')
if fp8_steps.max() > 1 or psnr < 50:
raise RuntimeError('Native M differs from independent PTX reference')
print(f'PASS M_PTX_REFERENCE layer={args.layer} blocks={len(selected)} nan_inf=0', flush=True)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--layer', choices=list(LAYERS), required=True)
parser.add_argument('--capture-dir', type=pathlib.Path, required=True)
parser.add_argument('--native-dir', type=pathlib.Path, required=True)
parser.add_argument('--dlss-dll', type=pathlib.Path, required=True)
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
parser.add_argument('--blocks', type=int, default=2)
parser.add_argument('--candidate-blocks', type=int, default=12,
help='NumPy scan size used to choose the worst blocks for independent PTX')
parser.add_argument('--block', type=lambda value: tuple(map(int, value.split(','))),
action='append', help='Explicit x,y regression block (repeatable)')
args = parser.parse_args()
if args.blocks < 1 or args.candidate_blocks < 1:
parser.error('--blocks must be positive')
try:
validate(args)
except Exception as error:
print(f'FAIL M_PTX_REFERENCE {error}', file=sys.stderr)
raise SystemExit(5)
tools/windows/m_replay_validate.py
-179
View File
@@ -1,179 +0,0 @@
"""Public synthetic M fixtures and strict FP16-widening replay validation.
No NVIDIA binaries, weights or extracted PTX are needed to generate fixtures.
The wire format is upstream's replay format, also accepted by native_replay_probe.
"""
import argparse
import math
import pathlib
import struct
import sys
import numpy as np
ROOT = pathlib.Path(__file__).resolve().parents[2]
REFERENCE = pathlib.Path(__file__).resolve().parent / 'reference'
if not (REFERENCE / 'swin_check.py').exists():
REFERENCE = ROOT / 'kernels/tools'
sys.path.insert(0, str(REFERENCE))
import model_enc3 as me
import swin_model as sm
import swin_check
swin_check.CLAMP_PATCH_MERGE = True
from swin_check import LAYERS, blocks, model_block, codes_at
def encode(values):
"""Encode already quantised finite values without depending on vendor FP8."""
values = me.q8(values)
table = me.E4M3[:127]
codes = np.searchsorted(table, np.abs(values)).astype(np.uint8)
return codes | ((values < 0).astype(np.uint8) << 7)
def generate(layer, directory, size):
C, heads, NPM, CIN = LAYERS[layer]
pre = 4 * C * CIN + 8 * C if CIN else 0
layout = sm.layout(C, heads, pre)
weights = np.zeros(layout['end'] + 4 * C * NPM + 2 * NPM, np.uint8)
def half(offset, values):
data = np.asarray(values, np.float16).reshape(-1).view(np.uint8)
weights[offset:offset + data.size] = data
def matrix(base, ks, ns, k, n, source, dest, value=.25):
offsets = sm.woff_table(base, ks, ns, k, n)
weights[offsets[source, dest]] = encode(np.array(value))
if CIN:
cols = 4 * C // heads
for head in range(heads):
for n in range(cols):
matrix(head * CIN * cols, 512, 16 * CIN, CIN, cols,
(head * cols + n) % CIN, n)
half(4 * C * CIN, np.full(4 * C, 1 / 64))
half(layout['g1'], np.ones(C))
half(layout['g2'], np.ones(C))
half(layout['bo'], np.full(C, 1 / 64))
half(layout['b2'], np.full(C, 1 / 64))
for head, base in enumerate(layout['head']):
for n in range(32):
source = (32 * head + n) % C
matrix(base, 1024, 512, C, 32, source, n)
matrix(base + 32 * C, 1024, 512, C, 32, source, n, .5)
matrix(base + 64 * C + 512, 0, 512, 32, C, n, source, .5)
for group, base in enumerate(layout['b1']):
half(base + 32 * C, np.full(32, 1 / 64))
for n in range(32):
source = (32 * group + n) % C
matrix(base, 512, 16 * C, C, 32, source, n, .5)
matrix(base + 32 * C + 64, 0, 512, 32, C, n, source, .25)
for group in range(NPM // 32):
for n in range(32):
matrix(layout['end'] + group * 128 * C, 512, 64 * C,
4 * C, 32, (group * 32 + n) % (4 * C), n, 8.)
half(layout['end'] + 4 * C * NPM + 64 * group, np.full(32, 1 / 64))
input_count = size * size * C if not CIN else (size // 2) ** 2 * CIN
input_data = encode((np.arange(input_count) % 9 - 4) / 4)
skip = encode((np.arange(size * size * C) % 7 - 3) / 8) if CIN else np.empty(0, np.uint8)
allocations = {0: weights, 1: np.concatenate([input_data, skip]),
2: np.full(size * size * C, 0x7f, np.uint8)}
if NPM:
allocations[3] = np.full((size // 2) ** 2 * NPM, 0x7f, np.uint8)
bases = {i: 0x100000000 + i * 0x1000000 for i in allocations}
parameters = bytearray(88 if layer == 'enc3_tube' else 56)
struct.pack_into('<iiii', parameters, 8, 0, 0, size, size)
pointers = {0: (0, 0), 24: (1, 0), 40: (2, 0)}
if CIN:
pointers[32] = (1, input_count)
if NPM:
pointers[48] = (3, 0)
for offset, (identifier, delta) in pointers.items():
struct.pack_into('<Q', parameters, offset, bases[identifier] + delta)
directory.mkdir(parents=True, exist_ok=True)
for identifier, data in allocations.items():
data.tofile(directory / f'alloc-{identifier}.bin')
(directory / 'args.bin').write_bytes(parameters)
lines = [f'kernel rrlite_{layer}_4x4', f'launch {size // 8} {size // 8} 1 32 1 {heads} 0']
lines += [f'alloc {i} {bases[i]:016x} {data.size}' for i, data in allocations.items()]
lines += [f'pointer {offset} {i} {delta}' for offset, (i, delta) in pointers.items()]
(directory / 'manifest.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
print(f'PASS M_FIXTURE kernel={layer} size={size} weights={weights.size} public_synthetic=1')
def stored(parameters, layer, bx, by, allocations, model):
C, _, NPM, _ = LAYERS[layer]
def read(ptr, w, h, x, y, channels):
base, _ = parameters.alloc_of(ptr)
identifier = next(i for i, (b, _) in parameters.allocs.items() if b == base)
valid = (x >= 0) & (x < w) & (y >= 0) & (y < h)
codes = codes_at(allocations[identifier], base, ptr, w, h, x[valid], y[valid], channels)
return me.E4M3[codes].ravel(), valid
x, y = parameters.token_xy(bx, by)
actual, valid = read(parameters.out, parameters.W, parameters.H, x, y, C)
got, want = [actual], [model[0][valid].ravel()]
if NPM:
r = np.arange(16)
mx = (8 * bx - parameters.sx) // 2 + (r & 3)
my = (8 * by - parameters.sy) // 2 + (r >> 2)
actual, valid = read(parameters.p48, parameters.W // 2, parameters.H // 2, mx, my, NPM)
got.append(actual)
want.append(model[1][valid].ravel())
return np.concatenate(want), np.concatenate(got)
def metrics(expected, actual, label):
if not np.isfinite(expected).all() or not np.isfinite(actual).all():
raise RuntimeError(f'{label}: NaN/Inf or unwritten output')
error = np.abs(actual.astype(np.float64) - expected.astype(np.float64))
peak = max(1., float(np.abs(expected).max()))
maximum = float(error.max())
relative = float((error / np.maximum(np.abs(expected), .001)).max())
rms = float(np.sqrt(np.mean(error ** 2)))
psnr = math.inf if rms == 0 else 20 * math.log10(peak / rms)
rank = lambda v: np.searchsorted(me._POS, np.abs(v)) * np.sign(v)
steps = np.abs(rank(actual) - rank(expected))
print(f'{label} elements={actual.size} max_abs={maximum:.9g} max_rel={relative:.9g} '
f'rms={rms:.9g} psnr_db={psnr:.6g} identical={np.mean(error == 0):.9g} '
f'max_fp8_steps={int(steps.max())}', flush=True)
return maximum, peak, psnr, steps
def validate(layer, directory, output, block_count, real_capture, exact=False):
parameters = me.Params(str(directory))
if parameters.kernel != f'rrlite_{layer}_4x4':
raise RuntimeError('Replay kernel does not match requested M layer')
allocations = {i: np.fromfile(output / f'alloc-{i}.bin', np.uint8) for i in parameters.allocs}
pairs = [stored(parameters, layer, bx, by, allocations, model_block(parameters, layer, bx, by))
for bx, by in blocks(parameters.grid, block_count)]
expected, actual = (np.concatenate(values) for values in zip(*pairs))
maximum, peak, psnr, steps = metrics(expected, actual, f'M_REPLAY_REFERENCE kernel={layer}')
if ((exact or not real_capture) and maximum != 0) or (real_capture and (steps.max() > 1 or psnr < 50)):
raise RuntimeError(f'M mismatch: exact synthetic / <=1 FP8 step and >=50 dB real gate; peak={peak}')
print(f'PASS M_REPLAY_REFERENCE kernel={layer} blocks={len(pairs)} reference=logical nan_inf=0')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['generate', 'validate'])
parser.add_argument('--layer', choices=list(LAYERS), required=True)
parser.add_argument('--fixture-dir', type=pathlib.Path, required=True)
parser.add_argument('--output-dir', type=pathlib.Path)
parser.add_argument('--size', type=int, choices=[8, 16, 24, 32], default=16)
parser.add_argument('--blocks', type=int, default=12)
parser.add_argument('--real-capture', action='store_true')
parser.add_argument('--exact', action='store_true', help='Require every stored real FP8 value to match')
args = parser.parse_args()
try:
if args.mode == 'generate':
generate(args.layer, args.fixture_dir, args.size)
else:
if not args.output_dir or args.blocks < 1:
parser.error('validate requires --output-dir and positive --blocks')
validate(args.layer, args.fixture_dir, args.output_dir, args.blocks, args.real_capture, args.exact)
except Exception as error:
print(f'FAIL M_REPLAY {error}', file=sys.stderr)
raise SystemExit(5)
tools/windows/m_temporal_validate.py
-53
View File
@@ -1,53 +0,0 @@
"""Validate every captured M launch and spatial block; all allocations stay private."""
import argparse
import contextlib
import json
import pathlib
import sys
import traceback
from m_replay_validate import LAYERS, validate
def main(args):
args.output_dir.mkdir(parents=True, exist_ok=True)
summary = {'passed': False, 'captured_outputs': True, 'exact': args.exact, 'launches': []}
status = 1
try:
captures = sorted(args.capture_dir.glob('replay-*-rrlite_*_4x4'))
if not captures:
raise RuntimeError('No audited M captures were found')
for capture in captures:
layer = capture.name.split('rrlite_', 1)[1][:-4]
if layer not in LAYERS:
raise RuntimeError(f'Unknown captured M layer: {layer}')
logfile = args.output_dir / (capture.name + '.log')
with logfile.open('w', encoding='utf-8') as output, contextlib.redirect_stdout(output):
validate(layer, capture, capture / 'gpu-output', args.blocks, True, args.exact)
print(capture.name + ' ' + logfile.read_text().splitlines()[0], flush=True)
summary['launches'].append({'capture': capture.name, 'layer': layer, 'passed': True})
present = {item['layer'] for item in summary['launches']}
if present != set(LAYERS):
raise RuntimeError('Capture set does not contain all five M layers')
summary['passed'] = True
print(f'PASS M_TEMPORAL_REFERENCE launches={len(captures)} blocks={args.blocks} exact={int(args.exact)}', flush=True)
status = 0
except Exception as error:
summary['error'] = str(error)
summary['failed_capture'] = capture.name if 'capture' in locals() else None
traceback.print_exc()
finally:
(args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
return status
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--capture-dir', type=pathlib.Path, required=True)
parser.add_argument('--output-dir', type=pathlib.Path, required=True)
parser.add_argument('--blocks', type=int, default=10000)
parser.add_argument('--exact', action='store_true')
args = parser.parse_args()
if args.blocks < 1:
parser.error('--blocks must be positive')
sys.exit(main(args))
tools/windows/native_replay_probe.cpp
-249
View File
@@ -1,249 +0,0 @@
#include "hip_api.h"
#include <fstream>
#include <sstream>
#include <memory>
#include <chrono>
#include <cmath>
#include <limits>
namespace {
std::vector<uint8_t> read_file(const std::filesystem::path& path) {
std::ifstream file(path, std::ios::binary);
if (!file) throw std::runtime_error("Cannot read replay file " + d4r::diag::utf8(path.c_str()));
return {std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()};
}
struct Allocation {
d4r::diag::HipApi& hip;
int id;
size_t size;
void* device = nullptr;
std::vector<uint8_t> initial;
Allocation(d4r::diag::HipApi& api, int identifier, const std::vector<uint8_t>& bytes)
: hip(api), id(identifier), size(bytes.size()), initial(bytes) {
hip.check(hip.hipMalloc(&device, size), "hipMalloc(replay)");
try { hip.check(hip.hipMemcpy(device, bytes.data(), size, hipMemcpyHostToDevice), "hipMemcpy(replay upload)"); }
catch (...) { (void)hip.hipFree(device); device = nullptr; throw; }
}
~Allocation() { if (device) (void)hip.hipFree(device); }
};
}
int main(int argc, char** argv) {
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.module.empty() || args.fixture_dir.empty() || args.output_dir.empty())
throw std::runtime_error("--module, --fixture-dir and --output-dir are required");
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
const std::filesystem::path input(wide(args.fixture_dir)), output(wide(args.output_dir));
auto parameters = read_file(input / L"args.bin");
if (parameters.empty() || parameters.size() > 4096) throw std::runtime_error("Invalid replay parameter size");
std::ifstream manifest(input / L"manifest.txt");
if (!manifest) throw std::runtime_error("Replay manifest is missing");
std::vector<std::string> lines;
std::string line, kernel;
unsigned grid[3] = {1, 1, 1}, block[3] = {32, 1, 4}, shared = 0;
std::vector<std::unique_ptr<Allocation>> allocations;
while (std::getline(manifest, line)) {
lines.push_back(line);
std::istringstream stream(line);
std::string kind;
stream >> kind;
if (kind == "kernel") stream >> kernel;
else if (kind == "launch") stream >> grid[0] >> grid[1] >> grid[2] >> block[0] >> block[1] >> block[2] >> shared;
else if (kind == "texture" || kind == "surface") throw std::runtime_error("This replay requires a texture/surface recreation backend");
else if (kind == "alloc") {
int id = -1; std::string base; size_t size = 0;
stream >> id >> base >> size;
if (!stream || id < 0 || size == 0 || size > props.totalGlobalMem / 2)
throw std::runtime_error("Invalid replay allocation record");
for (const auto& allocation : allocations)
if (allocation->id == id) throw std::runtime_error("Duplicate replay allocation ID");
auto bytes = read_file(input / ("alloc-" + std::to_string(id) + ".bin"));
if (bytes.size() != size) throw std::runtime_error("Replay allocation size mismatch");
allocations.push_back(std::make_unique<Allocation>(hip, id, bytes));
}
}
if (kernel.empty() || allocations.empty() || !grid[0] || !grid[1] || !grid[2])
throw std::runtime_error("Incomplete replay manifest");
for (const auto& record : lines) {
std::istringstream stream(record); std::string kind;
stream >> kind;
if (kind != "pointer") continue;
size_t offset = 0, delta = 0; int id = -1;
stream >> offset >> id >> delta;
if (!stream || offset > parameters.size() || parameters.size() - offset < 8)
throw std::runtime_error("Invalid replay pointer offset");
auto allocation = std::find_if(allocations.begin(), allocations.end(), [&](const auto& a) { return a->id == id; });
if (allocation == allocations.end() || delta >= (*allocation)->size)
throw std::runtime_error("Invalid replay pointer relocation");
const uint64_t pointer = reinterpret_cast<uintptr_t>((*allocation)->device) + delta;
std::memcpy(parameters.data() + offset, &pointer, 8);
}
hipModule_t module = nullptr;
hip.check(hip.hipModuleLoad(&module, args.module.c_str()), "hipModuleLoad(replay)");
struct Cleanup { HipApi& hip; hipModule_t module; ~Cleanup() { (void)hip.hipModuleUnload(module); } } cleanup{hip, module};
if (const char* debugBlock = std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
int coordinates[2]{};
if (std::sscanf(debugBlock, "%d,%d", &coordinates[0], &coordinates[1]) != 2) throw std::runtime_error("D4R_SWIN_DEBUG_BLOCK must be x,y");
hipDeviceptr_t address = nullptr; size_t size = 0;
hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_block"), "Find Swin diagnostic block");
if (size != sizeof(coordinates)) throw std::runtime_error("Swin diagnostic coordinates ABI");
hip.check(hip.hipMemcpy(address, coordinates, size, hipMemcpyHostToDevice), "Set Swin diagnostic block");
}
auto module_scalar = [&](hipModule_t owner, const char* name, unsigned fallback) {
hipDeviceptr_t address = nullptr; size_t size = 0;
if (hip.hipModuleGetGlobal(&address, &size, owner, name) != hipSuccess) return fallback;
if (size != 4) throw std::runtime_error(std::string("Invalid native metadata ") + name);
unsigned value = 0;
hip.check(hip.hipMemcpy(&value, address, 4, hipMemcpyDeviceToHost), name);
return value ? value : fallback;
};
auto scalar = [&](const char* name, unsigned fallback) { return module_scalar(module, name, fallback); };
const std::vector<unsigned> captured_grid{grid[0], grid[1], grid[2]};
const unsigned captured_block_z = block[2];
block[2] = scalar("d4r_block_z", block[2]);
const unsigned persistentGrid = scalar("d4r_grid_x", 0);
if (persistentGrid) { grid[0] = persistentGrid; grid[1] = grid[2] = 1; }
hipFunction_t function = nullptr, prep = nullptr;
hip.check(hip.hipModuleGetFunction(&function, module, kernel.c_str()), "hipModuleGetFunction(replay)");
void* launchArguments[] = {parameters.data()};
if (hip.hipModuleGetFunction(&prep, module, (kernel + "_prep").c_str()) == hipSuccess) {
const unsigned prepBlocks = scalar("d4r_prep_blocks", 0);
if (!prepBlocks) throw std::runtime_error("Native prep has no valid d4r_prep_blocks");
hip.check(hip.hipModuleLaunchKernel(prep, prepBlocks, 1, 1, 128, 1, 1, 0, nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay prep)");
}
std::printf("REPLAY kernel=%s grid=%u,%u,%u block=%u,%u,%u allocations=%zu arguments=%zu\n",
kernel.c_str(), grid[0], grid[1], grid[2], block[0], block[1], block[2], allocations.size(), parameters.size());
if (!args.benchmark_module.empty()) {
hipModule_t candidate = nullptr;
hip.check(hip.hipModuleLoad(&candidate, args.benchmark_module.c_str()), "Load paired benchmark candidate");
Cleanup candidate_cleanup{hip, candidate};
unsigned grids[2][3] = {{grid[0], grid[1], grid[2]},
{captured_grid[0], captured_grid[1], captured_grid[2]}};
unsigned blocks[2][3] = {{block[0], block[1], block[2]},
{block[0], block[1], module_scalar(candidate, "d4r_block_z", captured_block_z)}};
const unsigned candidate_persistent = module_scalar(candidate, "d4r_grid_x", 0);
if (candidate_persistent) { grids[1][0] = candidate_persistent; grids[1][1] = grids[1][2] = 1; }
for (unsigned variant = 0; variant < 2; ++variant)
std::printf("D4R_REPLAY_MODULE variant=%s grid=%u,%u,%u block=%u,%u,%u geometry_source=native_metadata\n",
variant ? "candidate" : "control", grids[variant][0], grids[variant][1], grids[variant][2],
blocks[variant][0], blocks[variant][1], blocks[variant][2]);
hipFunction_t other = nullptr, other_prep = nullptr;
hip.check(hip.hipModuleGetFunction(&other, candidate, kernel.c_str()), "Find paired benchmark function");
if (hip.hipModuleGetFunction(&other_prep, candidate, (kernel + "_prep").c_str()) == hipSuccess) {
const auto blocks = module_scalar(candidate, "d4r_prep_blocks", 0);
if (!blocks) throw std::runtime_error("Paired candidate prep metadata is missing");
hip.check(hip.hipModuleLaunchKernel(other_prep, blocks, 1, 1, 128, 1, 1, 0, nullptr,
launchArguments, nullptr), "Paired candidate prep");
}
auto create = hip.library.symbol<decltype(&::hipEventCreate)>("hipEventCreate");
auto destroy = hip.library.symbol<decltype(&::hipEventDestroy)>("hipEventDestroy");
auto record = hip.library.symbol<decltype(&::hipEventRecord)>("hipEventRecord");
auto synchronize = hip.library.symbol<decltype(&::hipEventSynchronize)>("hipEventSynchronize");
auto elapsed = hip.library.symbol<decltype(&::hipEventElapsedTime)>("hipEventElapsedTime");
// Documented hip/hip_ext.h ABI. Resolve dynamically like the other
// HIP APIs so the host probe does not depend on HIP host linking.
using DispatchFn = hipError_t (*)(hipFunction_t, uint32_t, uint32_t, uint32_t,
uint32_t, uint32_t, uint32_t, size_t, hipStream_t, void**, void**,
hipEvent_t, hipEvent_t, uint32_t);
DispatchFn dispatch = nullptr;
uint32_t global[2][3]{};
if (args.benchmark_timing == "dispatch") {
dispatch = hip.library.symbol<DispatchFn>("hipExtModuleLaunchKernel");
for (unsigned variant = 0; variant < 2; ++variant) for (unsigned axis = 0; axis < 3; ++axis) {
const auto items = uint64_t(grids[variant][axis]) * blocks[variant][axis];
if (!items || items > std::numeric_limits<uint32_t>::max())
throw std::runtime_error("Dispatch profiling work-item dimension overflow");
global[variant][axis] = static_cast<uint32_t>(items);
}
}
struct Event {
decltype(destroy) release; hipEvent_t value = nullptr;
~Event() { if (value) (void)release(value); }
} begin{destroy}, end{destroy};
hip.check(create(&begin.value), "Create paired start event");
hip.check(create(&end.value), "Create paired end event");
auto launch = [&](hipFunction_t fn) {
const unsigned variant = fn == other;
hip.check(hip.hipModuleLaunchKernel(fn, grids[variant][0], grids[variant][1], grids[variant][2],
blocks[variant][0], blocks[variant][1], blocks[variant][2],
shared, nullptr, launchArguments, nullptr), "Paired benchmark launch");
};
// Both modules and weight images stay resident on the same GPU.
// Warm continuous work first, then alternate AB/BA order. Restore
// captured inputs outside the event so neither module consumes
// modified temporal inputs from a preceding sample.
const auto verbose = hip.verbose; hip.verbose = false;
for (unsigned warm = 0; warm < 16; ++warm)
for (auto fn : {function, other}) for (unsigned batch = 0; batch < 16; ++batch) launch(fn);
hip.check(hip.hipDeviceSynchronize(), "Paired benchmark warmup");
for (unsigned pair = 0; pair < args.iterations; ++pair) for (unsigned order = 0; order < 2; ++order) {
const unsigned variant = order ^ (pair & 1u);
for (const auto& allocation : allocations)
hip.check(hip.hipMemcpy(allocation->device, allocation->initial.data(), allocation->size,
hipMemcpyHostToDevice), "Restore paired captured input");
const auto host_begin = std::chrono::steady_clock::now();
if (!dispatch) hip.check(record(begin.value, nullptr), "Record paired start");
for (unsigned batch = 0; batch < args.benchmark_batch; ++batch) {
if (dispatch) {
// Flags zero preserve stream order. Global sizes are
// work-items, unlike hipModuleLaunchKernel's blocks.
hip.check(dispatch(variant ? other : function, global[variant][0], global[variant][1], global[variant][2],
blocks[variant][0], blocks[variant][1], blocks[variant][2], shared, nullptr, launchArguments, nullptr,
batch == 0 ? begin.value : nullptr,
batch + 1 == args.benchmark_batch ? end.value : nullptr, 0),
"Paired dispatch-profiled launch");
} else launch(variant ? other : function);
}
if (!dispatch) hip.check(record(end.value, nullptr), "Record paired end");
hip.check(synchronize(end.value), "Complete paired sample");
const auto host_ms = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - host_begin).count();
float ms = 0; hip.check(elapsed(&ms, begin.value, end.value), "Paired GPU elapsed time");
const bool valid = std::isfinite(ms) && ms > 0 && double(ms) <= host_ms;
std::printf("D4R_REPLAY_PROFILE%s kernel=%s variant=%s pair=%u gpu_ms=%.6f "
"host_ms=%.6f timing_source=%s launches=%u warmup_launches=512 paired=1 "
"input_restore_outside_event=1\n", valid ? "" : "_INVALID",
kernel.c_str(), variant ? "candidate" : "control", pair,
double(ms) / args.benchmark_batch, host_ms / args.benchmark_batch,
args.benchmark_timing.c_str(), args.benchmark_batch);
}
hip.verbose = verbose;
// Leave a single ordinary control replay for numerical checking.
for (const auto& allocation : allocations)
hip.check(hip.hipMemcpy(allocation->device, allocation->initial.data(), allocation->size,
hipMemcpyHostToDevice), "Restore final reference input");
launch(function);
}
for (unsigned iteration = 0; args.benchmark_module.empty() && iteration < args.iterations; ++iteration)
hip.check(hip.hipModuleLaunchKernel(function, grid[0], grid[1], grid[2], block[0], block[1], block[2], shared,
nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)");
hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)");
std::filesystem::create_directories(output);
if (std::getenv("D4R_SWIN_DEBUG_BLOCK")) {
hipDeviceptr_t address = nullptr; size_t size = 0;
hip.check(hip.hipModuleGetGlobal(&address, &size, module, "d4r_swin_debug_values"), "Find Swin diagnostic stages");
std::vector<uint8_t> bytes(size);
hip.check(hip.hipMemcpy(bytes.data(), address, size, hipMemcpyDeviceToHost), "Download Swin diagnostic stages");
std::ofstream file(output / "swin-stages.f16", std::ios::binary);
file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
if (!file) throw std::runtime_error("Cannot save Swin diagnostic stages");
}
for (const auto& allocation : allocations) {
std::vector<uint8_t> bytes(allocation->size);
hip.check(hip.hipMemcpy(bytes.data(), allocation->device, bytes.size(), hipMemcpyDeviceToHost), "hipMemcpy(replay result)");
std::ofstream file(output / ("alloc-" + std::to_string(allocation->id) + ".bin"), std::ios::binary);
file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
if (!file) throw std::runtime_error("Cannot save replay allocation");
}
std::printf("PASS NATIVE_REPLAY architecture=%s kernel=%s iterations=%u benchmark_pairs=%u numerical_reference=pending\n", d4r::diag::HipApi::target_arch(),
kernel.c_str(), args.benchmark_module.empty() ? args.iterations : 1, args.benchmark_module.empty() ? 0 : args.iterations);
return 0;
} catch (const std::exception& error) {
std::fprintf(stderr, "FAIL NATIVE_REPLAY %s\n", error.what()); loaded_modules(); return 4;
}
}
tools/windows/ngx_cuda_evaluate.h
-147
View File
@@ -1,147 +0,0 @@
#pragma once
#include "cuda_image_api.h"
#include <cmath>
#include <chrono>
#include <fstream>
extern "C" void d4r_ngx_set_uint(void*, const char*, unsigned);
extern "C" void d4r_ngx_set_int(void*, const char*, int);
extern "C" void d4r_ngx_set_float(void*, const char*, float);
extern "C" void d4r_ngx_set_ull(void*, const char*, unsigned long long);
extern "C" void d4r_ngx_set_void(void*, const char*, void*);
namespace d4r::diag {
inline float half_value(uint16_t bits) {
const unsigned exponent = (bits >> 10) & 31, mantissa = bits & 1023;
const float sign = bits & 0x8000 ? -1.0f : 1.0f;
if (exponent == 31) return mantissa ? NAN : sign * INFINITY;
if (!exponent) return sign * std::ldexp(float(mantissa), -24);
return sign * std::ldexp(float(1024 + mantissa), int(exponent) - 25);
}
inline void save_output(const std::filesystem::path& directory, const std::vector<uint16_t>& output,
unsigned width, unsigned height, unsigned frame) {
const auto stem = "output-" + std::to_string(frame);
std::ofstream raw(directory / (stem + ".rgba16f"), std::ios::binary);
raw.write(reinterpret_cast<const char*>(output.data()), std::streamsize(output.size() * 2));
// BMP is only a diagnostic preview; the raw file preserves unmodified HDR values.
const unsigned stride = (width * 3 + 3) & ~3u;
std::vector<unsigned char> bytes(54 + size_t(stride) * height, 0);
auto u32 = [&](size_t offset, uint32_t value) {
for (int j = 0; j < 4; ++j) bytes[offset + j] = static_cast<unsigned char>(value >> (8 * j));
};
bytes[0] = 'B'; bytes[1] = 'M'; u32(2, static_cast<uint32_t>(bytes.size())); u32(10, 54);
u32(14, 40); u32(18, width); u32(22, height); bytes[26] = 1; bytes[28] = 24;
for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < width; ++x)
for (unsigned c = 0; c < 3; ++c) {
const float value = half_value(output[(size_t(y) * width + x) * 4 + c]);
bytes[54 + size_t(height - y - 1) * stride + x * 3 + (2 - c)] =
static_cast<unsigned char>(std::clamp(value, 0.0f, 1.0f) * 255.0f + 0.5f);
}
std::ofstream bmp(directory / (stem + ".bmp"), std::ios::binary);
bmp.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
if (!raw || !bmp) throw std::runtime_error("Saving DLSS output failed");
}
inline void run_synthetic_dlss(CudaApi& cuda, Library& core, void* parameters,
unsigned preset, unsigned frames, const std::filesystem::path& directory) {
// CPU upload/readback is confined to this synthetic diagnostic. The game
// backend uses external D3D12 memory and GPU copies instead.
constexpr unsigned width = 256, height = 144, outWidth = 512, outHeight = 288;
using Create = unsigned(*)(unsigned, void*, void**);
using Evaluate = unsigned(*)(void*, void*, void*);
using Release = unsigned(*)(void*);
const auto create = core.symbol<Create>("NVSDK_NGX_CUDA_CreateFeature");
const auto evaluate = core.symbol<Evaluate>("NVSDK_NGX_CUDA_EvaluateFeature");
const auto release = core.symbol<Release>("NVSDK_NGX_CUDA_ReleaseFeature");
auto ngx_check = [](unsigned result, const char* operation) {
std::printf("NGX %s -> 0x%08x\n", operation, result);
if (result != 1) throw std::runtime_error(std::string(operation) + " result=" + std::to_string(result));
};
d4r_ngx_set_uint(parameters, "Width", width);
d4r_ngx_set_uint(parameters, "Height", height);
d4r_ngx_set_uint(parameters, "OutWidth", outWidth);
d4r_ngx_set_uint(parameters, "OutHeight", outHeight);
d4r_ngx_set_int(parameters, "PerfQualityValue", 2);
d4r_ngx_set_int(parameters, "DLSS.Feature.Create.Flags", 0);
d4r_ngx_set_int(parameters, "DLSS.Enable.Output.Subrects", 0);
d4r_ngx_set_uint(parameters, "CreationNodeMask", 1);
d4r_ngx_set_uint(parameters, "VisibilityNodeMask", 1);
for (const char* name : {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality",
"DLSS.Hint.Render.Preset.Balanced", "DLSS.Hint.Render.Preset.Performance",
"DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"})
d4r_ngx_set_uint(parameters, name, preset);
CUdeviceptr scratch = 0;
cuda.check(cuda.cuMemAlloc_v2(&scratch, 64ull * 1024 * 1024), "cuMemAlloc_v2(NGX scratch)");
struct ScratchCleanup { CudaApi& api; CUdeviceptr pointer; ~ScratchCleanup() { (void)api.cuMemFree_v2(pointer); } } sc{cuda, scratch};
d4r_ngx_set_void(parameters, "Scratch", reinterpret_cast<void*>(uintptr_t(scratch)));
d4r_ngx_set_ull(parameters, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
void* handle = nullptr;
std::printf("STAGE NGX_CREATE preset=%u dimensions=%ux%u->%ux%u\n", preset, width, height, outWidth, outHeight);
ngx_check(create(1, parameters, &handle), "NVSDK_NGX_CUDA_CreateFeature");
if (!handle) throw std::runtime_error("NGX CreateFeature returned a null handle");
struct FeatureCleanup { Release release; void*& handle; ~FeatureCleanup() { if (handle) (void)release(handle); } } fc{release, handle};
cuda::ImageApi images(cuda);
cuda::Image color(images, width, height, 16, 4, false, 1);
cuda::Image depth(images, width, height, 32, 1, false);
cuda::Image motion(images, width, height, 16, 2, false);
cuda::Image exposure(images, 1, 1, 32, 1, false);
cuda::Image output(images, outWidth, outHeight, 16, 4, true);
std::vector<uint16_t> input(size_t(width) * height * 4), zeroMotion(size_t(width) * height * 2, 0);
std::vector<float> inputDepth(size_t(width) * height, 0.5f);
for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < width; ++x) {
const size_t index = (size_t(y) * width + x) * 4;
input[index] = ((x / 16 + y / 16) & 1) ? 0x3a00 : 0x3400; // .75 / .25
input[index + 1] = (x & 32) ? 0x3800 : 0x3400;
input[index + 2] = (y & 32) ? 0x3a00 : 0x3800;
input[index + 3] = 0x3c00;
}
const float one = 1.0f;
color.upload(input.data()); depth.upload(inputDepth.data()); motion.upload(zeroMotion.data()); exposure.upload(&one);
d4r_ngx_set_void(parameters, "Color", &color.object);
d4r_ngx_set_void(parameters, "Depth", &depth.object);
d4r_ngx_set_void(parameters, "MotionVectors", &motion.object);
d4r_ngx_set_void(parameters, "ExposureTexture", &exposure.object);
d4r_ngx_set_void(parameters, "Output", &output.object);
for (const char* name : {"TransparencyMask", "DLSS.Input.Bias.Current.Color.Mask"})
d4r_ngx_set_void(parameters, name, nullptr);
for (const char* name : {"Jitter.Offset.X", "Jitter.Offset.Y", "Sharpness"})
d4r_ngx_set_float(parameters, name, 0.0f);
for (const char* name : {"MV.Scale.X", "MV.Scale.Y", "DLSS.Pre.Exposure", "DLSS.Exposure.Scale"})
d4r_ngx_set_float(parameters, name, 1.0f);
d4r_ngx_set_float(parameters, "FrameTimeDeltaInMsec", 16.666667f);
d4r_ngx_set_int(parameters, "Disable.Watermark", 1);
d4r_ngx_set_uint(parameters, "DLSS.Render.Subrect.Dimensions.Width", width);
d4r_ngx_set_uint(parameters, "DLSS.Render.Subrect.Dimensions.Height", height);
for (const char* name : {"DLSS.Input.Color.Subrect.Base.X", "DLSS.Input.Color.Subrect.Base.Y",
"DLSS.Input.Depth.Subrect.Base.X", "DLSS.Input.Depth.Subrect.Base.Y", "DLSS.Input.MV.Subrect.Base.X",
"DLSS.Input.MV.Subrect.Base.Y", "DLSS.Output.Subrect.Base.X", "DLSS.Output.Subrect.Base.Y"})
d4r_ngx_set_uint(parameters, name, 0);
std::vector<uint16_t> pixels(size_t(outWidth) * outHeight * 4, 0x7e00);
for (unsigned frame = 0; frame < frames; ++frame) {
std::fill(pixels.begin(), pixels.end(), 0x7e00); // unwritten output is an explicit failure
output.upload(pixels.data());
d4r_ngx_set_int(parameters, "Reset", frame == 0 ? 1 : 0);
std::printf("STAGE NGX_EVALUATE preset=%u frame=%u\n", preset, frame);
const auto begin = std::chrono::steady_clock::now();
ngx_check(evaluate(handle, parameters, nullptr), "NVSDK_NGX_CUDA_EvaluateFeature");
cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize(NGX Evaluate)");
const double milliseconds = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count();
output.download(pixels.data());
size_t nonfinite = 0; double sum = 0, square = 0;
for (size_t i = 0; i < pixels.size(); ++i) {
const float value = half_value(pixels[i]);
if (!std::isfinite(value)) ++nonfinite;
if ((i & 3) < 3) { sum += value; square += double(value) * value; }
}
const double count = outWidth * outHeight * 3;
const double variance = square / count - (sum / count) * (sum / count);
std::printf("NGX_OUTPUT preset=%u frame=%u nonfinite=%zu mean=%.9g variance=%.9g evaluate_ms=%.3f\n",
preset, frame, nonfinite, sum / count, variance, milliseconds);
if (nonfinite || !std::isfinite(variance) || variance < 1e-6)
throw std::runtime_error("DLSS output has NaN/Inf, unwritten pixels or no synthetic pattern");
save_output(directory, pixels, outWidth, outHeight, frame);
}
ngx_check(release(handle), "NVSDK_NGX_CUDA_ReleaseFeature"); handle = nullptr;
std::printf("PASS NGX_EVALUATE preset=%u frames=%u finite=1 network_validation=pending\n", preset, frames);
}
}
tools/windows/ngx_cuda_init_probe.cpp
-135
View File
@@ -1,135 +0,0 @@
#include "hip_api.h"
#include "cuda_api.h"
#include "ngx_cuda_evaluate.h"
// Same NGX C ABI and freestanding MSVC parameter accessors as the upstream shim.
// NVIDIA SDK headers and proprietary binaries are not build dependencies.
struct NgxPathListInfo { const wchar_t* const* Path; unsigned Length; };
struct NgxLoggingInfo { void* LoggingCallback; int MinimumLoggingLevel; bool DisableOtherLoggingSinks; };
struct NgxFeatureCommonInfo { NgxPathListInfo PathListInfo; void* InternalData; NgxLoggingInfo LoggingInfo; };
static_assert(sizeof(NgxFeatureCommonInfo) == 40);
extern "C" void d4r_ngx_set_uint(void*, const char*, unsigned);
extern "C" unsigned d4r_ngx_get_uint(void*, const char*, unsigned*);
extern "C" unsigned d4r_ngx_get_int(void*, const char*, int*);
static void ngx_log(const char* message, int level, unsigned component)
{
if (message) std::printf("NGX_LOG level=%d component=%u %s\n", level, component, message);
}
static void check(unsigned result, const char* call)
{
std::printf("NGX %s -> 0x%08x\n", call, result);
if (result != 1) throw std::runtime_error(std::string(call) + " NGX result=" + std::to_string(result));
}
int main(int argc, char** argv)
{
using namespace d4r::diag;
start();
try {
Args args(argc, argv);
if (args.cuda_dll.empty() || args.ngx_core.empty() || args.dlss_dll.empty())
throw std::runtime_error("--cuda-dll, --ngx-core and --dlss-dll absolute paths required");
const std::filesystem::path core_path(wide(args.ngx_core)), dlss_path(wide(args.dlss_dll));
if (dlss_path.filename() != L"nvngx_dlss.dll") throw std::runtime_error("DLSS file must be named nvngx_dlss.dll");
HipApi hip(args.hip_root);
hipDeviceProp_t props{};
hip.select_architecture(args.device, props);
SearchDirectory cuda_search(std::filesystem::path(wide(args.cuda_dll)).parent_path());
CudaApi cuda(args.cuda_dll);
cuda.check(cuda.cuInit(0), "cuInit");
int count = 0;
cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
CUdevice device = -1;
for (int i = 0; i < count; ++i) {
CUdevice candidate = -1;
int bus = -1, pci_device = -1, domain = -1;
cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "cuDeviceGetAttribute(PCI_BUS_ID)");
cuda.check(cuda.cuDeviceGetAttribute(&pci_device, 34, candidate), "cuDeviceGetAttribute(PCI_DEVICE_ID)");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "cuDeviceGetAttribute(PCI_DOMAIN_ID)");
if (bus == props.pciBusID && pci_device == props.pciDeviceID && domain == props.pciDomainID) device = candidate;
}
if (device < 0) throw std::runtime_error("NGX CUDA/HIP PCI identity mismatch");
CUcontext context = nullptr;
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, device), "cuDevicePrimaryCtxRetain");
struct ContextCleanup {
CudaApi& cuda; CUcontext& context; CUdevice device;
~ContextCleanup() { if (context) { (void)cuda.cuCtxSetCurrent(nullptr); (void)cuda.cuDevicePrimaryCtxRelease_v2(device); } }
} cc{cuda, context, device};
cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent");
auto nvapi_path = std::filesystem::path(wide(args.cuda_dll)).parent_path() / L"nvapi64.dll";
if (!std::filesystem::exists(nvapi_path)) nvapi_path = nvapi_path.parent_path().parent_path() / L"nvapi64.dll";
if (!args.nvapi_dll.empty()) nvapi_path = wide(args.nvapi_dll);
Library nvapi(nvapi_path);
SearchDirectory feature_search(dlss_path.parent_path());
// Preload the supplied absolute file so a system copy cannot replace it.
Library dlss(dlss_path), core(core_path);
using Init = unsigned(*)(unsigned long long, const wchar_t*, const NgxFeatureCommonInfo*, unsigned);
using DriverInit = unsigned(*)(unsigned long long, const wchar_t*, unsigned);
using ProjectInit = unsigned(*)(const char*, int, const char*, const wchar_t*, unsigned, const NgxFeatureCommonInfo*);
using Shutdown = unsigned(*)();
using Parameters = unsigned(*)(void**);
using Destroy = unsigned(*)(void*);
const auto init = core.symbol<Init>("NVSDK_NGX_CUDA_Init");
const auto driver_init = core.symbol<DriverInit>("NVSDK_NGX_CUDA_Init");
const auto shutdown = core.symbol<Shutdown>("NVSDK_NGX_CUDA_Shutdown");
const auto capabilities = core.symbol<Parameters>("NVSDK_NGX_CUDA_GetCapabilityParameters");
const auto allocate = core.symbol<Parameters>("NVSDK_NGX_CUDA_AllocateParameters");
const auto destroy = core.symbol<Destroy>("NVSDK_NGX_CUDA_DestroyParameters");
const std::wstring directory = dlss_path.parent_path().wstring();
const wchar_t* paths[] = {directory.c_str()};
const char* diagnostic_dir = std::getenv("D4R_DIAG_DIR");
const auto data_path = diagnostic_dir ? std::filesystem::path(wide(diagnostic_dir)) :
std::filesystem::current_path() / L"test-results" / L"ngx-init";
std::filesystem::create_directories(data_path);
NgxFeatureCommonInfo common{};
common.PathListInfo = {paths, 1};
common.LoggingInfo = {reinterpret_cast<void*>(&ngx_log), 2, false};
loaded_modules();
std::printf("STAGE NVSDK_NGX_CUDA_Init sdk=0x15 core=%s dlss=%s\n", args.ngx_core.c_str(), args.dlss_dll.c_str());
std::printf("NGX_APPLICATION id=%llu\n", args.ngx_app_id);
if (args.ngx_abi == "driver") check(driver_init(args.ngx_app_id, data_path.c_str(), 0x15), "NVSDK_NGX_CUDA_Init(driver ABI)");
else if (args.ngx_abi == "sdk") check(init(args.ngx_app_id, data_path.c_str(), &common, 0x15), "NVSDK_NGX_CUDA_Init(SDK ABI)");
else if (args.ngx_abi == "project") check(core.symbol<ProjectInit>("NVSDK_NGX_CUDA_Init_ProjectID")(
"24480451-f00d-face-1304-0308dabad187", 0, "1.0", data_path.c_str(), 0x15, &common), "NVSDK_NGX_CUDA_Init_ProjectID");
else throw std::runtime_error("--ngx-abi must be driver, sdk or project");
bool initialized = true;
struct NgxCleanup { Shutdown shutdown; bool& active; ~NgxCleanup() { if (active) (void)shutdown(); } } nc{shutdown, initialized};
void* caps = nullptr;
check(capabilities(&caps), "NVSDK_NGX_CUDA_GetCapabilityParameters");
if (!caps) throw std::runtime_error("NGX returned null capability parameters");
struct ParameterCleanup { Destroy destroy; void*& p; ~ParameterCleanup() { if (p) (void)destroy(p); } } pc{destroy, caps};
int available = -1, init_result = -1, needs_driver = -1;
check(d4r_ngx_get_int(caps, "SuperSampling.Available", &available), "Get(SuperSampling.Available)");
check(d4r_ngx_get_int(caps, "SuperSampling.FeatureInitResult", &init_result), "Get(SuperSampling.FeatureInitResult)");
check(d4r_ngx_get_int(caps, "SuperSampling.NeedsUpdatedDriver", &needs_driver), "Get(SuperSampling.NeedsUpdatedDriver)");
std::printf("CAPABILITIES available=%d feature_init=0x%08x needs_driver=%d\n", available, static_cast<unsigned>(init_result), needs_driver);
if (available != 1 || init_result != 1 || needs_driver != 0)
throw std::runtime_error("DLSS CUDA feature is unavailable; initialization alone is insufficient");
void* parameters = nullptr;
check(allocate(&parameters), "NVSDK_NGX_CUDA_AllocateParameters");
if (!parameters) throw std::runtime_error("Null NGX parameters");
ParameterCleanup ac{destroy, parameters};
d4r_ngx_set_uint(parameters, "Width", 640);
unsigned roundtrip = 0;
check(d4r_ngx_get_uint(parameters, "Width", &roundtrip), "MSVC parameter ABI roundtrip");
if (roundtrip != 640) throw std::runtime_error("NGX parameter ABI mismatch");
if (args.ngx_mode == "evaluate")
run_synthetic_dlss(cuda, core, parameters, args.preset, args.iterations, data_path);
else if (args.ngx_mode != "init") throw std::runtime_error("--ngx-mode must be init or evaluate");
check(destroy(parameters), "DestroyParameters(allocated)"); parameters = nullptr;
check(destroy(caps), "DestroyParameters(capabilities)"); caps = nullptr;
loaded_modules();
check(shutdown(), "NVSDK_NGX_CUDA_Shutdown"); initialized = false;
cuda.check(cuda.cuCtxSynchronize(), "cuCtxSynchronize");
cuda.check(cuda.cuCtxSetCurrent(nullptr), "cuCtxSetCurrent(NULL)");
cuda.check(cuda.cuDevicePrimaryCtxRelease_v2(device), "cuDevicePrimaryCtxRelease_v2"); context = nullptr;
std::printf("PASS NGX_INIT architecture=%s sr_available=1 transformer_validation=pending\n", d4r::diag::HipApi::target_arch());
return 0;
} catch (const std::exception& e) {
std::fprintf(stderr, "FAIL NGX_INIT %s\n", e.what());
loaded_modules();
return 4;
}
}
tools/windows/ngx_parameters.h
-56
View File
@@ -1,56 +0,0 @@
#pragma once
#include <cstdint>
struct ID3D12Resource;
extern "C" {
void d4r_ngx_set_ull(void*, const char*, unsigned long long);
void d4r_ngx_set_float(void*, const char*, float);
void d4r_ngx_set_uint(void*, const char*, unsigned);
void d4r_ngx_set_int(void*, const char*, int);
void d4r_ngx_set_void(void*, const char*, void*);
void d4r_ngx_set_d3d12_resource(void*, const char*, ID3D12Resource*);
unsigned d4r_ngx_get_ull(void*, const char*, unsigned long long*);
unsigned d4r_ngx_get_float(void*, const char*, float*);
unsigned d4r_ngx_get_uint(void*, const char*, unsigned*);
unsigned d4r_ngx_get_int(void*, const char*, int*);
unsigned d4r_ngx_get_void(void*, const char*, void**);
unsigned d4r_ngx_get_d3d12_resource(void*, const char*, ID3D12Resource**);
void* d4r_ngx_parameters_create();
void d4r_ngx_parameters_destroy(void*);
}
namespace d4r::ngx {
inline unsigned uint_value(void* p, const char* name, unsigned fallback = 0) {
unsigned value = fallback; (void)d4r_ngx_get_uint(p, name, &value); return value;
}
inline int int_value(void* p, const char* name, int fallback = 0) {
int value = fallback; (void)d4r_ngx_get_int(p, name, &value); return value;
}
inline float float_value(void* p, const char* name, float fallback = 0) {
float value = fallback; (void)d4r_ngx_get_float(p, name, &value); return value;
}
inline void copy_create(void* source, void* dest) {
for (const char* name : {"Width", "Height", "OutWidth", "OutHeight", "CreationNodeMask", "VisibilityNodeMask",
"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"}) {
unsigned value = 0;
if (d4r_ngx_get_uint(source, name, &value) == 1) d4r_ngx_set_uint(dest, name, value);
}
for (const char* name : {"PerfQualityValue", "DLSS.Feature.Create.Flags", "DLSS.Enable.Output.Subrects"}) {
int value = 0;
if (d4r_ngx_get_int(source, name, &value) == 1) d4r_ngx_set_int(dest, name, value);
}
}
inline void copy_frame(void* source, void* dest) {
for (const char* name : {"Jitter.Offset.X", "Jitter.Offset.Y", "Sharpness", "MV.Scale.X", "MV.Scale.Y",
"DLSS.Pre.Exposure", "DLSS.Exposure.Scale", "FrameTimeDeltaInMsec"}) {
const float fallback = name[0] == 'M' || name[0] == 'D' ? 1.f : name[0] == 'F' ? 16.666667f : 0.f;
d4r_ngx_set_float(dest, name, float_value(source, name, fallback));
}
for (const char* name : {"Reset", "Disable.Watermark", "DLSS.Indicator.Invert.X.Axis", "DLSS.Indicator.Invert.Y.Axis"})
d4r_ngx_set_int(dest, name, int_value(source, name));
for (const char* name : {"DLSS.Render.Subrect.Dimensions.Width", "DLSS.Render.Subrect.Dimensions.Height",
"DLSS.Input.Color.Subrect.Base.X", "DLSS.Input.Color.Subrect.Base.Y", "DLSS.Input.Depth.Subrect.Base.X",
"DLSS.Input.Depth.Subrect.Base.Y", "DLSS.Input.MV.Subrect.Base.X", "DLSS.Input.MV.Subrect.Base.Y",
"DLSS.Output.Subrect.Base.X", "DLSS.Output.Subrect.Base.Y"})
d4r_ngx_set_uint(dest, name, uint_value(source, name));
}
}
tools/windows/ngx_public_abi.h
-25
View File
@@ -1,25 +0,0 @@
#pragma once
#include <cstdint>
#include <string>
// Public NGX driver export ABI, also used by the upstream shim and OptiScaler.
// NVIDIA SDK headers or binaries are not build dependencies.
namespace d4r::ngx {
struct PathListInfo { const wchar_t* const* Path; unsigned Length; };
struct LoggingInfo { void* LoggingCallback; int MinimumLoggingLevel; bool DisableOtherLoggingSinks; };
struct FeatureCommonInfo { PathListInfo PathListInfo; void* InternalData; LoggingInfo LoggingInfo; };
struct ProjectDescription { const char* ProjectId; int EngineType; const char* EngineVersion; };
struct ApplicationIdentifier {
unsigned IdentifierType;
union Value { ProjectDescription ProjectDesc; unsigned long long ApplicationId; } v;
};
struct FeatureDiscoveryInfo {
unsigned SDKVersion, FeatureID;
ApplicationIdentifier Identifier;
const wchar_t* ApplicationDataPath;
const FeatureCommonInfo* FeatureInfo;
};
struct FeatureRequirement { unsigned FeatureSupported, MinHWArchitecture; char MinOSVersion[255]; };
struct ProjectIdentity { std::string id; int engine; std::string version; };
static_assert(sizeof(FeatureCommonInfo) == 40 && sizeof(FeatureDiscoveryInfo) == 56 && sizeof(FeatureRequirement) == 264);
}
tools/windows/ngx_windows_runtime.h
-652
View File
@@ -1,652 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include "d3d12_boundary_timing.h"
#include "pixel_conversion.h"
#include "cuda_image_api.h"
#include "ngx_parameters.h"
#include "ngx_public_abi.h"
#include "d3d12_command_hooks.h"
#include "d3d12_retirement.h"
#include <condition_variable>
#include <deque>
#include <thread>
#include <chrono>
#include <memory>
#include <mutex>
namespace d4r::win {
inline std::string env_path(const char* name) {
const char* value = std::getenv(name);
if (!value || !*value) throw std::runtime_error(std::string("Set the absolute path in ") + name);
if (!std::filesystem::path(diag::wide(value)).is_absolute()) throw std::runtime_error(std::string(name) + " must be absolute");
return value;
}
inline void ngx_check(unsigned result, const char* operation) {
if (result != 1 || !std::getenv("D4R_QUIET_API")) std::printf("D4R_NGX %s result=0x%08x\n", operation, result);
if (result != 1) throw std::runtime_error(std::string(operation) + " NGX=" + std::to_string(result));
}
inline ResourceAccess ngx_resource_access(void* parameters, const char* name, D3D12_RESOURCE_STATES fallback) {
ResourceAccess result(D3D12_RESOURCE_STATES(ngx::uint_value(parameters, name, unsigned(fallback))));
const std::string prefix(name);
result.enhanced = ngx::uint_value(parameters, (prefix + ".Enhanced").c_str()) != 0;
if (result.enhanced) {
result.layout = D3D12_BARRIER_LAYOUT(ngx::uint_value(parameters, (prefix + ".Layout").c_str()));
result.access = D3D12_BARRIER_ACCESS(ngx::uint_value(parameters, (prefix + ".Access").c_str()));
}
return result;
}
inline std::filesystem::path pixel_module_path() {
if (const char* path = std::getenv("D4R_FORMAT_MODULE")) return diag::wide(env_path("D4R_FORMAT_MODULE"));
HMODULE module = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
reinterpret_cast<LPCWSTR>(&pixel_module_path), &module)) throw std::runtime_error("Locate GPU conversion module directory");
wchar_t path[32768]{}; const DWORD length = GetModuleFileNameW(module, path, 32768);
if (!length || length == 32768) throw std::runtime_error("GPU conversion module path is invalid");
return std::filesystem::path(path).parent_path() / diag::wide(std::string("pixel_convert_") + diag::HipApi::target_arch() + ".hsaco");
}
struct Runtime {
diag::HipApi hip;
CudaApi cuda;
std::unique_ptr<diag::Library> nvapi;
diag::SearchDirectory featureSearch;
std::unique_ptr<diag::Library> dlss, core;
ExternalApi external;
cuda::ImageApi images;
ComPtr<ID3D12Device> device;
CUdevice ordinal = -1;
CUcontext context = nullptr;
bool initialized = false;
std::unique_ptr<SharedTimeline> timeline;
std::unique_ptr<PixelProgram> pixels;
std::mutex mutex;
// Features share one CUDA context. Follow submission order even when
// separate feature workers wake in a different order; otherwise a worker
// could hold the context while waiting for an earlier worker's output.
std::mutex async_mutex;
std::condition_variable async_changed;
std::deque<uint64_t> async_tickets;
std::string async_failure;
ComPtr<ID3D12CommandQueue> async_queue;
ngx::ProjectIdentity project;
ngx::FeatureCommonInfo common{};
std::vector<std::wstring> featurePaths;
std::vector<const wchar_t*> featurePathPointers;
using Init = unsigned(*)(unsigned long long, const wchar_t*, unsigned);
using Shutdown = unsigned(*)();
using Allocate = unsigned(*)(void**);
using Destroy = unsigned(*)(void*);
using Create = unsigned(*)(unsigned, void*, void**);
using Evaluate = unsigned(*)(void*, void*, void*);
using Release = unsigned(*)(void*);
Allocate allocate = nullptr, capabilities = nullptr;
Destroy destroy = nullptr;
Create create = nullptr;
Evaluate evaluate = nullptr;
Release release = nullptr;
Shutdown shutdown = nullptr;
Runtime(ID3D12Device* d3d, unsigned long long app, const wchar_t* data, unsigned sdk,
const ngx::FeatureCommonInfo* info = nullptr, const ngx::ProjectIdentity* identity = nullptr)
: hip(env_path("D4R_HIP_ROOT")), cuda(env_path("D4R_NVCUDA_DLL")),
featureSearch(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path()),
external{hip}, images(cuda), device(d3d) {
if (!d3d) throw std::runtime_error("Null D3D12 device");
try {
hipDeviceProp_t props{};
const auto luid = adapter_luid(d3d);
hip.select_architecture(-1, props, &luid);
hip.verbose = cuda.verbose = !std::getenv("D4R_QUIET_API");
if (std::memcmp(&luid, props.luid, sizeof(luid))) throw std::runtime_error("D3D12/HIP LUID mismatch");
cuda.check(cuda.cuInit(0), "cuInit(Windows runtime)");
int count = 0; cuda.check(cuda.cuDeviceGetCount(&count), "cuDeviceGetCount");
for (int i = 0; i < count; ++i) {
CUdevice candidate = -1; cuda.check(cuda.cuDeviceGet(&candidate, i), "cuDeviceGet");
int bus = -1, pciDevice = -1, domain = -1;
cuda.check(cuda.cuDeviceGetAttribute(&bus, 33, candidate), "PCI bus");
cuda.check(cuda.cuDeviceGetAttribute(&pciDevice, 34, candidate), "PCI device");
cuda.check(cuda.cuDeviceGetAttribute(&domain, 50, candidate), "PCI domain");
if (bus == props.pciBusID && pciDevice == props.pciDeviceID && domain == props.pciDomainID) ordinal = candidate;
}
if (ordinal < 0) throw std::runtime_error("CUDA/HIP PCI identity mismatch");
cuda.check(cuda.cuDevicePrimaryCtxRetain(&context, ordinal), "cuDevicePrimaryCtxRetain");
current();
timeline = std::make_unique<SharedTimeline>(external, d3d);
std::printf("D4R_INTEROP async=%u batch_inputs=%u cpu_image_copies=0\n",
unsigned(std::getenv("D4R_ASYNC_INTEROP") != nullptr), unsigned(std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr));
// NGX's loader can probe CUDA during DLL initialization: establish
// the correct primary context before loading either NVIDIA DLL.
nvapi = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NVAPI_DLL")));
dlss = std::make_unique<diag::Library>(diag::wide(env_path("D4R_DLSS_DLL")));
core = std::make_unique<diag::Library>(diag::wide(env_path("D4R_NGX_CORE")));
allocate = core->symbol<Allocate>("NVSDK_NGX_CUDA_AllocateParameters");
capabilities = core->symbol<Allocate>("NVSDK_NGX_CUDA_GetCapabilityParameters");
destroy = core->symbol<Destroy>("NVSDK_NGX_CUDA_DestroyParameters");
create = core->symbol<Create>("NVSDK_NGX_CUDA_CreateFeature");
evaluate = core->symbol<Evaluate>("NVSDK_NGX_CUDA_EvaluateFeature");
release = core->symbol<Release>("NVSDK_NGX_CUDA_ReleaseFeature");
shutdown = core->symbol<Shutdown>("NVSDK_NGX_CUDA_Shutdown");
if (identity) {
if (identity->id.empty()) throw std::runtime_error("Empty NGX project identity");
project = *identity;
featurePaths.push_back(std::filesystem::path(diag::wide(env_path("D4R_DLSS_DLL"))).parent_path().wstring());
if (info) {
if (info->PathListInfo.Length > 1024 || (info->PathListInfo.Length && !info->PathListInfo.Path)) throw std::runtime_error("Invalid NGX feature path list");
for (unsigned i = 0; i < info->PathListInfo.Length; ++i) if (info->PathListInfo.Path[i]) featurePaths.emplace_back(info->PathListInfo.Path[i]);
if (sdk >= 0x14) common.LoggingInfo = info->LoggingInfo;
}
for (const auto& path : featurePaths) featurePathPointers.push_back(path.c_str());
common.PathListInfo = {featurePathPointers.data(), unsigned(featurePathPointers.size())};
using ProjectInit = unsigned(*)(const char*, int, const char*, const wchar_t*, unsigned, const ngx::FeatureCommonInfo*);
std::printf("D4R_IDENTITY project=%s engine=%d version=%s sdk=0x%x\n", project.id.c_str(),project.engine,project.version.c_str(),sdk);
ngx_check(core->symbol<ProjectInit>("NVSDK_NGX_CUDA_Init_ProjectID")(project.id.c_str(),project.engine,project.version.c_str(),data,sdk,&common), "CUDA Init_ProjectID");
} else {
std::printf("D4R_IDENTITY application=%llu sdk=0x%x\n", app,sdk);
ngx_check(core->symbol<Init>("NVSDK_NGX_CUDA_Init")(app, data, sdk), "CUDA Init(driver ABI)");
}
initialized = true;
void* caps = nullptr;
ngx_check(capabilities(&caps), "CUDA feature capabilities");
if (!caps) throw std::runtime_error("Null CUDA capability parameters");
const int available = ngx::int_value(caps, "SuperSampling.Available", -1);
const int initResult = ngx::int_value(caps, "SuperSampling.FeatureInitResult", -1);
const int needsDriver = ngx::int_value(caps, "SuperSampling.NeedsUpdatedDriver", -1);
std::printf("D4R_CAPABILITIES available=%d feature_init=0x%08x needs_driver=%d\n", available, unsigned(initResult), needsDriver);
ngx_check(destroy(caps), "Destroy capability parameters");
if (available != 1 || initResult != 1 || needsDriver != 0)
throw std::runtime_error("CUDA DLSS is unavailable after capability initialization");
std::printf("D4R_RUNTIME platform=Windows architecture=%s cpu_frame_copies=0 frame_age=0\n", d4r::diag::HipApi::target_arch());
} catch (...) { cleanup(); throw; }
}
Runtime(const Runtime&) = delete;
~Runtime() { cleanup(); }
void current() { cuda.check(cuda.cuCtxSetCurrent(context), "cuCtxSetCurrent(runtime)"); }
void drain_async_queue() {
ComPtr<ID3D12CommandQueue> queue;
{ std::lock_guard<std::mutex> lock(async_mutex); queue = async_queue; }
if (!queue) return;
// Creation/free/import may synchronize the AMD device internally.
// Drain queued D3D12 consumers before taking the CUDA mutex: workers
// must remain able to satisfy any already-published output waits.
ComPtr<ID3D12Fence> completion;
dx(device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(completion.GetAddressOf())), "Reconfigure fence");
Handle event; event.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!event.value) throw std::runtime_error("Reconfigure event");
dx(queue->Signal(completion.Get(), 1), "Reconfigure queue Signal");
if (completion->GetCompletedValue() < 1) {
dx(completion->SetEventOnCompletion(1, event.value), "Reconfigure completion event");
if (WaitForSingleObject(event.value, 30000) != WAIT_OBJECT_0) throw std::runtime_error("Reconfigure GPU timeout");
}
dx(device->GetDeviceRemovedReason(), "Reconfigure device status");
}
void cleanup() noexcept {
if (context) (void)cuda.cuCtxSetCurrent(context);
if (initialized) { (void)shutdown(); initialized = false; }
timeline.reset();
pixels.reset();
if (context) {
(void)cuda.cuCtxSynchronize(); (void)cuda.cuCtxSetCurrent(nullptr);
(void)cuda.cuDevicePrimaryCtxRelease_v2(ordinal); context = nullptr;
}
}
};
class Feature {
std::shared_ptr<Runtime> rt_;
void* parameters_ = nullptr;
void* handle_ = nullptr;
CUdeviceptr scratch_ = 0;
ComPtr<ID3D12CommandQueue> queue_;
// A boundary drains both copies before returning. Reuse their allocators
// only after that completion; no game allocator is retained here.
ComPtr<ID3D12CommandAllocator> input_allocator_, output_allocator_;
ComPtr<ID3D12GraphicsCommandList> input_list_, output_list_;
struct CopySlot {
ComPtr<ID3D12CommandAllocator> input_allocator, output_allocator;
ComPtr<ID3D12GraphicsCommandList> input, output;
std::unique_ptr<BoundaryTiming> timing;
uint64_t done = 0;
};
struct AsyncFrame {
void* parameters = d4r_ngx_parameters_create();
std::vector<ComPtr<ID3D12Resource>> resources;
SharedTimeline::Ticket ticket{};
~AsyncFrame() { if (parameters) d4r_ngx_parameters_destroy(parameters); }
};
std::vector<std::unique_ptr<CopySlot>> copy_slots_;
ComPtr<ID3D12Fence> copy_completion_;
Handle copy_event_;
uint64_t copy_value_ = 0;
std::mutex jobs_mutex_;
std::mutex submit_mutex_;
std::condition_variable jobs_changed_;
std::deque<std::shared_ptr<AsyncFrame>> jobs_;
std::thread worker_;
bool worker_stop_ = false, worker_active_ = false;
std::string worker_failure_;
struct Plane {
ComPtr<ID3D12Resource> texture;
D3D12_RESOURCE_DESC desc{};
std::unique_ptr<SharedPlane> shared;
std::unique_ptr<cuda::Image> image;
std::unique_ptr<PixelAllocation> canonical;
PixelSpec spec{};
} planes_[5];
void cleanup() noexcept {
(void)rt_->cuda.cuCtxSetCurrent(rt_->context);
(void)rt_->cuda.cuCtxSynchronize();
if (handle_) { (void)rt_->release(handle_); handle_ = nullptr; }
for (auto& plane : planes_) {
plane.image.reset(); plane.canonical.reset(); plane.shared.reset(); plane.texture.Reset();
}
if (scratch_) { (void)rt_->cuda.cuMemFree_v2(scratch_); scratch_ = 0; }
if (parameters_) { (void)rt_->destroy(parameters_); parameters_ = nullptr; }
}
public:
unsigned width = 0, height = 0;
explicit Feature(std::shared_ptr<Runtime> runtime, void* parameters) : rt_(std::move(runtime)) {
if (std::getenv("D4R_ASYNC_INTEROP")) rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current();
try {
ngx_check(rt_->allocate(&parameters_), "CUDA AllocateParameters");
ngx::copy_create(parameters, parameters_);
d4r_ngx_set_int(parameters_, "DLSS.Enable.Output.Subrects", ngx::int_value(parameters, "DLSS.Enable.Output.Subrects"));
width = ngx::uint_value(parameters, "Width"); height = ngx::uint_value(parameters, "Height");
std::printf("D4R_CREATE source=%ux%u->%ux%u cuda=%ux%u->%ux%u\n", width, height,
ngx::uint_value(parameters, "OutWidth"), ngx::uint_value(parameters, "OutHeight"),
ngx::uint_value(parameters_, "Width"), ngx::uint_value(parameters_, "Height"),
ngx::uint_value(parameters_, "OutWidth"), ngx::uint_value(parameters_, "OutHeight"));
if (!width || !height) throw std::runtime_error("Zero NGX input dimensions");
rt_->cuda.check(rt_->cuda.cuMemAlloc_v2(&scratch_, 64ull * 1024 * 1024), "NGX VRAM scratch");
d4r_ngx_set_void(parameters_, "Scratch", reinterpret_cast<void*>(uintptr_t(scratch_)));
d4r_ngx_set_ull(parameters_, "Scratch.SizeInBytes", 64ull * 1024 * 1024);
ngx_check(rt_->create(1, parameters_, &handle_), "CUDA CreateFeature");
} catch (...) { cleanup(); throw; }
}
~Feature() {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_stop_ = true; }
jobs_changed_.notify_all();
if (worker_.joinable()) worker_.join();
if (copy_completion_) {
try { wait_copies(copy_value_); report_completed_copies(); }
catch (const std::exception& e) { std::fprintf(stderr, "D4R_ASYNC_CLEANUP_FAILURE %s\n", e.what()); }
}
std::lock_guard<std::mutex> lock(rt_->mutex); cleanup();
}
Feature(const Feature&) = delete;
void drain() {
if (!std::getenv("D4R_ASYNC_INTEROP")) return;
wait_worker();
if (copy_value_) wait_copies(copy_value_);
report_completed_copies();
rt_->drain_async_queue();
}
// Explicit harness boundary: all earlier producers must already be queued.
// The forthcoming command-list backend calls the same stages at queue submit.
void evaluate_boundary(ID3D12CommandQueue* queue, void* parameters) {
if (std::getenv("D4R_ASYNC_INTEROP")) { evaluate_async(queue, parameters); return; }
const auto boundaryBegin = std::chrono::steady_clock::now();
auto boundaryStage = boundaryBegin;
const bool profile = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profile) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - boundaryStage).count());
boundaryStage = std::chrono::steady_clock::now();
};
commands::InternalScope internal;
std::lock_guard<std::mutex> lock(rt_->mutex);
if (!queue) throw std::runtime_error("Null D3D12 queue");
if (queue_ && queue_.Get() != queue) throw std::runtime_error("A feature cannot switch queues without an explicit drain");
queue_ = queue;
ComPtr<ID3D12Device> queueDevice;
dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "GetDevice(queue)");
ComPtr<IUnknown> queueIdentity, runtimeIdentity;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(queueIdentity.GetAddressOf())), "Query queue device identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(runtimeIdentity.GetAddressOf())), "Query runtime device identity");
if (queueIdentity.Get() != runtimeIdentity.Get()) throw std::runtime_error("Queue belongs to a different D3D12 device");
rt_->current();
prepare(parameters);
mark("boundary_prepare");
ComPtr<ID3D12CommandAllocator> inputAllocator, outputAllocator;
ComPtr<ID3D12GraphicsCommandList> inputList, outputList;
auto list = [&](ComPtr<ID3D12CommandAllocator>& allocator, ComPtr<ID3D12GraphicsCommandList>& commandList) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "CreateCommandAllocator(interop)");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr,
IID_PPV_ARGS(commandList.GetAddressOf())), "CreateCommandList(interop)");
};
const bool cached = !std::getenv("D4R_DISABLE_INTEROP_LIST_CACHE");
if (cached) {
if (!input_list_) {
// Publish the pair only after both creations succeed.
list(inputAllocator, inputList); list(outputAllocator, outputList);
input_allocator_ = inputAllocator; output_allocator_ = outputAllocator;
input_list_ = inputList; output_list_ = outputList;
} else {
dx(input_allocator_->Reset(), "Reset(input interop allocator)");
dx(output_allocator_->Reset(), "Reset(output interop allocator)");
dx(input_list_->Reset(input_allocator_.Get(), nullptr), "Reset(input interop list)");
dx(output_list_->Reset(output_allocator_.Get(), nullptr), "Reset(output interop list)");
}
inputAllocator = input_allocator_; outputAllocator = output_allocator_;
inputList = input_list_; outputList = output_list_;
} else { list(inputAllocator, inputList); list(outputAllocator, outputList); }
mark("boundary_copy_list_setup");
record_inputs(inputList.Get(), parameters); dx(inputList->Close(), "Close(input copies)");
ID3D12CommandList* before[] = {inputList.Get()}; queue->ExecuteCommandLists(1, before);
mark("boundary_input_copy_submit");
try {
execute_cuda(queue);
mark("boundary_cuda_transaction");
record_output(outputList.Get(), parameters); dx(outputList->Close(), "Close(output copy)");
ID3D12CommandList* after[] = {outputList.Get()}; queue->ExecuteCommandLists(1, after);
mark("boundary_output_copy_submit");
rt_->timeline->drain(queue);
mark("boundary_output_copy_drain");
} catch (...) {
// Keep all recorded resources alive until submitted work is complete.
rt_->timeline->drain(queue); throw;
}
}
private:
void report_completed_copies() {
if (!copy_completion_ || !queue_) return;
const auto completed = copy_completion_->GetCompletedValue();
if (completed == UINT64_MAX) throw std::runtime_error("Device removed during boundary timing");
for (auto& slot : copy_slots_) if (slot->timing && slot->done && slot->done <= completed)
slot->timing->report(queue_.Get(), this);
}
void wait_copies(uint64_t value) {
if (copy_completion_->GetCompletedValue() >= value) return;
dx(copy_completion_->SetEventOnCompletion(value, copy_event_.value), "Async copy completion event");
if (WaitForSingleObject(copy_event_.value, 30000) != WAIT_OBJECT_0)
throw std::runtime_error("Async interop copy completion timeout");
}
void wait_worker() {
std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return jobs_.empty() && !worker_active_; });
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_);
}
CopySlot& copy_slot() {
report_completed_copies();
if (!copy_completion_) {
dx(rt_->device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(copy_completion_.GetAddressOf())), "Async copy fence");
copy_event_.value = CreateEventW(nullptr, FALSE, FALSE, nullptr);
if (!copy_event_.value) throw std::runtime_error("Async copy event");
}
CopySlot* selected = nullptr;
const auto completed = copy_completion_->GetCompletedValue();
for (auto& slot : copy_slots_) if (slot->done <= completed) { selected = slot.get(); break; }
if (!selected && copy_slots_.size() == 3) {
selected = copy_slots_.front().get();
for (auto& slot : copy_slots_) if (slot->done < selected->done) selected = slot.get();
// Bounded ownership; this wait is outside the runtime mutex.
wait_copies(selected->done);
report_completed_copies();
}
if (selected) {
// The fence can advance between report_completed_copies() and
// slot selection. Collect that now-complete ticket before Reset
// and before submitted() overwrites its timing identity.
if (selected->timing) selected->timing->report(queue_.Get(), this);
dx(selected->input_allocator->Reset(), "Async input allocator Reset");
dx(selected->output_allocator->Reset(), "Async output allocator Reset");
dx(selected->input->Reset(selected->input_allocator.Get(), nullptr), "Async input list Reset");
dx(selected->output->Reset(selected->output_allocator.Get(), nullptr), "Async output list Reset");
return *selected;
}
auto slot = std::make_unique<CopySlot>();
auto create = [&](auto& allocator, auto& list) {
dx(rt_->device->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_DIRECT, IID_PPV_ARGS(allocator.GetAddressOf())), "Async copy allocator");
dx(rt_->device->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_DIRECT, allocator.Get(), nullptr, IID_PPV_ARGS(list.GetAddressOf())), "Async copy list");
};
create(slot->input_allocator, slot->input); create(slot->output_allocator, slot->output);
if (std::getenv("D4R_PROFILE_GPU_BOUNDARY")) slot->timing = std::make_unique<BoundaryTiming>(rt_->device.Get());
copy_slots_.push_back(std::move(slot)); return *copy_slots_.back();
}
bool resize_needed(void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing async resource ") + names[i]);
continue;
}
const auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Async interop requires single-sample 2D textures");
if (!planes_[i].shared || planes_[i].desc.Width != desc.Width || planes_[i].desc.Height != desc.Height || planes_[i].desc.Format != desc.Format) return true;
}
return false;
}
void evaluate_async(ID3D12CommandQueue* queue, void* parameters) {
commands::InternalScope internal;
std::lock_guard<std::mutex> submitted(submit_mutex_);
if (!queue || (queue_ && queue_.Get() != queue)) throw std::runtime_error("Async interop requires one feature queue");
ComPtr<ID3D12Device> queueDevice; dx(queue->GetDevice(IID_PPV_ARGS(queueDevice.GetAddressOf())), "Async queue device");
ComPtr<IUnknown> a, b;
dx(queueDevice->QueryInterface(IID_PPV_ARGS(a.GetAddressOf())), "Async queue identity");
dx(rt_->device->QueryInterface(IID_PPV_ARGS(b.GetAddressOf())), "Async runtime identity");
if (a.Get() != b.Get()) throw std::runtime_error("Async queue device mismatch");
{ std::lock_guard<std::mutex> lock(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
{ std::lock_guard<std::mutex> lock(rt_->async_mutex);
if (!rt_->async_failure.empty()) throw std::runtime_error(rt_->async_failure);
if (rt_->async_queue && rt_->async_queue.Get() != queue) throw std::runtime_error("Async runtime features must share one D3D12 queue");
rt_->async_queue = queue; }
auto& slot = copy_slot();
if (resize_needed(parameters)) {
wait_worker(); if (copy_value_) wait_copies(copy_value_);
rt_->drain_async_queue();
std::lock_guard<std::mutex> lock(rt_->mutex);
rt_->current(); prepare(parameters);
}
// Steady-state recording touches immutable allocation metadata and the
// caller's resource snapshot. It does not contend with CUDA or mutate
// the worker's NGX parameters / texture references.
queue_ = queue;
auto frame = std::make_shared<AsyncFrame>();
if (!frame->parameters) throw std::runtime_error("Async frame parameters allocation");
ngx::copy_create(parameters, frame->parameters); ngx::copy_frame(parameters, frame->parameters);
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* resource = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &resource);
frame->resources.emplace_back(resource);
d4r_ngx_set_d3d12_resource(frame->parameters, names[i], resource);
}
if (slot.timing) slot.timing->input_begin(slot.input.Get());
record_inputs(slot.input.Get(), parameters);
if (slot.timing) slot.timing->input_end(slot.input.Get());
dx(slot.input->Close(), "Async input Close");
if (slot.timing) slot.timing->output_begin(slot.output.Get());
record_output(slot.output.Get(), parameters);
if (slot.timing) slot.timing->output_end(slot.output.Get());
dx(slot.output->Close(), "Async output Close");
if (!worker_.joinable()) worker_ = std::thread([this] { work(); });
ID3D12CommandList* input[] = {slot.input.Get()}, *output[] = {slot.output.Get()};
// Allocate the job before any GPU wait is published. Worker cannot
// acquire the runtime mutex until every queue dependency is enqueued.
std::lock_guard<std::mutex> pending(jobs_mutex_);
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
frame->ticket = rt_->timeline->reserve(); rt_->async_tickets.push_back(frame->ticket.output); }
try { jobs_.push_back(frame); }
catch (...) {
std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_back(); throw;
}
try {
queue->ExecuteCommandLists(1, input);
rt_->timeline->enqueue(queue, frame->ticket);
queue->ExecuteCommandLists(1, output);
slot.done = ++copy_value_;
dx(queue->Signal(copy_completion_.Get(), slot.done), "Async output-copy retirement Signal");
if (slot.timing) slot.timing->submitted(frame->ticket.output);
} catch (const std::exception& e) {
jobs_.pop_back(); worker_failure_ = e.what();
{ std::lock_guard<std::mutex> order(rt_->async_mutex);
rt_->async_tickets.pop_back(); rt_->async_failure = e.what(); }
(void)rt_->timeline->fence->Signal(frame->ticket.output);
jobs_changed_.notify_all(); rt_->async_changed.notify_all(); throw;
}
jobs_changed_.notify_one();
std::printf("D4R_ASYNC_SUBMIT ticket=%llu copy_slots=%zu cpu_frame_copies=0 frame_age=0\n",
(unsigned long long)frame->ticket.output, copy_slots_.size());
}
void work() noexcept {
commands::InternalScope internal;
for (;;) {
std::shared_ptr<AsyncFrame> frame;
{ std::unique_lock<std::mutex> lock(jobs_mutex_);
jobs_changed_.wait(lock, [&] { return worker_stop_ || !jobs_.empty(); });
if (jobs_.empty()) return;
frame = std::move(jobs_.front()); jobs_.pop_front(); worker_active_ = true; }
{ std::unique_lock<std::mutex> order(rt_->async_mutex);
rt_->async_changed.wait(order, [&] { return !rt_->async_tickets.empty() && rt_->async_tickets.front() == frame->ticket.output; }); }
try {
std::lock_guard<std::mutex> lock(rt_->mutex);
{ std::lock_guard<std::mutex> pending(jobs_mutex_);
if (!worker_failure_.empty()) throw std::runtime_error(worker_failure_); }
rt_->current(); prepare(frame->parameters); execute_cuda(nullptr, &frame->ticket);
} catch (const std::exception& e) {
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_failure_ = e.what(); }
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_failure = e.what(); }
std::fprintf(stderr, "D4R_ASYNC_INTEROP_FAILURE %s\n", e.what());
// Poison the feature and release the queue for error handling;
// subsequent evaluates fail explicitly, never reuse an old frame.
(void)rt_->timeline->fence->Signal(frame->ticket.output);
}
{ std::lock_guard<std::mutex> order(rt_->async_mutex); rt_->async_tickets.pop_front(); }
rt_->async_changed.notify_all();
frame.reset();
{ std::lock_guard<std::mutex> lock(jobs_mutex_); worker_active_ = false; }
jobs_changed_.notify_all();
}
}
void prepare(void* parameters) {
static const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i = 0; i < 5; ++i) {
ID3D12Resource* texture = nullptr;
(void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) {
if (i < 4) throw std::runtime_error(std::string("Missing resource ") + names[i]);
planes_[i].texture.Reset(); d4r_ngx_set_void(parameters_, names[i], nullptr); continue;
}
auto desc = resource_desc(texture);
if (desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || desc.SampleDesc.Count != 1 || desc.DepthOrArraySize != 1)
throw std::runtime_error("Interop baseline requires single-sample 2D textures");
const bool half = i == 0 || i == 2 || i == 3;
const unsigned channels = i == 0 || i == 3 ? 4 : i == 2 ? 2 : 1;
if (desc.Width > UINT32_MAX) throw std::runtime_error("Texture width exceeds GPU conversion ABI");
const PixelSpec spec = pixel_spec(i, desc.Format);
if (!pixel_supported(i, spec.storage)) throw std::runtime_error(std::string("Unsupported ") + names[i] + " DXGI format=" + std::to_string(desc.Format));
auto& plane = planes_[i];
if (!plane.shared || plane.desc.Width != desc.Width || plane.desc.Height != desc.Height || plane.desc.Format != desc.Format) {
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Synchronize(resize interop)");
plane.image.reset(); plane.canonical.reset(); plane.shared.reset();
plane.shared = std::make_unique<SharedPlane>(rt_->external, rt_->device.Get(), desc);
if (!spec.direct) {
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
plane.canonical = std::make_unique<PixelAllocation>(rt_->hip, unsigned(desc.Width), desc.Height, channels * (half ? 2 : 4));
}
plane.image = std::make_unique<cuda::Image>(rt_->images, unsigned(desc.Width), desc.Height, half ? 16 : 32, channels, i == 3, i == 0 ? 1 : 0);
plane.desc = desc; plane.spec = spec;
std::printf("D4R_FORMAT plane=%s dxgi=%u canonical_channels=%u canonical_bits=%u gpu_conversion=%u cpu_copy=0\n",
names[i], unsigned(desc.Format), channels, half ? 16u : 32u, !spec.direct);
}
plane.texture = texture;
d4r_ngx_set_void(parameters_, names[i], &plane.image->object);
}
ngx::copy_frame(parameters, parameters_);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Width")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Width", width);
if (!ngx::uint_value(parameters_, "DLSS.Render.Subrect.Dimensions.Height")) d4r_ngx_set_uint(parameters_, "DLSS.Render.Subrect.Dimensions.Height", height);
for (const char* name : {"TransparencyMask", "DLSS.Input.Bias.Current.Color.Mask"}) d4r_ngx_set_void(parameters_, name, nullptr);
}
void record_inputs(ID3D12GraphicsCommandList* list, void* parameters) {
const char* names[] = {"Color", "Depth", "MotionVectors", "Output", "ExposureTexture"};
for (unsigned i : {0u, 1u, 2u, 4u}) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, names[i], &texture);
if (!texture) continue;
static const char* states[] = {"D4R.Color.State", "D4R.Depth.State", "D4R.Motion.State", "D4R.Output.State", "D4R.Exposure.State"};
planes_[i].shared->copy_input(list, texture, ngx_resource_access(parameters, states[i], D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE));
}
}
void execute_cuda(ID3D12CommandQueue* queue, const SharedTimeline::Ticket* ticket = nullptr) {
const auto begin = std::chrono::steady_clock::now();
auto stage = begin;
const bool profileStages = std::getenv("D4R_PROFILE_STAGES") != nullptr;
auto mark = [&](const char* name) {
if (!profileStages) return;
const auto now = std::chrono::steady_clock::now();
std::printf("D4R_STAGE name=%s cpu_ms=%.6f\n", name,
std::chrono::duration<double, std::milli>(now - stage).count());
stage = std::chrono::steady_clock::now();
};
if (ticket) rt_->timeline->wait_ready(ticket->input); else rt_->timeline->wait_input(queue);
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input producer completion");
mark("input_fence_wait");
const bool batchInputs = std::getenv("D4R_BATCH_INPUT_COPIES") != nullptr;
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
auto& plane = planes_[i];
void* data = plane.shared->mapped; uint64_t pitch = plane.shared->footprint.Footprint.RowPitch;
if (plane.canonical) {
if (batchInputs) rt_->pixels->convert_deferred(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
else rt_->pixels->convert(false, data, pitch, plane.canonical->data, plane.canonical->pitch,
unsigned(plane.desc.Width), plane.desc.Height, plane.spec.storage, i);
data = plane.canonical->data; pitch = plane.canonical->pitch;
}
plane.image->upload_device(reinterpret_cast<uintptr_t>(data), pitch, batchInputs);
}
// HIP conversion and CUDA array copies are ordered on the shared
// legacy default stream. Retain an all-stream completion before NGX,
// including any internal nonblocking streams it might choose to use.
if (batchInputs) rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "Interop input batch completion");
mark("input_conversion_array_upload");
if (std::getenv("D4R_INTEROP_VERIFY")) {
for (unsigned i : {0u, 1u, 2u, 4u}) if (planes_[i].texture) {
uint32_t sample[8]{};
rt_->hip.check(rt_->hip.hipMemcpy(sample, planes_[i].shared->mapped,
std::min<size_t>(sizeof(sample), planes_[i].shared->bytes), hipMemcpyDeviceToHost), "DIAGNOSTIC external-memory sample");
std::printf("D4R_VRAM_VERIFY plane=%u words=%08x,%08x,%08x,%08x\n", i, sample[0], sample[1], sample[2], sample[3]);
const unsigned bytesPerPixel = i == 0 ? 8 : i == 2 ? 4 : 4;
std::vector<uint8_t> array(size_t(planes_[i].desc.Width) * planes_[i].desc.Height * bytesPerPixel);
planes_[i].image->download(array.data());
uint32_t first = 0; std::memcpy(&first, array.data(), 4);
std::printf("D4R_ARRAY_VERIFY plane=%u first=%08x\n", i, first);
}
}
ngx_check(rt_->evaluate(handle_, parameters_, nullptr), "CUDA EvaluateFeature");
mark("ngx_evaluate_host");
// NGX can use internal nonblocking streams. The default stream alone
// does not establish completion of those streams.
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "NGX all-stream completion");
mark("ngx_all_stream_completion");
auto& output = planes_[3];
output.image->download_device(reinterpret_cast<uintptr_t>(output.canonical ? output.canonical->data : output.shared->mapped),
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch);
rt_->cuda.check(rt_->cuda.cuCtxSynchronize(), "VRAM output completion");
mark("output_array_download");
if (std::getenv("D4R_VALIDATE_OUTPUT")) {
if (!rt_->pixels) rt_->pixels = std::make_unique<PixelProgram>(rt_->hip, pixel_module_path());
const auto counts = rt_->pixels->validate(output.canonical ? output.canonical->data : output.shared->mapped,
output.canonical ? output.canonical->pitch : output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height);
std::printf("D4R_OUTPUT_VALIDATION elements=%llu nan=%u inf=%u diagnostics_cpu_bytes=8\n", output.desc.Width * output.desc.Height * 4, counts[0], counts[1]);
if (counts[0] || counts[1]) throw std::runtime_error("DLSS output contains NaN/Inf");
}
mark("output_diagnostic");
if (output.canonical) rt_->pixels->convert(true, output.canonical->data, output.canonical->pitch, output.shared->mapped,
output.shared->footprint.Footprint.RowPitch, unsigned(output.desc.Width), output.desc.Height, output.spec.storage, 3);
if (ticket) rt_->timeline->signal_ready(ticket->output); else rt_->timeline->signal_output(queue);
mark("output_conversion_fence_signal");
std::printf("D4R_FRAME cpu_frame_copies=%u frame_age=0 interop_ngx_ms=%.3f\n", std::getenv("D4R_INTEROP_VERIFY") ? 1u : 0u,
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - begin).count());
}
void record_output(ID3D12GraphicsCommandList* list, void* parameters) {
ID3D12Resource* texture = nullptr; (void)d4r_ngx_get_d3d12_resource(parameters, "Output", &texture);
if (!texture) throw std::runtime_error("Missing output resource");
planes_[3].shared->copy_output(list, texture, ngx_resource_access(parameters, "D4R.Output.State", D3D12_RESOURCE_STATE_UNORDERED_ACCESS));
}
};
}
tools/windows/nvapi_trace.cpp
-113
View File
@@ -1,113 +0,0 @@
#include "diagnostic.h"
#if defined(D4R_NVAPI_COMPAT)
#include "hip_api.h"
#include <memory>
#include <mutex>
#include <atomic>
namespace {
// Public NVAPI ABI: NVIDIA/nvapi nvapi.h and nvapi_interface.h.
// AD100 describes the CUDA/NGX model profile, not the physical HIP target.
struct ArchInfo { unsigned version, architecture, implementation, revision; };
struct LogicalGpuData { unsigned version; void* os_adapter_id; unsigned count; void* physical[64]; unsigned reserved[8]; };
static_assert(sizeof(LogicalGpuData) == 568);
std::unique_ptr<d4r::diag::HipApi> hip;
hipDeviceProp_t properties{};
int device_ordinal = -1;
int physical_handle;
int logical_handle;
std::mutex init_mutex;
std::atomic<bool> runtime_ready{false};
bool valid(void* handle) { return runtime_ready.load(std::memory_order_acquire) && handle == &physical_handle; }
int __cdecl initialize()
{
try {
std::lock_guard<std::mutex> lock(init_mutex);
if (!runtime_ready.load(std::memory_order_acquire)) {
const char* root = std::getenv("HIP_PATH");
if (!root) return -5;
auto runtime = std::make_unique<d4r::diag::HipApi>(root);
device_ordinal = runtime->select_architecture(-1, properties);
hip = std::move(runtime);
runtime_ready.store(true, std::memory_order_release);
}
std::printf("NVAPI_COMPAT initialize physical=%s CUDA_model_profile=AD100\n", properties.gcnArchName);
return 0;
} catch (const std::exception& e) { std::fprintf(stderr, "NVAPI_COMPAT init failed: %s\n", e.what()); return -1; }
}
int __cdecl enumerate(void** handles, unsigned* count)
{
if (!handles || !count) return -5;
if (!runtime_ready.load(std::memory_order_acquire)) return -4;
handles[0] = &physical_handle; *count = 1;
std::printf("NVAPI_COMPAT EnumPhysicalGPUs count=1 HIP_ordinal=%d\n", device_ordinal);
return 0;
}
int __cdecl architecture(void* handle, ArchInfo* info)
{
if (!valid(handle) || !info) return -5;
if (info->version != (sizeof(ArchInfo) | 1u << 16) && info->version != (sizeof(ArchInfo) | 2u << 16)) return -9;
info->architecture = 0x190; info->implementation = 2; info->revision = 0x11;
std::printf("NVAPI_COMPAT GetArchInfo profile=AD102 physical=%s\n", properties.gcnArchName);
return 0;
}
int __cdecl adapter_id(void* handle, void* luid)
{
if (!valid(handle) || !luid) return -5;
std::memcpy(luid, properties.luid, sizeof(LUID));
std::printf("NVAPI_COMPAT GetAdapterIdFromPhysicalGpu matched_HIP_LUID=1\n");
return 0;
}
int __cdecl logical_from_physical(void* handle, void** logical)
{
if (!valid(handle) || !logical) return -5;
*logical = &logical_handle;
std::printf("NVAPI_COMPAT GetLogicalGPUFromPhysicalGPU count=1\n");
return 0;
}
int __cdecl logical_info(void* logical, LogicalGpuData* info)
{
std::printf("NVAPI_COMPAT GetLogicalGpuInfo handle_valid=%d info=%p version=0x%08x os_adapter_buffer=%p\n",
runtime_ready.load(std::memory_order_acquire) && logical == &logical_handle, static_cast<void*>(info), info ? info->version : 0,
info ? info->os_adapter_id : nullptr);
if (!runtime_ready.load(std::memory_order_acquire) || logical != &logical_handle || !info || !info->os_adapter_id) return -5;
if (info->version != (sizeof(LogicalGpuData) | 1u << 16)) return -9;
std::memcpy(info->os_adapter_id, properties.luid, sizeof(LUID));
info->count = 1;
std::memset(info->physical, 0, sizeof(info->physical));
std::memset(info->reserved, 0, sizeof(info->reserved));
info->physical[0] = &physical_handle;
std::printf("NVAPI_COMPAT GetLogicalGpuInfo matched_HIP_LUID=1\n");
return 0;
}
}
#endif
// Trace build forwards responses unchanged. The compatibility build adds the
// public topology APIs above; other queries still go to upstream ZLUDA NVAPI.
extern "C" __declspec(dllexport) void* __cdecl nvapi_QueryInterface(unsigned id)
{
try {
#if defined(D4R_NVAPI_COMPAT)
switch (id) {
case 0x0150e828: return reinterpret_cast<void*>(&initialize);
case 0xe5ac921f: return reinterpret_cast<void*>(&enumerate);
case 0xd8265d24: return reinterpret_cast<void*>(&architecture);
case 0x0ff07fde: return reinterpret_cast<void*>(&adapter_id);
case 0xadd604d1: return reinterpret_cast<void*>(&logical_from_physical);
case 0x842b066e: return reinterpret_cast<void*>(&logical_info);
}
#endif
const char* path = std::getenv("D4R_NVAPI_BACKEND");
if (!path || !*path) throw std::runtime_error("D4R_NVAPI_BACKEND absolute path required");
static d4r::diag::Library backend(d4r::diag::wide(path));
using Query = void* (__cdecl*)(unsigned);
static Query query = backend.symbol<Query>("nvapi_QueryInterface");
void* function = query(id);
std::printf("NVAPI_QUERY id=0x%08x supported=%d\n", id, function != nullptr);
return function;
} catch (const std::exception& e) {
std::fprintf(stderr, "NVAPI_TRACE_ERROR id=0x%08x %s\n", id, e.what());
return nullptr;
}
}
tools/windows/package_quick_test_sources.py
-99
View File
@@ -1,99 +0,0 @@
"""Separate corresponding-source asset for the lightweight Windows test ZIP.
Reads committed trees only, including the exact patched GPL OptiScaler tree and
its pinned submodules. Never copies a working directory or NVIDIA binaries.
"""
import argparse
import hashlib
import io
import json
from pathlib import Path
import subprocess
import zipfile
def git(root, *args):
return subprocess.check_output(["git", "-C", str(root), *args])
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2])
args = parser.parse_args()
repo = args.repo.resolve()
info = json.loads((repo / "dist/optiscaler-windows-d4r/build-info.json").read_text(encoding="utf-8-sig"))
opti = repo / "external/OptiScaler"
if info["sourceDirty"] or git(opti, "status", "--porcelain").strip():
raise RuntimeError("OptiScaler must match its clean binary source tree")
if git(opti, "rev-parse", "HEAD").decode().strip() != info["sourceCommit"]:
raise RuntimeError("OptiScaler source does not match the binary")
if git(repo, "status", "--porcelain").strip():
raise RuntimeError("Commit the packaging sources before publishing")
args.output.parent.mkdir(parents=True, exist_ok=True)
if args.output.exists():
raise RuntimeError("Use a new output archive")
provenance = {"d4rCommit": git(repo, "rev-parse", "HEAD").decode().strip(),
"optiScalerBinary": info["dllSha256"], "sourceTrees": [], "excludedNvidiaBinaries": []}
with zipfile.ZipFile(args.output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as output:
def tree(root, revision, prefix):
provenance["sourceTrees"].append({"path": prefix, "commit": revision})
raw = git(root, "archive", "--format=zip", revision)
with zipfile.ZipFile(io.BytesIO(raw)) as source:
for item in source.infolist():
if item.is_dir():
continue
name = prefix + "/" + item.filename
lowered = name.lower()
nvidia_binary = ("/nvapi/" in lowered or "nvngx" in Path(lowered).name) and Path(lowered).suffix in {
".dll", ".exe", ".lib", ".bin", ".cubin", ".fatbin", ".pdb"}
if nvidia_binary:
provenance["excludedNvidiaBinaries"].append(name)
continue
output.writestr(name, source.read(item))
for entry in git(root, "ls-tree", "-rz", revision).split(b"\0"):
if not entry:
continue
header, path = entry.split(b"\t", 1)
mode, _, commit = header.split()
if mode != b"160000":
continue
child = path.decode("utf-8")
child_root = root / child
child_commit = commit.decode()
if git(child_root, "rev-parse", "HEAD").decode().strip() != child_commit:
raise RuntimeError(f"Submodule does not match committed source: {child}")
if git(child_root, "status", "--porcelain").strip():
raise RuntimeError(f"Dirty source submodule: {child}")
tree(child_root, child_commit, prefix + "/" + child)
tree(repo, provenance["d4rCommit"], "d4r")
tree(opti, info["sourceCommit"], "OptiScaler")
output.writestr("source-provenance.json", json.dumps(provenance, indent=2) + "\n")
output.writestr("READ-ME-SOURCES.txt", """CORRESPONDING SOURCES -- NOT NEEDED TO TEST THE GAME
d4r/ contains the committed Windows port, build scripts, dependency patches
and licenses. OptiScaler/ contains the exact modified GPL-3.0 source tree and
its pinned source submodules corresponding to the distributed OptiScaler.dll.
See source-provenance.json for exact commits and the binary SHA256.
Build instructions: d4r/docs/windows-rdna4-port.md and
d4r/patches/optiscaler/README.md. The normal build script retrieves the pinned
upstream checkout, submodules and applies the exported source patches.
NVIDIA import-library binaries are intentionally excluded from this archive;
the pinned public NVIDIA/nvapi SDK used by the builder supplies them locally.
No proprietary NVIDIA runtime DLL or NVIDIA-derived private kernel is included.
The separately bundled ZLUDA/LLVM changes are in d4r/patches/zluda and
d4r/patches/zluda-llvm, with upstream source commits and build instructions.
""")
with args.output.open("rb") as stream:
sha256 = hashlib.file_digest(stream, "sha256").hexdigest()
print(json.dumps({"archive": str(args.output), "bytes": args.output.stat().st_size,
"sha256": sha256,
"sourceTrees": len(provenance["sourceTrees"])}))
if __name__ == "__main__":
main()
tools/windows/pixel_conversion.h
-109
View File
@@ -1,109 +0,0 @@
#pragma once
#include "d3d12_external.h"
#include "pixel_format.h"
#include <array>
namespace d4r::win {
struct PixelSpec {
pixel::Storage storage;
unsigned bytes;
bool direct;
};
inline PixelSpec pixel_spec(unsigned plane, DXGI_FORMAT format) {
using namespace pixel;
// Typeless resources use the corresponding floating-point/UNORM view,
// as NGX's resource-only ABI provides no SRV descriptor to disambiguate.
switch (format) {
case DXGI_FORMAT_R16G16B16A16_FLOAT: case DXGI_FORMAT_R16G16B16A16_TYPELESS:
return {rgba16f, 8, plane == color || plane == output};
case DXGI_FORMAT_R32G32B32A32_FLOAT: return {rgba32f, 16, false};
case DXGI_FORMAT_R11G11B10_FLOAT: return {r11g11b10f, 4, false};
case DXGI_FORMAT_R10G10B10A2_UNORM: case DXGI_FORMAT_R10G10B10A2_TYPELESS: return {rgb10a2, 4, false};
case DXGI_FORMAT_R8G8B8A8_UNORM: case DXGI_FORMAT_R8G8B8A8_TYPELESS: return {rgba8, 4, false};
case DXGI_FORMAT_B8G8R8A8_UNORM: case DXGI_FORMAT_B8G8R8A8_TYPELESS: return {bgra8, 4, false};
case DXGI_FORMAT_R16G16_FLOAT: case DXGI_FORMAT_R16G16_TYPELESS: return {rg16f, 4, plane == motion};
case DXGI_FORMAT_R32G32_FLOAT: case DXGI_FORMAT_R32G32_TYPELESS: return {rg32f, 8, false};
case DXGI_FORMAT_R16G16_SNORM: return {rg16snorm, 4, false};
case DXGI_FORMAT_R16G16_UNORM: return {rg16unorm, 4, false};
case DXGI_FORMAT_R32_FLOAT: case DXGI_FORMAT_R32_TYPELESS: case DXGI_FORMAT_D32_FLOAT:
return {r32f, 4, plane == depth || plane == exposure};
case DXGI_FORMAT_R32G8X24_TYPELESS: case DXGI_FORMAT_D32_FLOAT_S8X24_UINT: case DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS:
if (plane != depth) throw std::runtime_error("Depth/stencil resources are supported only for the depth input");
return {r32f, 4, true}; // D3D12 subresource 0 is the R32 depth plane.
case DXGI_FORMAT_R16_FLOAT: return {r16f, 2, false};
case DXGI_FORMAT_D24_UNORM_S8_UINT: case DXGI_FORMAT_R24G8_TYPELESS: case DXGI_FORMAT_R24_UNORM_X8_TYPELESS:
return {depth24, 4, false};
case DXGI_FORMAT_R16_TYPELESS: return {plane == exposure ? r16f : r16unorm, 2, false};
case DXGI_FORMAT_D16_UNORM: case DXGI_FORMAT_R16_UNORM:
return {r16unorm, 2, false};
default: throw std::runtime_error("No validated GPU conversion for DXGI format=" + std::to_string(format));
}
}
inline bool pixel_supported(unsigned plane, pixel::Storage storage) {
using namespace pixel;
switch (plane) {
case color: case output: return storage <= rgb10a2;
case motion: return storage == rgba16f || storage == rgba32f || (storage >= rg16f && storage <= rg16unorm);
case depth: return storage == r32f || storage == depth24 || storage == r16unorm;
case exposure: return storage == r32f || storage == r16f || storage == rgba16f || storage == rgba32f;
default: return false;
}
}
class PixelProgram {
diag::HipApi& hip_;
hipModule_t module_ = nullptr;
hipFunction_t decode_ = nullptr, encode_ = nullptr;
hipFunction_t validate_ = nullptr;
void* counters_ = nullptr;
public:
PixelProgram(diag::HipApi& hip, const std::filesystem::path& module) : hip_(hip) {
try {
diag::require_code_object_target(module, diag::HipApi::target_arch());
hip.check(hip.hipModuleLoad(&module_, diag::utf8(module.c_str()).c_str()), "Load native GPU format conversion");
hip.check(hip.hipModuleGetFunction(&decode_, module_, "d4r_pixel_decode"), "Resolve GPU pixel decode");
hip.check(hip.hipModuleGetFunction(&encode_, module_, "d4r_pixel_encode"), "Resolve GPU pixel encode");
} catch (...) { if (module_) (void)hip.hipModuleUnload(module_); throw; }
}
~PixelProgram() { if (counters_) (void)hip_.hipFree(counters_); if (module_) (void)hip_.hipModuleUnload(module_); }
PixelProgram(const PixelProgram&) = delete;
std::array<unsigned,2> validate(const void* source, uint64_t pitch, unsigned width, unsigned height) {
if (!width || !height || uint64_t(width) * height > UINT32_MAX / 4) throw std::runtime_error("Invalid output validation dimensions");
if (!validate_) hip_.check(hip_.hipModuleGetFunction(&validate_, module_, "d4r_pixel_validate"), "Resolve output validation kernel");
if (!counters_) hip_.check(hip_.hipMalloc(&counters_, 2 * sizeof(unsigned)), "Allocate validation counters");
std::array<unsigned,2> result{};
hip_.check(hip_.hipMemcpy(counters_, result.data(), sizeof(result), hipMemcpyHostToDevice), "Clear validation counters");
void* args[] = {&source, &pitch, &width, &height, &counters_};
hip_.check(hip_.hipModuleLaunchKernel(validate_, unsigned((uint64_t(width) * height + 255) / 256),1,1,256,1,1,0,nullptr,args,nullptr), "Validate output on GPU");
hip_.check(hip_.hipDeviceSynchronize(), "Output validation completion");
hip_.check(hip_.hipMemcpy(result.data(), counters_, sizeof(result), hipMemcpyDeviceToHost), "Read validation counters");
return result;
}
void convert_deferred(bool encode, const void* source, uint64_t sourcePitch, void* dest, uint64_t destPitch,
unsigned width, unsigned height, pixel::Storage format, unsigned plane) {
if (!width || !height || uint64_t(width) * height > UINT32_MAX) throw std::runtime_error("Invalid GPU conversion dimensions");
unsigned storage = unsigned(format);
void* args[] = {&source, &sourcePitch, &dest, &destPitch, &width, &height, &storage, &plane};
hip_.check(hip_.hipModuleLaunchKernel(encode ? encode_ : decode_, unsigned((uint64_t(width) * height + 255) / 256), 1, 1,
256, 1, 1, 0, nullptr, args, nullptr), encode ? "GPU pixel encode" : "GPU pixel decode");
}
void convert(bool encode, const void* source, uint64_t sourcePitch, void* dest, uint64_t destPitch,
unsigned width, unsigned height, pixel::Storage format, unsigned plane) {
convert_deferred(encode, source, sourcePitch, dest, destPitch, width, height, format, plane);
// CUDA may use other nonblocking streams. Standalone conversions must
// establish completion before handing the allocation to that API.
hip_.check(hip_.hipDeviceSynchronize(), "GPU conversion completion");
}
};
class PixelAllocation {
diag::HipApi& hip_;
public:
void* data = nullptr;
uint64_t pitch;
PixelAllocation(diag::HipApi& hip, unsigned width, unsigned height, unsigned canonicalBytes) : hip_(hip) {
pitch = (uint64_t(width) * canonicalBytes + 255) & ~uint64_t(255);
hip.check(hip.hipMalloc(&data, size_t(pitch * height)), "Allocate canonical GPU pixels");
}
~PixelAllocation() { if (data) (void)hip_.hipFree(data); }
PixelAllocation(const PixelAllocation&) = delete;
};
}
tools/windows/pixel_convert_gfx1201.hip
-147
View File
@@ -1,147 +0,0 @@
#include "../../kernels/common/hip_device_minimal.h"
#include "pixel_format.h"
namespace {
using namespace d4r::pixel;
using u8 = unsigned char;
using u16 = unsigned short;
using u32 = unsigned;
using u64 = unsigned long long;
__device__ inline float as_float(u32 bits) { return __builtin_bit_cast(float, bits); }
__device__ inline u32 as_uint(float value) { return __builtin_bit_cast(u32, value); }
__device__ inline float from_half(u16 bits) {
u32 sign = u32(bits & 0x8000u) << 16, exp = (bits >> 10) & 31, m = bits & 1023;
if (!exp) return as_float(as_uint(float(m) * 0x1p-24f) | sign);
return as_float(sign | (exp == 31 ? 0x7f800000u : (exp + 112u) << 23) | (m << 13));
}
__device__ inline u16 to_half(float value) {
u32 bits = as_uint(value), sign = (bits >> 16) & 0x8000u, exponent = (bits >> 23) & 255u;
u32 mantissa = bits & 0x7fffffu;
if (exponent == 255) return u16(sign | 0x7c00u | (mantissa ? 0x200u : 0));
int e = int(exponent) - 112;
if (e >= 31) return u16(sign | 0x7c00u);
unsigned shift = 13;
if (e <= 0) {
if (e < -10) return u16(sign);
mantissa |= 0x800000u; shift = unsigned(14 - e); e = 0;
}
u32 kept = mantissa >> shift, remainder = mantissa & ((1u << shift) - 1), midpoint = 1u << (shift - 1);
u32 result = sign | (u32(e) << 10);
result += kept & (e == 0 ? 0x7ffu : 0x3ffu);
return u16(result + (remainder > midpoint || (remainder == midpoint && (kept & 1))));
}
__device__ inline u32 quantize(float value, unsigned maximum) {
if (!(value > 0)) return 0;
if (value >= 1) return maximum;
float scaled = value * float(maximum);
u32 integer = u32(scaled); float remainder = scaled - float(integer);
return integer + (remainder > .5f || (remainder == .5f && (integer & 1)));
}
__device__ inline u32 half_to_unsigned(u16 h, unsigned mantissa_bits) {
if ((h & 0x7c00u) == 0x7c00u && (h & 1023u)) return (31u << mantissa_bits) | (1u << (mantissa_bits - 1));
if (h & 0x8000u) return 0;
// D3D floating-point format narrowing truncates. RNE here differs from
// the driver's typed UAV store and can turn finite HDR values into Inf.
return h >> (10 - mantissa_bits);
}
__device__ inline unsigned pixel_bytes(Storage fmt) {
switch (fmt) {
case rgba32f: return 16;
case rgba16f: case rg32f: return 8;
case r16f: case r16unorm: return 2;
default: return 4;
}
}
__device__ inline void decode(const u8* p, Storage fmt, float* value) {
const auto* halves = reinterpret_cast<const u16*>(p);
const auto* singles = reinterpret_cast<const float*>(p);
value[0] = value[1] = value[2] = 0; value[3] = 1;
switch (fmt) {
case rgba16f: for (unsigned i = 0; i < 4; ++i) value[i] = from_half(halves[i]); break;
case rgba32f: for (unsigned i = 0; i < 4; ++i) value[i] = singles[i]; break;
case rgba8: case bgra8:
for (unsigned i = 0; i < 4; ++i) value[i] = float(p[i]) / 255.f;
if (fmt == bgra8) { float swap = value[0]; value[0] = value[2]; value[2] = swap; } break;
case r11g11b10f: {
u32 packed = *reinterpret_cast<const u32*>(p);
value[0] = from_half(u16((packed & 2047) << 4));
value[1] = from_half(u16(((packed >> 11) & 2047) << 4));
value[2] = from_half(u16((packed >> 22) << 5)); break;
}
case rgb10a2: {
u32 packed = *reinterpret_cast<const u32*>(p);
for (unsigned i = 0; i < 3; ++i) value[i] = float((packed >> (10 * i)) & 1023) / 1023.f;
value[3] = float(packed >> 30) / 3.f; break;
}
case rg16f: value[0] = from_half(halves[0]); value[1] = from_half(halves[1]); break;
case rg32f: value[0] = singles[0]; value[1] = singles[1]; break;
case rg16snorm:
for (unsigned i = 0; i < 2; ++i) {
int v = reinterpret_cast<const short*>(p)[i];
value[i] = v == -32768 ? -1.f : float(v) / 32767.f;
} break;
case rg16unorm: for (unsigned i = 0; i < 2; ++i) value[i] = float(halves[i]) / 65535.f; break;
case r16f: value[0] = from_half(halves[0]); break;
case r32f: value[0] = singles[0]; break;
case depth24: value[0] = float(*reinterpret_cast<const u32*>(p) & 0xffffffu) / 16777215.f; break;
case r16unorm: value[0] = float(halves[0]) / 65535.f; break;
default: break; // Host rejects formats which have no tested decoder.
}
}
__device__ inline void encode(u8* p, Storage fmt, const float* value) {
auto* halves = reinterpret_cast<u16*>(p);
auto* singles = reinterpret_cast<float*>(p);
switch (fmt) {
case rgba16f: for (unsigned i = 0; i < 4; ++i) halves[i] = to_half(value[i]); break;
case rgba32f: for (unsigned i = 0; i < 4; ++i) singles[i] = value[i]; break;
case rgba8: case bgra8:
for (unsigned i = 0; i < 4; ++i) p[i] = u8(quantize(value[i], 255));
if (fmt == bgra8) { u8 swap = p[0]; p[0] = p[2]; p[2] = swap; } break;
case r11g11b10f:
*reinterpret_cast<u32*>(p) = half_to_unsigned(to_half(value[0]), 6) |
(half_to_unsigned(to_half(value[1]), 6) << 11) | (half_to_unsigned(to_half(value[2]), 5) << 22); break;
case rgb10a2:
*reinterpret_cast<u32*>(p) = quantize(value[0], 1023) | (quantize(value[1], 1023) << 10) |
(quantize(value[2], 1023) << 20) | (quantize(value[3], 3) << 30); break;
default: break; // Only color/output formats are encoded in the runtime.
}
}
}
extern "C" __global__ void d4r_pixel_decode(const u8* source, u64 source_pitch, u8* dest, u64 dest_pitch,
unsigned width, unsigned height, unsigned format, unsigned plane) {
unsigned i = blockIdx.x * 256u + threadIdx.x;
if (u64(i) >= u64(width) * height) return;
unsigned x = i % width, y = i / width;
float value[4]; decode(source + u64(y) * source_pitch + u64(x) * pixel_bytes(Storage(format)), Storage(format), value);
if (plane == depth || plane == exposure) reinterpret_cast<float*>(dest + u64(y) * dest_pitch)[x] = value[0];
else {
unsigned channels = plane == motion ? 2 : 4;
auto* halves = reinterpret_cast<u16*>(dest + u64(y) * dest_pitch) + x * channels;
for (unsigned c = 0; c < channels; ++c) halves[c] = to_half(value[c]);
}
}
extern "C" __global__ void d4r_pixel_encode(const u8* source, u64 source_pitch, u8* dest, u64 dest_pitch,
unsigned width, unsigned height, unsigned format, unsigned plane) {
unsigned i = blockIdx.x * 256u + threadIdx.x;
if (u64(i) >= u64(width) * height) return;
unsigned x = i % width, y = i / width;
const auto* halves = reinterpret_cast<const u16*>(source + u64(y) * source_pitch) + x * 4;
float value[4]; for (unsigned c = 0; c < 4; ++c) value[c] = from_half(halves[c]);
encode(dest + u64(y) * dest_pitch + u64(x) * pixel_bytes(Storage(format)), Storage(format), value);
}
// Diagnostic scan of the canonical RGBA16F output; no image readback.
extern "C" __global__ void d4r_pixel_validate(const u8* source, u64 pitch,
unsigned width, unsigned height, unsigned* counters) {
unsigned i = blockIdx.x * 256u + threadIdx.x;
if (u64(i) >= u64(width) * height) return;
const auto* values = reinterpret_cast<const u16*>(source + u64(i / width) * pitch) + (i % width) * 4;
unsigned nans = 0, infinities = 0;
for (unsigned c = 0; c < 4; ++c) if ((values[c] & 0x7c00u) == 0x7c00u) {
if (values[c] & 0x3ffu) ++nans; else ++infinities;
}
if (nans) __atomic_fetch_add(counters, nans, __ATOMIC_RELAXED);
if (infinities) __atomic_fetch_add(counters + 1, infinities, __ATOMIC_RELAXED);
}
tools/windows/pixel_format.h
-12
View File
@@ -1,12 +0,0 @@
#pragma once
// Numeric storage descriptions shared by the native HIP kernel and Windows
// host. These are independent of Win32, Wine, KFD and NVIDIA SDK headers.
namespace d4r::pixel {
enum Storage : unsigned {
rgba16f, rgba32f, rgba8, bgra8, r11g11b10f, rgb10a2,
rg16f, rg32f, rg16snorm, rg16unorm, r16f, r32f,
depth24, r16unorm, rgba8srgb, bgra8srgb
};
enum Plane : unsigned { color, depth, motion, output, exposure };
}
tools/windows/player_config.h
-312
View File
@@ -1,312 +0,0 @@
#pragma once
// Drag-in install, as on Linux: the game folder holds dxgi.dll (OptiScaler),
// OptiScaler.ini and a d4r folder containing this shim and d4r.ini. When the
// launcher has not set D4R_HIP_ROOT and d4r.ini exists beside this DLL, the
// shim derives the backend's environment itself, so the game can be started
// from Steam, Epic or a shortcut. Without d4r.ini nothing changes: diagnostic
// runs keep using the launcher's environment.
//
// Codegen switches are fixed to the Windows-validated values; d4r.ini only
// selects the model (K/M), diagnostics and the cache folder. NVIDIA DLLs must
// match an identity in dll-pins.txt before anything is loaded.
#include "hip_api.h"
#include "ngx_parameters.h"
#include <bcrypt.h>
#include <fstream>
#include <mutex>
#include <optional>
#include <sstream>
namespace d4r::win::player {
struct IniEntry { std::string section, key, value; };
struct Pin { std::string kind, sha256, version, status; };
struct Settings {
unsigned preset = 11; // K
bool asyncInterop = false, validateOutput = false, verbose = false, showWatermark = false;
std::wstring cacheDir;
};
struct State {
bool active = false;
unsigned preset = 0;
std::filesystem::path dir;
std::vector<std::string> notes;
};
inline State& state() { static State value; return value; }
inline std::string trim(const std::string& value) {
const auto first = value.find_first_not_of(" \t\r\n");
if (first == std::string::npos) return {};
return value.substr(first, value.find_last_not_of(" \t\r\n") - first + 1);
}
inline std::string lower(std::string value) {
for (char& c : value) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
return value;
}
inline std::string upper(std::string value) {
for (char& c : value) c = static_cast<char>(std::toupper(static_cast<unsigned char>(c)));
return value;
}
inline std::filesystem::path module_directory() {
HMODULE module = nullptr;
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
reinterpret_cast<LPCWSTR>(&module_directory), &module)) throw std::runtime_error("Locate the d4r shim directory");
wchar_t path[32768]{}; const DWORD length = GetModuleFileNameW(module, path, 32768);
if (!length || length == 32768) throw std::runtime_error("d4r shim path is invalid");
return std::filesystem::path(path).parent_path();
}
inline std::string read_text(const std::filesystem::path& path) {
std::ifstream file(path, std::ios::binary);
if (!file) throw std::runtime_error("Cannot read " + diag::utf8(path.c_str()));
std::stringstream text; text << file.rdbuf(); return text.str();
}
// The configparser dialect of packaging/d4r.ini and the Linux shim: [Section],
// key = value, full-line ; or # comments, inline comments after whitespace.
inline std::vector<IniEntry> parse_ini(const std::string& text) {
std::vector<IniEntry> entries; std::string section;
size_t start = text.compare(0, 3, "\xEF\xBB\xBF") == 0 ? 3 : 0;
while (start < text.size()) {
size_t end = text.find('\n', start);
if (end == std::string::npos) end = text.size();
const std::string line = trim(text.substr(start, end - start));
start = end + 1;
if (line.empty() || line[0] == ';' || line[0] == '#') continue;
if (line[0] == '[') { section = trim(line.substr(1, line.find(']') - 1)); continue; }
const size_t equals = line.find('=');
if (equals == std::string::npos) continue;
std::string value = line.substr(equals + 1);
for (size_t i = 1; i < value.size(); ++i)
if ((value[i] == ';' || value[i] == '#') && (value[i - 1] == ' ' || value[i - 1] == '\t')) { value.resize(i); break; }
entries.push_back({section, trim(line.substr(0, equals)), trim(value)});
}
return entries;
}
// Invalid values fall back to the safe default and are reported in the log
// (and by d4r-check.ps1), rather than silently changing behaviour.
inline Settings read_settings(const std::vector<IniEntry>& ini, std::vector<std::string>& notes) {
Settings settings;
const auto flag = [&](const IniEntry& entry, bool fallback) {
const auto value = lower(entry.value);
if (value.empty() || value == "auto") return fallback;
if (value == "1" || value == "true" || value == "yes" || value == "on") return true;
if (value == "0" || value == "false" || value == "no" || value == "off") return false;
notes.push_back("d4r.ini: [" + entry.section + "] " + entry.key + " must be true or false, not '" + entry.value +
"'; using " + (fallback ? "true" : "false"));
return fallback;
};
for (const auto& entry : ini) {
const auto name = lower(entry.section + "." + entry.key); // Matched case-insensitively, reported as written.
if (name == "dlss.model") {
const auto value = upper(entry.value);
if (value.empty() || value == "AUTO" || value == "K" || value == "DLSS4" || value == "11") settings.preset = 11;
else if (value == "M" || value == "DLSS4.5" || value == "13") settings.preset = 13;
else notes.push_back("d4r.ini: [DLSS] Model '" + entry.value + "' is not available on Windows (use K or M); using K");
} else if (name == "dlss.showwatermark") settings.showWatermark = flag(entry, false);
else if (name == "interop.asyncinterop") settings.asyncInterop = flag(entry, false);
else if (name == "debug.validateoutput") settings.validateOutput = flag(entry, false);
else if (name == "debug.log") {
const auto value = lower(entry.value);
if (value == "verbose") settings.verbose = true;
else if (!value.empty() && value != "normal" && value != "auto")
notes.push_back("d4r.ini: [Debug] Log must be normal or verbose, not '" + entry.value + "'; using normal");
} else if (name == "paths.cachedir") {
if (!entry.value.empty() && lower(entry.value) != "auto") settings.cacheDir = diag::wide(entry.value);
} else notes.push_back("d4r.ini: [" + entry.section + "] " + entry.key + " is not a Windows setting; ignored");
}
return settings;
}
// "kind sha256 version status" lines; kind is dlss or ngx.
inline std::vector<Pin> parse_pins(const std::string& text) {
std::vector<Pin> pins; std::istringstream lines(text); std::string line;
while (std::getline(lines, line)) {
line = trim(line);
if (line.empty() || line[0] == '#') continue;
std::istringstream fields(line); Pin pin;
if (!(fields >> pin.kind >> pin.sha256 >> pin.version >> pin.status) || pin.sha256.size() != 64 ||
(pin.kind != "dlss" && pin.kind != "ngx")) throw std::runtime_error("dll-pins.txt is damaged; extract the d4r ZIP again");
pin.sha256 = upper(pin.sha256); pins.push_back(pin);
}
return pins;
}
inline std::string sha256(const std::filesystem::path& path) {
std::ifstream file(path, std::ios::binary);
if (!file) return {};
BCRYPT_ALG_HANDLE algorithm = nullptr; BCRYPT_HASH_HANDLE hash = nullptr;
struct Cleanup { BCRYPT_ALG_HANDLE& a; BCRYPT_HASH_HANDLE& h;
~Cleanup() { if (h) BCryptDestroyHash(h); if (a) BCryptCloseAlgorithmProvider(a, 0); } } cleanup{algorithm, hash};
if (BCryptOpenAlgorithmProvider(&algorithm, BCRYPT_SHA256_ALGORITHM, nullptr, 0) < 0 ||
BCryptCreateHash(algorithm, &hash, nullptr, 0, nullptr, 0, 0) < 0) throw std::runtime_error("SHA256 unavailable");
std::vector<char> buffer(1 << 20);
while (file) {
file.read(buffer.data(), std::streamsize(buffer.size()));
if (file.gcount() > 0 && BCryptHashData(hash, reinterpret_cast<PUCHAR>(buffer.data()), ULONG(file.gcount()), 0) < 0)
throw std::runtime_error("SHA256 failed");
}
unsigned char digest[32]{};
if (BCryptFinishHash(hash, digest, sizeof(digest), 0) < 0) throw std::runtime_error("SHA256 failed");
static const char hex[] = "0123456789ABCDEF"; std::string text;
for (unsigned char byte : digest) { text += hex[byte >> 4]; text += hex[byte & 15]; }
return text;
}
inline std::string file_version(const std::filesystem::path& path) {
DWORD ignored = 0; const DWORD size = GetFileVersionInfoSizeW(path.c_str(), &ignored);
std::vector<char> data(size);
VS_FIXEDFILEINFO* info = nullptr; UINT length = 0;
if (!size || !GetFileVersionInfoW(path.c_str(), 0, size, data.data()) ||
!VerQueryValueW(data.data(), L"\\", reinterpret_cast<void**>(&info), &length) || !info) return "unknown";
return std::to_string(HIWORD(info->dwFileVersionMS)) + "." + std::to_string(LOWORD(info->dwFileVersionMS)) + "." +
std::to_string(HIWORD(info->dwFileVersionLS)) + "." + std::to_string(LOWORD(info->dwFileVersionLS));
}
// Exact identity check with the fix spelled out: what was found, what is needed.
inline void verify_dll(const std::filesystem::path& dir, const std::vector<Pin>& pins, const char* kind,
const wchar_t* relative, const char* label, std::vector<std::string>& notes) {
const auto path = dir / relative;
std::string wanted;
for (const auto& pin : pins) if (pin.kind == kind) wanted += (wanted.empty() ? "" : " or ") + pin.version;
const auto hash = sha256(path);
if (hash.empty())
throw std::runtime_error(std::string(label) + " is missing. Copy NVIDIA's " + label + " (version " + wanted +
") to d4r\\" + diag::utf8(relative) + " and restart the game.");
for (const auto& pin : pins) if (pin.kind == kind && pin.sha256 == hash) {
if (pin.status != "validated")
notes.push_back(std::string(label) + " " + pin.version + " is accepted but not yet validated on Windows");
return;
}
throw std::runtime_error("Wrong " + std::string(label) + ": you have version " + file_version(path) +
" (or a modified file); d4r on Windows needs exactly " + wanted + ". Replace d4r\\" + diag::utf8(relative) +
" and restart the game.");
}
// The installer used to check these before touching the game; a drag-in
// install checks them at startup instead.
inline void verify_code_objects(const std::filesystem::path& dir) {
const std::string arch = diag::HipApi::target_arch();
std::vector<std::filesystem::path> objects{dir / (L"pixel_convert_" + diag::wide(arch) + L".hsaco")};
const auto native = dir / L"native" / diag::wide(arch);
if (!std::filesystem::is_directory(native)) throw std::runtime_error("d4r\\native\\" + arch + " is missing; extract the d4r ZIP again");
for (const auto& entry : std::filesystem::directory_iterator(native))
if (entry.path().extension() == L".hsaco") objects.push_back(entry.path());
for (const auto& object : objects) {
try { diag::require_code_object_target(object, arch); }
catch (const std::exception&) {
throw std::runtime_error(diag::utf8(object.filename().c_str()) + " is not built for " + arch +
" or is damaged; extract the d4r ZIP again (do not rename or mix files from other GPUs)");
}
}
}
inline void reject_translation_layer() {
wchar_t exe[32768]{}; GetModuleFileNameW(nullptr, exe, 32768);
const auto proxy = std::filesystem::path(exe).parent_path() / L"d3d12.dll";
std::error_code error;
const auto size = std::filesystem::file_size(proxy, error);
if (error || size > 64ull << 20) return;
auto text = lower(read_text(proxy));
if (text.find("vkd3d") != std::string::npos || text.find("wined3d") != std::string::npos)
throw std::runtime_error("the game folder contains a d3d12.dll from vkd3d-proton/Wine (left over from a Linux "
"install?). Move d3d12.dll and d3d12core.dll out of the game folder; d4r on Windows uses native D3D12");
}
// Update the OS block (ZLUDA/Rust, later DLLs) and this CRT's copy (getenv).
inline void set_variable(const wchar_t* name, const std::optional<std::wstring>& value) {
SetEnvironmentVariableW(name, value ? value->c_str() : nullptr);
_wputenv_s(name, value ? value->c_str() : L"");
}
// Player mode only runs when no launcher set up the environment, so no launcher is
// reading these streams. A game started from Steam can have a valid but unread pipe
// as stderr; always writing to the log keeps errors visible.
inline void redirect(FILE* crt, DWORD stream, const std::filesystem::path& path) {
FILE* reopened = nullptr;
if (_wfreopen_s(&reopened, path.c_str(), L"a", crt) || !reopened) return;
std::setvbuf(crt, nullptr, _IONBF, 0);
SetStdHandle(stream, reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(crt))));
}
inline void apply(const std::filesystem::path& dir, const Settings& settings) {
std::filesystem::path cache = settings.cacheDir;
if (cache.empty()) {
const wchar_t* localAppData = _wgetenv(L"LOCALAPPDATA");
if (!localAppData || !*localAppData) throw std::runtime_error("LOCALAPPDATA is not set; set [Paths] CacheDir in d4r.ini");
cache = std::filesystem::path(localAppData) / L"d4r" / L"cache";
} else if (cache.is_relative()) cache = dir / cache;
std::filesystem::create_directories(cache);
const auto p = [&](const std::wstring& relative) { return std::optional<std::wstring>((dir / relative).wstring()); };
const std::optional<std::wstring> on = std::wstring(L"1"), off = std::wstring(L"0"), unset;
const wchar_t* path = _wgetenv(L"PATH");
// Same codegen and paths as windows-game.ps1; only diagnostics differ.
const std::pair<const wchar_t*, std::optional<std::wstring>> values[] = {
{L"D4R_HIP_ROOT", p(L"hip")}, {L"HIP_PATH", p(L"hip")},
{L"D4R_NVCUDA_DLL", p(L"zluda\\nvcuda.dll")}, {L"ZLUDA_CUDA_LIB", p(L"zluda\\nvcuda.dll")},
{L"D4R_NVAPI_DLL", p(L"nvapi\\" + diag::wide(diag::HipApi::target_arch()) + L"\\nvapi64.dll")}, {L"D4R_NVAPI_BACKEND", p(L"zluda\\nvapi64.dll")},
{L"D4R_NGX_CORE", p(L"ngx\\_nvngx.dll")}, {L"D4R_DLSS_DLL", p(L"nvngx_dlss.dll")},
{L"D4R_FORMAT_MODULE", p(L"pixel_convert_" + diag::wide(diag::HipApi::target_arch()) + L".hsaco")},
{L"D4R_ZLUDA_NATIVE_DIR", p(L"native\\" + diag::wide(diag::HipApi::target_arch()))},{L"D4R_D3D12_COMMAND_BACKEND", on},
{L"D4R_ZLUDA_WMMA", on}, {L"D4R_ZLUDA_WMMA_FP8", on}, {L"D4R_ZLUDA_WMMA_FP8_NATIVE", off},
{L"D4R_ZLUDA_WMMA_F16_REFERENCE", on},
{L"D4R_ZLUDA_IGNORE_DENORMAL", unset}, {L"D4R_ZLUDA_FAST_MATH", unset}, {L"D4R_ZLUDA_WMMA_F32ACC", unset},
{L"D4R_ZLUDA_WAVE64", unset}, {L"D4R_ZLUDA_WMMA_LAYOUT", unset},
// Per-launch kernel lines are diagnostics; ZLUDA excludes this flag from its cache key.
{L"D4R_ZLUDA_VERBOSE", settings.verbose ? on : unset},
{L"D4R_QUIET_API", settings.verbose ? unset : on},
{L"D4R_VALIDATE_OUTPUT", settings.validateOutput ? on : unset},
{L"D4R_ASYNC_INTEROP", settings.asyncInterop ? on : unset},
{L"D4R_SHIM_WATERMARK", settings.showWatermark ? on : unset},
{L"D4R_PROFILE_STAGES", unset}, {L"D4R_ZLUDA_PROFILE", unset}, {L"D4R_ZLUDA_PROFILE_DEFERRED", unset},
{L"D4R_ZLUDA_PROFILE_ALLOW_LEGACY", unset}, {L"D4R_ZLUDA_PROFILE_EVERY", unset},
{L"D4R_PROFILE_COMMAND_HOOKS", unset}, {L"D4R_ZLUDA_PROFILE_API", unset}, {L"D4R_PROFILE_GPU_BOUNDARY", unset},
{L"D4R_DISABLE_INTEROP_LIST_CACHE", unset}, {L"D4R_BATCH_INPUT_COPIES", unset}, {L"D4R_INTEROP_VERIFY", unset},
{L"D4R_DIAG_DIR", unset}, {L"ZLUDA_LOG_DIR", unset},
{L"ZLUDA_CACHE_DIR", std::optional<std::wstring>(cache.wstring())},
{L"PATH", std::optional<std::wstring>((dir / L"hip\\bin").wstring() + L";" + (dir / L"zluda").wstring() +
(path ? std::wstring(L";") + path : std::wstring()))},
};
for (const auto& [name, value] : values) set_variable(name, value);
const wchar_t* wgp = _wgetenv(L"D4R_ZLUDA_WGP");
if (wgp && std::wstring(wgp) != L"1") set_variable(L"D4R_ZLUDA_WGP", unset);
}
// Called before any path variable is read. A failure leaves DLSS unavailable
// (the game keeps its own upscaler) and is written to d4r\d4r_nvngx.log.
inline void ensure_environment() {
static std::once_flag once; static std::string failure;
std::call_once(once, [] {
try {
if (const char* root = std::getenv("D4R_HIP_ROOT"); root && *root) return; // Launcher/diagnostic run.
const auto dir = module_directory();
if (!std::filesystem::exists(dir / L"d4r.ini")) return; // Developer layout: keep the original requirement.
auto& current = state(); current.dir = dir;
const auto log = dir / L"d4r_nvngx.log";
{ std::ofstream truncate(log, std::ios::trunc); } // Rewritten at every launch, as on Linux.
redirect(stderr, STD_ERROR_HANDLE, log);
const auto settings = read_settings(parse_ini(read_text(dir / L"d4r.ini")), current.notes);
if (settings.verbose) redirect(stdout, STD_OUTPUT_HANDLE, log);
const auto pins = parse_pins(read_text(dir / L"dll-pins.txt"));
verify_dll(dir, pins, "dlss", L"nvngx_dlss.dll", "nvngx_dlss.dll", current.notes);
verify_dll(dir, pins, "ngx", L"ngx\\_nvngx.dll", "_nvngx.dll", current.notes);
reject_translation_layer();
verify_code_objects(dir);
apply(dir, settings);
current.active = true; current.preset = settings.preset;
std::fprintf(stderr, "d4r: player mode, model=%s validate=%u async=%u log=%s, architecture=%s\n",
settings.preset == 13 ? "M" : "K", unsigned(settings.validateOutput), unsigned(settings.asyncInterop),
settings.verbose ? "verbose" : "normal", diag::HipApi::target_arch());
} catch (const std::exception& error) { failure = error.what(); }
for (const auto& note : state().notes) std::fprintf(stderr, "d4r: %s\n", note.c_str());
if (!failure.empty()) std::fprintf(stderr, "d4r: DLSS disabled: %s\n", failure.c_str());
});
if (!failure.empty()) throw std::runtime_error(failure);
}
// [DLSS] Model forces one render preset for every quality mode, as the Linux
// shim does. Applied at create and evaluate so a recreated feature cannot fall
// back to the game's preset. Not used by launcher runs (OptiScaler sets it).
inline void apply_preset(void* parameters) {
const auto& current = state();
if (!current.active || !current.preset || !parameters) return;
for (const char* name : {"DLSS.Hint.Render.Preset.DLAA", "DLSS.Hint.Render.Preset.Quality", "DLSS.Hint.Render.Preset.Balanced",
"DLSS.Hint.Render.Preset.Performance", "DLSS.Hint.Render.Preset.UltraPerformance", "DLSS.Hint.Render.Preset.UltraQuality"})
d4r_ngx_set_uint(parameters, name, current.preset);
}
}
tools/windows/player_config_probe.cpp
-82
View File
@@ -1,82 +0,0 @@
// Checks the drag-in d4r.ini parser and NVIDIA DLL pin decisions with
// fixtures. No GPU work, no NVIDIA binaries.
#include "player_config.h"
#include <functional>
namespace player = d4r::win::player;
namespace {
int failures = 0;
void expect(bool condition, const std::string& what) {
std::printf("%s %s\n", condition ? "ok " : "FAIL", what.c_str());
if (!condition) ++failures;
}
bool has_note(const std::vector<std::string>& notes, const std::string& text) {
for (const auto& note : notes) if (note.find(text) != std::string::npos) return true;
return false;
}
player::Settings settings(const std::string& ini, std::vector<std::string>& notes) {
notes.clear(); return player::read_settings(player::parse_ini(ini), notes);
}
std::string error_of(const std::function<void()>& fn) {
try { fn(); } catch (const std::exception& error) { return error.what(); }
return {};
}
}
int wmain(int argc, wchar_t** argv) {
if (argc != 2) { std::printf("usage: player_config_probe FIXTURE_DIR\n"); return 2; }
const std::filesystem::path dir(argv[1]);
std::vector<std::string> notes;
auto s = settings("", notes);
expect(s.preset == 11 && !s.asyncInterop && !s.validateOutput && !s.verbose && notes.empty(), "empty d4r.ini uses K, sync, no validation");
s = settings("\xEF\xBB\xBF[DLSS]\nModel = M ; DLSS 4.5\n", notes);
expect(s.preset == 13 && notes.empty(), "BOM and inline comment; Model = M");
s = settings("[dlss]\nmodel = dlss4.5\n", notes);
expect(s.preset == 13, "case-insensitive section/key and alias DLSS4.5");
for (const char* model : {"E", "L", "game", "7"}) {
s = settings(std::string("[DLSS]\nModel = ") + model + "\n", notes);
expect(s.preset == 11 && has_note(notes, "not available on Windows"), std::string("Model = ") + model + " is rejected with a note and K");
}
s = settings("[Interop]\nAsyncInterop = yes\n[Debug]\nValidateOutput = on\nLog = verbose\n", notes);
expect(s.asyncInterop && s.validateOutput && s.verbose && notes.empty(), "boolean spellings and Log = verbose");
s = settings("[Debug]\nValidateOutput = maybe\nLog = loud\n", notes);
expect(!s.validateOutput && !s.verbose && has_note(notes, "must be true or false, not 'maybe'") &&
has_note(notes, "must be normal or verbose"), "invalid values fall back with plain notes");
s = settings("[Kernels]\nNativeFp8 = on\n[Latency]\nFrameAge = 1\n", notes);
expect(has_note(notes, "[Kernels] NativeFp8 is not a Windows setting") && has_note(notes, "[Latency] FrameAge"),
"Linux-only settings are reported as ignored");
s = settings("", notes);
expect(!s.showWatermark, "watermark is hidden by default");
s = settings("[DLSS]\nShowWatermark = true\n", notes);
expect(s.showWatermark && notes.empty(), "[DLSS] ShowWatermark = true keeps NVIDIA's stamp");
s = settings("[DLSS]\nShowWatermark = sometimes\n", notes);
expect(!s.showWatermark && has_note(notes, "ShowWatermark must be true or false"), "invalid ShowWatermark falls back to hidden with a note");
s = settings("[Paths]\nCacheDir = D:\\d4r-cache\n", notes);
expect(s.cacheDir == L"D:\\d4r-cache" && notes.empty(), "CacheDir path");
// Pins: fixture DLL contents are ordinary text; their hashes are computed here.
std::filesystem::create_directories(dir / L"ngx");
{ std::ofstream(dir / L"nvngx_dlss.dll", std::ios::binary) << "validated dlss fixture"; }
{ std::ofstream(dir / L"ngx" / L"_nvngx.dll", std::ios::binary) << "unvalidated core fixture"; }
const auto dlss = player::sha256(dir / L"nvngx_dlss.dll"), core = player::sha256(dir / L"ngx" / L"_nvngx.dll");
const auto pins = player::parse_pins("# comment\ndlss " + dlss + " 310.9.1 validated\nngx " + core + " 32.0.15.9636 unvalidated\n"
"dlss " + std::string(64, 'A') + " 310.7.0 unvalidated\n");
notes.clear();
expect(error_of([&] { player::verify_dll(dir, pins, "dlss", L"nvngx_dlss.dll", "nvngx_dlss.dll", notes); }).empty() && notes.empty(),
"validated DLSS pin accepted silently");
expect(error_of([&] { player::verify_dll(dir, pins, "ngx", L"ngx\\_nvngx.dll", "_nvngx.dll", notes); }).empty() &&
has_note(notes, "not yet validated on Windows"), "unvalidated pin accepted with a note");
{ std::ofstream(dir / L"nvngx_dlss.dll", std::ios::binary) << "changed"; }
auto error = error_of([&] { player::verify_dll(dir, pins, "dlss", L"nvngx_dlss.dll", "nvngx_dlss.dll", notes); });
expect(error.find("Wrong nvngx_dlss.dll: you have version") != std::string::npos && error.find("310.9.1 or 310.7.0") != std::string::npos,
"changed DLL rejected with you-have/you-need: " + error);
std::filesystem::remove(dir / L"nvngx_dlss.dll");
error = error_of([&] { player::verify_dll(dir, pins, "dlss", L"nvngx_dlss.dll", "nvngx_dlss.dll", notes); });
expect(error.find("is missing. Copy NVIDIA's nvngx_dlss.dll") != std::string::npos, "missing DLL explains where to copy it");
expect(!error_of([] { player::parse_pins("dlss short 1 validated\n"); }).empty(), "damaged pins file rejected");
expect(!error_of([] { player::parse_pins("other " + std::string(64, 'A') + " 1 validated\n"); }).empty(), "unknown pin kind rejected");
std::printf("%s PLAYER_CONFIG failures=%d\n", failures ? "FAIL" : "PASS", failures);
return failures ? 1 : 0;
}
tools/windows/probe_kernel.hip
-30
View File
@@ -1,30 +0,0 @@
// No host SDK, CRT, or device library dependency. Compiled only for gfx1201.
extern "C" __attribute__((global)) void d4r_ptx_pattern(
unsigned int* output, unsigned int count, unsigned int seed)
{
const unsigned int i = __builtin_amdgcn_workgroup_id_x() * 128u +
__builtin_amdgcn_workitem_id_x();
if (i < count)
output[i] = (i * 1664525u + seed) ^ 0xa5a55a5au;
}
extern "C" __attribute__((global)) void d4r_probe_pattern(
unsigned int* output, unsigned int count, unsigned int seed)
{
const unsigned int i = __builtin_amdgcn_workgroup_id_x() * 128u +
__builtin_amdgcn_workitem_id_x();
if (i < count)
output[i] = (i * 1664525u + seed) ^ 0xa5a55a5au;
}
extern "C" __attribute__((global)) void d4r_probe_interop(
unsigned int* shared, unsigned int count, unsigned int seed, unsigned int* errors)
{
const unsigned int i = __builtin_amdgcn_workgroup_id_x() * 128u +
__builtin_amdgcn_workitem_id_x();
if (i < count) {
const unsigned int expected = (i * 1664525u + seed) ^ 0xa5a55a5au;
if (shared[i] != expected) __atomic_fetch_add(errors, 1u, __ATOMIC_RELAXED);
shared[i] ^= 0x3579bdf1u;
}
}
tools/windows/profile_report.py
-306
View File
@@ -1,306 +0,0 @@
"""Summarize native Windows d4r CPU stages and optional HIP event diagnostics.
CPU completion time is not GPU execution time. HIP occupancy API predictions
are not hardware counters. No bandwidth / WMMA utilization is inferred here.
"""
import argparse
import collections
import csv
import json
import math
import pathlib
import re
import statistics
def stats(values):
values = sorted(values)
def quantile(q):
index = (len(values) - 1) * q
lower = int(index)
return values[lower] + (values[min(lower + 1, len(values) - 1)] - values[lower]) * (index - lower)
return dict(samples=len(values), total=sum(values), mean=statistics.mean(values),
median=statistics.median(values), p95=quantile(.95), maximum=values[-1])
def metadata(directory):
result = {}
if directory is None:
return result
for path in sorted(directory.glob('*.txt')):
text = path.read_text(encoding='utf-8-sig', errors='replace')
for block in re.split(r'(?m)^ - \.args:', text)[1:]:
fields = dict(re.findall(r'(?m)^ \.(\w+):\s*([^\r\n]+)', block))
if 'name' in fields:
result[fields['name'].strip()] = {key: int(fields[key]) for key in (
'wavefront_size', 'vgpr_count', 'sgpr_count', 'vgpr_spill_count', 'sgpr_spill_count',
'group_segment_fixed_size', 'private_segment_fixed_size') if key in fields}
return result
def presentmon(directory):
# Keep swapchains separate: combining startup/menu/overlay chains invents
# frame intervals. Missing/NA metrics remain absent rather than becoming 0.
captures = []
for path in sorted(directory.glob('presentmon*.csv')):
groups = collections.defaultdict(list)
with path.open(encoding='utf-8-sig', newline='') as stream:
for row in csv.DictReader(stream):
if row.get('ProcessID') and row.get('SwapChainAddress'):
groups[(row['ProcessID'], row['SwapChainAddress'])].append(row)
for (process, swapchain), rows in groups.items():
metrics = {}
for name in ('MsBetweenPresents', 'MsBetweenAppStart', 'MsCPUBusy', 'MsCPUWait',
'MsGPULatency', 'MsGPUTime', 'MsGPUBusy', 'MsGPUWait',
'MsInPresentAPI', 'DisplayedTime', 'DisplayLatency'):
values = []
for row in rows:
raw = row.get(name, row.get(name.removeprefix('Ms'), ''))
try:
value = float(raw)
except (ValueError, TypeError):
continue
if math.isfinite(value) and value >= 0:
values.append(value)
if values:
metrics[name] = stats(values)
item = dict(file=path.name, processId=int(process), swapchain=swapchain,
frames=len(rows), metrics=metrics,
presentModes=dict(collections.Counter(row.get('PresentMode', '') for row in rows)),
syncIntervals=dict(collections.Counter(row.get('SyncInterval', '') for row in rows)))
interval = metrics.get('MsBetweenPresents', metrics.get('MsBetweenAppStart'))
if interval and interval['mean'] > 0:
item['presentFps'] = 1000. / interval['mean']
captures.append(item)
captures.sort(key=lambda row: row['frames'], reverse=True)
return captures
def gpu_telemetry(directory):
captures = []
for path in sorted(directory.glob('gpu-telemetry*.csv')):
unique = {}
with path.open(encoding='utf-8-sig', newline='') as stream:
rows = list(csv.DictReader(stream))
# Driver telemetry may repeat a cached measurement. Count polls but
# use each sensor timestamp once; missing/unsupported values stay absent.
for row in rows:
try:
timestamp = int(row.get('sensor_timestamp_ms', ''))
except (ValueError, TypeError):
continue
if timestamp > 0:
unique.setdefault(timestamp, row)
metrics = {}
for name in ('gpu_usage_pct', 'gpu_clock_mhz', 'vram_clock_mhz', 'gpu_power_w',
'board_power_w', 'temperature_c', 'hotspot_c', 'poll_ms'):
values = []
for row in unique.values():
try:
value = float(row.get(name, ''))
except (ValueError, TypeError):
continue
if math.isfinite(value) and value >= 0 and (name != 'gpu_usage_pct' or value <= 100):
values.append(value)
if values:
metrics[name] = stats(values)
captures.append(dict(file=path.name, polls=len(rows), uniqueSamples=len(unique), metrics=metrics))
return captures
def report(directory, metadata_directory=None):
kernel_metadata = metadata(metadata_directory)
stages = collections.defaultdict(list)
commands = collections.defaultdict(list)
kernels = collections.defaultdict(list)
hook_threads = collections.defaultdict(list)
apis = collections.defaultdict(list)
sample_skips = []
invalid_samples = []
boundaries = collections.defaultdict(list)
replay = collections.defaultdict(lambda: collections.defaultdict(dict))
invalid_replay = []
for path in sorted(directory.glob('*.log')):
data = path.read_bytes()
text = data.decode('utf-16' if data.startswith((b'\xff\xfe', b'\xfe\xff')) else 'utf-8-sig', errors='replace')
for line in text.splitlines():
if line.startswith('D4R_REPLAY_PROFILE_INVALID '):
invalid_replay.append(dict(file=path.name, **dict(re.findall(r'(\w+)=(\S+)', line))))
if not line.startswith('D4R_REPLAY_PROFILE '):
continue
fields = dict(re.findall(r'(\w+)=(\S+)', line))
if not all(key in fields for key in ('kernel', 'variant', 'pair', 'gpu_ms')):
continue
value = float(fields['gpu_ms'])
if fields['variant'] not in ('control', 'candidate') or not math.isfinite(value) or value <= 0:
invalid_replay.append(dict(file=path.name, **fields))
continue
key = (path.name, fields['kernel'], fields.get('timing_source', 'events'), int(fields.get('launches', 1)))
replay[key][int(fields['pair'])][fields['variant']] = fields
for name, value in re.findall(r'D4R_STAGE name=(\S+) cpu_ms=([0-9.]+)', text):
stages[name].append(float(value))
for kind, fields in re.findall(r'(?m)^D4R_COMMAND_(RECORD|SUBMIT) ([^\r\n]+)', text):
for name, value in re.findall(r'(\w+_ms)=([0-9.]+)', fields):
number = float(value)
if name != 'interval_ms' or number > 0:
commands[kind.lower() + '.' + name].append(number)
for line in text.splitlines():
if line.startswith('D4R_GPU_BOUNDARY '):
fields = dict(re.findall(r'(\w+)=(\S+)', line))
if all(key in fields for key in ('feature', 'ticket', 'input_copy_ms', 'external_span_ms',
'output_copy_ms', 'total_ms', 'input_start_ticks',
'output_end_ticks', 'frequency')):
boundaries[fields['feature']].append(fields)
if line.startswith('D4R_CUDA_API_PROFILE '):
fields = dict(re.findall(r'(\w+)=(\S+)', line))
if all(key in fields for key in ('api', 'calls', 'total_ms', 'max_ms', 'period_ms', 'thread')):
apis[(fields['thread'], fields['api'])].append(fields)
if line.startswith('D4R_COMMAND_HOOK_SAMPLE '):
fields = dict(re.findall(r'(\w+)=(\S+)', line))
if all(name in fields for name in ('thread', 'period_ms', 'calls', 'samples', 'marked',
'access_sum_ms', 'capture_sum_ms', 'driver_sum_ms')):
hook_threads[fields['thread']].append(fields)
if not line.startswith('D4R_KERNEL_PROFILE '):
if line.startswith('D4R_KERNEL_PROFILE_SKIPPED '):
sample_skips.append(dict(re.findall(r'(\w+)=(\S+)', line)))
if line.startswith('D4R_KERNEL_PROFILE_INVALID '):
invalid_samples.append(dict(re.findall(r'(\w+)=("[^"]+"|\S+)', line)))
continue
fields = dict(re.findall(r'(\w+)=("[^"]+"|\S+)', line))
required = ('kernel', 'backend', 'phase', 'gpu_ms', 'enqueue_ms', 'completion_ms',
'grid', 'block', 'static_lds_bytes', 'dynamic_lds_bytes', 'regs', 'private_bytes',
'max_threads', 'predicted_blocks_per_multiprocessor', 'serializing')
if not all(key in fields for key in required):
continue # A bounded process stop can leave a partial final line.
kernels[(fields['kernel'].strip('"'), fields['backend'], fields['phase'],
fields['serializing'])].append(fields)
result = dict(cpuStages={name: stats(values) for name, values in stages.items()},
commandStages={name: stats(values) for name, values in commands.items()},
commandHookSamples=[],
cudaApi=[],
kernelSampleSkips=sample_skips,
invalidKernelSamples=invalid_samples,
gpuBoundaryStages={},
presentMon=presentmon(directory),
gpuTelemetry=gpu_telemetry(directory),
replayPairs=[],
invalidReplaySamples=invalid_replay,
kernels=[], notes=[
'CPU stage times include waits and host work; they are not isolated GPU timings.',
'CUDA API profiles measure host call duration without GPU events or added synchronization; they still include existing waits.',
'D3D12 boundary timestamps span copies and HIP/scheduling across the external fence; they do not isolate kernel execution or measure Present.',
'Boundary timestamps are read after existing completion fences; only 32 timing bytes are read, with no extra completion wait. GPU clock idle behavior can affect intervals.',
'PresentMon ETW GPU busy/wait are per-process frame estimates, not hardware sensor utilization. HWS and cross-API context attribution can affect them.',
'ADLX telemetry is driver-reported device-wide utilization/clocks/power, sampled in a separate read-only process. It does not isolate DLSS or WMMA occupancy.',
'Replay GPU/host intervals are normalized per launch; multi-launch batches include queue/dispatch gaps and are not isolated instruction time or application FPS.',
'Command record intervals are between NGX recording calls on one thread, not Present/FPS.',
'Hook estimates use random one-in-64 samples; driver timing covers generated forwarding methods only.',
'Hook access includes lock waits; summing threads does not measure serial frame latency or CPU execution time.',
'Serializing HIP-event profiles wait after every launch. Deferred profiles query ready events and never add completion waits; timing events still add overhead.',
'Deferred completionMs is host collection lag, not kernel completion latency. Legacy-stream sampling requires an opt-in because its timing markers retain legacy cross-stream ordering.',
'Occupancy is an API prediction, not a measured hardware counter.',
'Bandwidth and WMMA utilization require additional supported hardware tooling.'])
boundary_stages = collections.defaultdict(list)
for rows in boundaries.values():
rows.sort(key=lambda row: int(row['ticket']))
for row in rows:
for key in ('input_copy_ms', 'external_span_ms', 'output_copy_ms', 'total_ms'):
boundary_stages[key].append(float(row[key]))
for previous, current in zip(rows, rows[1:]):
if int(current['ticket']) != int(previous['ticket']) + 2 or current['frequency'] != previous['frequency']:
continue
frequency = int(current['frequency'])
before = int(current['input_start_ticks']) - int(previous['output_end_ticks'])
interval = int(current['input_start_ticks']) - int(previous['input_start_ticks'])
if frequency > 0 and before >= 0 and interval > 0:
boundary_stages['between_boundaries_ms'].append(before * 1000. / frequency)
boundary_stages['input_interval_ms'].append(interval * 1000. / frequency)
result['gpuBoundaryStages'] = {name: stats(values) for name, values in boundary_stages.items()}
for (file, name, timing, launches), pairs in replay.items():
complete = [row for row in pairs.values() if set(row) == {'control', 'candidate'}]
if complete:
values = {variant: [float(row[variant]['gpu_ms']) for row in complete] for variant in ('control', 'candidate')}
item = dict(file=file, kernel=name, timingSource=timing, launchesPerSample=launches,
controlGpuMs=stats(values['control']), candidateGpuMs=stats(values['candidate']),
candidateToControl=stats([candidate / control for control, candidate in zip(values['control'], values['candidate'])]))
for variant in ('control', 'candidate'):
host = [float(row[variant]['host_ms']) for row in complete if 'host_ms' in row[variant]]
if host:
item[variant + 'HostCompletionMs'] = stats(host)
result['replayPairs'].append(item)
for (thread, api), rows in apis.items():
calls = sum(int(row['calls']) for row in rows)
total = sum(float(row['total_ms']) for row in rows)
result['cudaApi'].append(dict(thread=thread, api=api, calls=calls, totalMs=total,
meanMs=total / calls, maxMs=max(float(row['max_ms']) for row in rows),
periodMs=sum(float(row['period_ms']) for row in rows)))
result['cudaApi'].sort(key=lambda row: row['totalMs'], reverse=True)
for thread, rows in hook_threads.items():
sums = {name: sum(float(row[name]) for row in rows) for name in
('period_ms', 'calls', 'samples', 'marked', 'access_sum_ms', 'capture_sum_ms', 'driver_sum_ms')}
if not sums['samples'] or not sums['period_ms']:
continue
scale = sums['calls'] / sums['samples']
estimate = scale * (sums['access_sum_ms'] + sums['capture_sum_ms'])
result['commandHookSamples'].append(dict(thread=int(thread), periods=len(rows), **sums,
accessMeanUs=1000 * sums['access_sum_ms'] / sums['samples'],
captureMeanUsPerSample=1000 * sums['capture_sum_ms'] / sums['samples'],
estimatedAccessCaptureMs=estimate, estimatedAccessCaptureMsPerSecond=1000 * estimate / sums['period_ms']))
for (name, backend, phase, serializing), rows in kernels.items():
item = dict(kernel=name, backend=backend, phase=phase,
serializing=serializing == '1',
legacyStream=any(row.get('legacy_stream') == '1' for row in rows),
completionMeaning='host_completion_wait' if serializing == '1' else 'host_collection_lag',
gpuMs=stats([float(row['gpu_ms']) for row in rows]),
enqueueMs=stats([float(row['enqueue_ms']) for row in rows]),
completionMs=stats([float(row['completion_ms']) for row in rows]),
configurations=[dict(config) for config in sorted({tuple((key, row[key]) for key in
('grid', 'block', 'static_lds_bytes', 'dynamic_lds_bytes', 'regs', 'private_bytes',
'max_threads', 'predicted_blocks_per_multiprocessor')) for row in rows})])
result['kernels'].append(item)
if backend == 'native':
item['elfMetadata'] = kernel_metadata.get(name + ('_prep' if phase == 'prep' else ''), {})
result['kernels'].sort(key=lambda row: row['gpuMs']['total'], reverse=True)
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('directory', type=pathlib.Path)
parser.add_argument('--output', type=pathlib.Path)
parser.add_argument('--metadata-directory', type=pathlib.Path, help='llvm-readobj --notes output for native code objects')
args = parser.parse_args()
result = report(args.directory, args.metadata_directory)
if not any(result[key] for key in ('cpuStages', 'kernels', 'commandStages', 'commandHookSamples', 'replayPairs', 'cudaApi', 'gpuBoundaryStages', 'presentMon', 'gpuTelemetry')):
parser.error('no complete stage or HIP-event records found')
if args.output:
args.output.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
for name, row in result['cpuStages'].items():
print(f"CPU {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
for name, row in result['commandStages'].items():
print(f"COMMAND {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
for name, row in result['gpuBoundaryStages'].items():
print(f"GPU BOUNDARY {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
for capture in result['presentMon']:
print(f"PRESENT {capture['file']} process={capture['processId']} swapchain={capture['swapchain']} frames={capture['frames']} fps={capture.get('presentFps', float('nan')):.3f}")
for name, row in capture['metrics'].items():
print(f" {name}: n={row['samples']} mean={row['mean']:.3f} ms median={row['median']:.3f} ms p95={row['p95']:.3f} ms")
for capture in result['gpuTelemetry']:
print(f"ADLX {capture['file']}: polls={capture['polls']} unique_samples={capture['uniqueSamples']}")
for name, row in capture['metrics'].items():
print(f" {name}: n={row['samples']} mean={row['mean']:.3f} median={row['median']:.3f} p95={row['p95']:.3f}")
for row in result['cudaApi'][:20]:
print(f"CUDA API {row['api']} {row['thread']}: n={row['calls']} mean={row['meanMs']:.6f} ms total={row['totalMs']:.3f} ms max={row['maxMs']:.3f} ms")
for row in result['commandHookSamples']:
print(f"HOOK thread={row['thread']} samples={int(row['samples'])} access={row['accessMeanUs']:.3f} us "
f"capture={row['captureMeanUsPerSample']:.3f} us estimated_cpu={row['estimatedAccessCaptureMsPerSecond']:.3f} ms/s")
for row in result['replayPairs']:
print(f"PAIRED {row['kernel']} timing={row['timingSource']} batch={row['launchesPerSample']}: n={row['controlGpuMs']['samples']} control_median={row['controlGpuMs']['median']:.6f} ms "
f"candidate_median={row['candidateGpuMs']['median']:.6f} ms ratio_median={row['candidateToControl']['median']:.6f}")
for row in result['kernels']:
times = row['gpuMs']
print(f"GPU {row['kernel']} {row['backend']} {row['phase']} serializing={int(row['serializing'])}: n={times['samples']} mean={times['mean']:.6f} ms total={times['total']:.3f} ms")
if __name__ == '__main__':
main()
tools/windows/swin_debug_compare.py
-26
View File
@@ -1,26 +0,0 @@
"""Compare compile-time Swin diagnostic snapshots with the NumPy intermediate values."""
import argparse
import numpy as np
from m_replay_validate import me, sm, LAYERS, metrics
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--capture-dir', required=True)
parser.add_argument('--stages', required=True)
parser.add_argument('--block', default='0,4')
parser.add_argument('--layer', choices=list(LAYERS), default='enc1')
args = parser.parse_args()
p = me.Params(args.capture_dir)
c, heads, _, cin = LAYERS[args.layer]
bx, by = map(int, args.block.split(','))
pre = 4 * c * cin + 8 * c if cin else 0
x0 = sm.decoder_input(p, bx, by, c, cin, heads, p.p32) if cin else sm.load_tokens(p, bx, by, c // 32)
stages = {}
sm.swin_block(p, x0, c, heads, pre, stages=stages)
actual = np.fromfile(args.stages, np.float16).reshape(4, 64, 128).astype(np.float64)
for index, name in enumerate(('h1', 'x1', 'h2', 'O')):
expected = stages[name]
values = actual[index, :, :expected.shape[1]]
metrics(expected, values, f'SWIN_STAGE {name}')
different = np.argwhere(expected != values)
for row, channel in different[:8]:
print(f'DIFFERENT stage={name} token={row} channel={channel} reference={expected[row, channel]} native={values[row, channel]}')
tools/windows/test_ptx_reference.py
-46
View File
@@ -1,46 +0,0 @@
"""Regression for divergent finite loops in the public CPU PTX reference."""
import pathlib
import sys
import tempfile
import unittest
import numpy as np
ROOT = pathlib.Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'kernels/tools'))
import ptxsim
class ControlFlow(unittest.TestCase):
def test_divergent_backward_loop_reconverges(self):
source = '''.visible .entry pattern(.param .align 8 .b8 pattern_param_0[8])
.maxntid 32, 1, 1
{
.reg .b32 %r<4>;
.reg .pred %p<1>;
mov.u32 %r0, %tid.x;
and.b32 %r0, %r0, 3;
add.u32 %r0, %r0, 1;
mov.u32 %r1, 0;
$loop:
add.u32 %r1, %r1, %r0;
sub.u32 %r0, %r0, 1;
setp.ne.u32 %p0, %r0, 0;
@%p0 bra $loop;
add.u32 %r2, %r1, 7;
ret;
}
'''
with tempfile.TemporaryDirectory() as directory:
path = pathlib.Path(directory) / 'pattern.ptx'
path.write_text(source)
statements, types, shared = ptxsim.parse_kernel(str(path), 'pattern')
block = ptxsim.Block(statements, types, shared, None, 'pattern_param_0', (32, 1, 1), (1, 1, 1), (0, 0, 0))
block.run(limit=1000)
n = np.arange(32) % 4 + 1
np.testing.assert_array_equal(block.regs['%r1'], n * (n + 1) // 2)
np.testing.assert_array_equal(block.regs['%r2'], n * (n + 1) // 2 + 7)
if __name__ == '__main__':
unittest.main()
tools/windows/wmma_f16_compare.hip
-40
View File
@@ -1,40 +0,0 @@
// Public synthetic reproducer: compare packed F16 WMMA with F32 WMMA followed
// by one FP16 rounding after each K16 step. Never part of game packaging.
#define D4R_K_F16_WMMA
#include "../../kernels/common/wmma_layout.h"
// The replay utility passes a single by-value parameter block, like K/M.
struct CompareParams { const _Float16* a; const _Float16* b; const float* c; float* output; };
extern "C" __attribute__((global)) void d4r_wmma_f16_compare(CompareParams params)
{
const _Float16* a = params.a;
const _Float16* b = params.b;
const float* c = params.c;
float* output = params.output;
const unsigned test = __builtin_amdgcn_workgroup_id_x();
const unsigned token = wm_lane() & 15;
const unsigned base = test * 256;
const wm_op aa = wm_op_load(a + base + token * 16);
const wm_op bb = wm_op_load(b + base + token * 16);
wm_f8v baseline, adapted;
wm_h8 packed;
for (unsigned i = 0; i < 8; ++i) {
baseline[i] = c[base + wm_acc_row(i) * 16 + token];
packed[i] = (_Float16)baseline[i];
}
adapted = baseline;
const unsigned steps = 1u << (test / 16);
for (unsigned step = 0; step < steps; ++step) {
baseline = wm_mma(aa, bb, baseline);
for (unsigned i = 0; i < 8; ++i) baseline[i] = (float)(_Float16)baseline[i];
packed = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(
__builtin_bit_cast(wm_h8, aa), __builtin_bit_cast(wm_h8, bb), packed);
adapted = wm_mma_f16_step(aa, bb, adapted);
}
for (unsigned i = 0; i < 8; ++i) {
const unsigned index = wm_acc_row(i) * 16 + token;
output[test * 768 + index] = baseline[i];
output[test * 768 + 256 + index] = (float)packed[i];
output[test * 768 + 512 + index] = adapted[i];
}
}
tools/windows/wmma_f16_compare.py
-67
View File
@@ -1,67 +0,0 @@
"""Public gfx1201 WMMA arithmetic comparison; does not load DLSS or weights."""
import argparse
import json
import os
import pathlib
import struct
import subprocess
import numpy as np
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--probe', type=pathlib.Path, required=True)
p.add_argument('--module', type=pathlib.Path, required=True)
p.add_argument('--hip-root', type=pathlib.Path, required=True)
p.add_argument('--output-dir', type=pathlib.Path, required=True)
args = p.parse_args()
root = args.output_dir.resolve()
root.mkdir(parents=True, exist_ok=True)
fixture = root / 'fixture'
fixture.mkdir(exist_ok=True)
rng = np.random.default_rng(90701201)
a = rng.uniform(-1, 1, (64, 16, 16)).astype(np.float16)
b = rng.uniform(-1, 1, a.shape).astype(np.float16)
c = rng.uniform(-.25, .25, a.shape).astype(np.float16).astype(np.float32)
initial_output = np.full((64, 3, 16, 16), np.nan, np.float32)
arrays = (a, b, c, initial_output)
bases = [0x10000000 * (i + 1) for i in range(4)]
(fixture / 'args.bin').write_bytes(struct.pack('<4Q', *bases))
lines = ['kernel d4r_wmma_f16_compare', 'launch 64 1 1 32 1 1 0', 'args 32']
for i, data in enumerate(arrays):
data.tofile(fixture / f'alloc-{i}.bin')
lines += [f'alloc {i} {bases[i]:#x} {data.nbytes}', f'pointer {i * 8} {i} 0']
(fixture / 'manifest.txt').write_text('\n'.join(lines) + '\n', encoding='utf-8')
env = os.environ.copy()
env['D4R_DIAG_DIR'] = str(root)
with (root / 'gpu.stdout.log').open('wb') as out, (root / 'gpu.stderr.log').open('wb') as err:
status = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
'--module', str(args.module.resolve()), '--fixture-dir', str(fixture), '--output-dir',
str(root / 'output'), '--iterations', '1'], env=env, stdout=out, stderr=err, timeout=120)
if status.returncode:
raise RuntimeError(f'GPU probe exit {status.returncode}; see {root}')
output = np.fromfile(root / 'output/alloc-3.bin', np.float32).reshape(64, 3, 16, 16)
if not np.isfinite(output).all():
raise RuntimeError('Nonfinite/unwritten WMMA output')
rows = []
for group, steps in enumerate((1, 2, 4, 8)):
values = output[group * 16:(group + 1) * 16]
difference = np.abs(values[:, 0].astype(np.float64) - values[:, 1].astype(np.float64))
row = dict(steps=steps, elements=difference.size, differing=int(np.count_nonzero(difference)),
maxAbsoluteError=float(difference.max()), rms=float(np.sqrt(np.mean(difference ** 2))))
rows.append(row)
print('WMMA_F16_COMPARE ' + json.dumps(row), flush=True)
adapter_exact = np.array_equal(output[:, 1].view(np.uint32), output[:, 2].view(np.uint32))
if not adapter_exact:
raise RuntimeError('Direct packed instruction and layout adapter differ')
result = dict(passed=True, architecture='gfx1201', publicSynthetic=True, adapterExact=adapter_exact,
baselineEquivalent=all(row['differing'] == 0 for row in rows), cases=rows,
gameOptimizationAccepted=False)
(root / 'validation.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
print('PASS WMMA_F16_ARITHMETIC_DIAGNOSTIC adapter_exact=1 baseline_equivalent=' +
str(int(result['baselineEquivalent'])), flush=True)
if __name__ == '__main__':
main()
tools/windows/wmma_gfx1201.hip
-112
View File
@@ -1,112 +0,0 @@
// Independent gfx12 WMMA layout probe. This deliberately does not use d4r fragments.
// A and B are row-major [logical row/column][K]; the intrinsic receives B,A
// so its output is row=A, column=B, as in AMD's RDNA 4 WMMA guide.
using h8 = _Float16 __attribute__((ext_vector_type(8)));
using f8 = float __attribute__((ext_vector_type(8)));
#include "../../kernels/common/wmma_backend.h"
#include "../../kernels/common/wmma_layout.h"
extern "C" __attribute__((global)) void d4r_wmma_swap_test(float* output)
{
const unsigned int lane = __builtin_amdgcn_workitem_id_x();
if (lane < 32u) {
output[lane] = d4r_wmma_other_half(static_cast<float>(lane));
output[32u + lane] = static_cast<float>(__builtin_amdgcn_mbcnt_lo(~0u, 0u));
}
}
extern "C" __attribute__((global)) void d4r_wmma_gfx1201(
const unsigned short* a_bits, const unsigned short* b_bits,
const float* initial_c, float* output, unsigned int steps)
{
const unsigned int lane = __builtin_amdgcn_workitem_id_x();
if (lane >= 32u) return;
const unsigned int row = lane & 15u;
const unsigned int group = lane >> 4;
const _Float16* a = reinterpret_cast<const _Float16*>(a_bits);
const _Float16* b = reinterpret_cast<const _Float16*>(b_bits);
#if defined(__GFX12__)
h8 a_frag, b_frag;
f8 c_frag;
for (unsigned int i = 0; i < 8u; ++i) {
a_frag[i] = a[row * 16u + group * 8u + i];
b_frag[i] = b[row * 16u + group * 8u + i];
c_frag[i] = initial_c[row * 16u + group * 8u + i];
}
f8 result = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(
b_frag, a_frag, c_frag);
if (steps == 2u)
result = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(
b_frag, a_frag, result);
for (unsigned int i = 0; i < 8u; ++i)
output[row * 16u + group * 8u + i] = result[i];
#else
// Adapted from realdody/d4r ae530b0: real gfx11 operands/accumulators,
// independent of the shared layout abstraction.
d4r_wmma_h16 a_frag, b_frag;
d4r_wmma_f8 c_frag;
for (unsigned int i = 0; i < 16u; ++i) {
a_frag[i] = a[row * 16u + i];
b_frag[i] = b[row * 16u + i];
}
for (unsigned int i = 0; i < 8u; ++i)
c_frag[i] = initial_c[(2u * i + group) * 16u + row];
d4r_wmma_f8 result = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, c_frag);
if (steps == 2u)
result = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a_frag, b_frag, result);
for (unsigned int i = 0; i < 8u; ++i)
output[(2u * i + group) * 16u + row] = result[i];
#endif
}
// d4r's original gfx11 fragment contract, exercised independently of K/M.
extern "C" __attribute__((global)) void d4r_wmma_legacy_contract(
const unsigned short* a_bits, const unsigned short* b_bits,
const float* initial_c, float* output, unsigned int steps)
{
const unsigned int lane = __builtin_amdgcn_workitem_id_x();
if (lane >= 32u) return;
const unsigned int token = lane & 15u;
const unsigned int group = lane >> 4;
const _Float16* a = reinterpret_cast<const _Float16*>(a_bits);
const _Float16* b = reinterpret_cast<const _Float16*>(b_bits);
d4r_wmma_h16 a_frag, b_frag;
d4r_wmma_f8 c_frag;
for (unsigned int i = 0; i < 16u; ++i) {
a_frag[i] = a[token * 16u + i];
b_frag[i] = b[token * 16u + i];
}
for (unsigned int i = 0; i < 8u; ++i)
c_frag[i] = initial_c[(2u * i + group) * 16u + token];
d4r_wmma_f8 result = d4r_wmma_legacy_layout(a_frag, b_frag, c_frag);
if (steps == 2u)
result = d4r_wmma_legacy_layout(a_frag, b_frag, result);
for (unsigned int i = 0; i < 8u; ++i)
output[(2u * i + group) * 16u + token] = result[i];
}
// Upstream's native per-target layout, including accumulator -> next-operand
// packing. It must match the same scalar reference as the independent probe.
extern "C" __attribute__((global)) void d4r_wmma_upstream_contract(
const unsigned short* a_bits, const unsigned short* b_bits,
const float* initial_c, float* output, unsigned int steps)
{
const unsigned int lane = __builtin_amdgcn_workitem_id_x();
if (lane >= 32u) return;
const unsigned int token = lane & 15u;
const _Float16* a = reinterpret_cast<const _Float16*>(a_bits);
const _Float16* b = reinterpret_cast<const _Float16*>(b_bits);
const wm_op a_frag = wm_op_load(a + token * 16u);
const wm_op b_frag = wm_op_load(b + token * 16u);
wm_f8v c_frag;
for (unsigned int i = 0; i < 8u; ++i)
c_frag[i] = initial_c[wm_acc_row(i) * 16u + token];
wm_f8v result = wm_mma(a_frag, b_frag, c_frag);
if (steps == 2u)
result = wm_mma(a_frag, b_frag, result);
// Fixture values are exactly representable as FP16. Checking this roundtrip
// catches a wrong gfx12 fragment reconstruction without a tolerance mask.
const wm_op next_operand = wm_op_from_f8(result);
for (unsigned int i = 0; i < 8u; ++i)
output[wm_acc_row(i) * 16u + token] = static_cast<float>(wm_op_acc_elem(next_operand, i));
}