Add MicroCUDA backend, adaptive output wait, wider native K coverage (improve perf)

Shim
- Adaptive output-event wait: sleep to ~120 us before the predicted
  completion (smoothed per wait site), spin near it, back off when late.
  D4R_SHIM_OUTPUT_POLL_US restores a fixed interval.

- Defer the input-marker wait into the worker thread with VRAM interop
  (D4R_SHIM_DEFER_MARKER, default on), taking the host setup off the
  critical path.

MicroCUDA
- Experimental ZLUDA-free backend in wine_nvcuda_bridge, plus
  tools/microcuda and scripts/build_microcuda.sh. Serves accuracy, fast
  and native kernels; unregistered modules fall back to the reference
  path. Unclassified writes invalidate prepared weights.

M kernels
- SWIN_PREP_IDENTITY_AT: key weight prep by allocation identity
  (MicroCUDA); enc3 tube uses it with 8 slots. ZLUDA ignores it.

- dec2 (gfx1101): SWIN_PACKED_RESIDUAL, byte-identical, ~1.6% faster.

- Opt-in gfx12 hardware fp8 conversion (SWIN_FP8_HWCVT), unverified
  on hardware.

Native K translators
- ctile: handle variants that re-materialize the parameter base (mvlo).

- rewrite_output: check live-in registers, not just outputs.

- build.sh: native input covers all 8 flag variants, output the 4 HDR
  "max" variants; depth-tile skip stays mvhi-only.

Tools
- dump_runner and bench_dlss_kernel harness additions.

tools/microcuda/arch_spoof.c

@@ -0,0 +1,35 @@
+/* LD_PRELOAD for zluda_precompile.py: report D4R_SPOOF_GFX (e.g. gfx1201) as the device's gcnArchName, so
+ * ZLUDA compiles (and caches) code objects for a GPU that is not installed. Loading them then fails, which
+ * the precompile treats as expected; the cache entry is written before the load. Never use it for running. */
+#define _GNU_SOURCE
+#define __HIP_PLATFORM_AMD__
+#include <hip/hip_runtime_api.h>
+#include <dlfcn.h>
+#include <stdlib.h>
+#include <string.h>
+
+hipError_t hipGetDevicePropertiesR0600(hipDeviceProp_tR0600* props, int device)
+{
+    static hipError_t (*real)(hipDeviceProp_tR0600*, int);
+    if (!real)
+        real = (hipError_t (*)(hipDeviceProp_tR0600*, int))dlsym(RTLD_NEXT, "hipGetDevicePropertiesR0600");
+    if (!real)
+    {
+        /* ZLUDA loaded RTLD_LOCAL (ctypes): its HIP is not in the global scope RTLD_NEXT searches */
+        void* hip = dlopen("libamdhip64.so.7", RTLD_NOW | RTLD_NOLOAD);
+        if (!hip)
+            hip = dlopen("libamdhip64.so", RTLD_NOW | RTLD_NOLOAD);
+        if (hip)
+            real = (hipError_t (*)(hipDeviceProp_tR0600*, int))dlsym(hip, "hipGetDevicePropertiesR0600");
+    }
+    if (!real)
+        return hipErrorNotInitialized;
+    hipError_t result = real(props, device);
+    const char* arch = getenv("D4R_SPOOF_GFX");
+    if (result == hipSuccess && props && arch && *arch)
+    {
+        memset(props->gcnArchName, 0, sizeof(props->gcnArchName));
+        strncpy(props->gcnArchName, arch, sizeof(props->gcnArchName) - 1);
+    }
+    return result;
+}

tools/microcuda/check_runtime.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""GPU integration checks for the MicroCUDA runtime; no games or NGX evaluation.
+usage: check_runtime.py LIBRARY PACK_DIR KNOWN_PTX NATIVE_DIR
+
+KNOWN_PTX is the K enc0 module text (a module in the pack with a native dltss_pwin_enc0_layer in NATIVE_DIR).
+Pack-state failures (switch mismatch, tampered object) run in child processes: the pack is read once.
+"""
+import ctypes as c
+import json
+import os
+from pathlib import Path
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+
+ptr, u64, size = c.c_void_p, c.c_uint64, c.c_size_t
+
+
+def api(lib, name, args):
+    f = getattr(lib, name)
+    f.argtypes = args
+    f.restype = c.c_int
+    return f
+
+
+def child(library, ptx):
+    """Prints the cuModuleLoadData result for ptx under the environment it was started with."""
+    lib = c.CDLL(library)
+    assert api(lib, 'cuInit', [c.c_uint])(0) == 0
+    context = ptr()
+    assert api(lib, 'cuCtxCreate_v2', [c.POINTER(ptr), c.c_uint, c.c_int])(c.byref(context), 0, 0) == 0
+    module = ptr()
+    print(api(lib, 'cuModuleLoadData', [c.POINTER(ptr), ptr])(c.byref(module), c.create_string_buffer(Path(ptx).read_bytes())))
+
+
+def run_child(library, ptx, **env):
+    out = subprocess.run([sys.executable, __file__, '--child', library, ptx], env={**os.environ, **env},
+                         capture_output=True, text=True, check=True)
+    return int(out.stdout.split()[-1])
+
+
+def main():
+    if sys.argv[1] == '--child':
+        return child(*sys.argv[2:])
+    library, pack, ptx, native = sys.argv[1:]
+    os.environ['D4R_MICROCUDA_PACK'] = pack
+    os.environ['D4R_MICROCUDA_NATIVE_DIR'] = native
+    switches = next(l[9:].strip() for l in open(Path(pack) / 'microcuda-pack.txt') if l.startswith('switches'))
+    for key in [k for k in os.environ if k.startswith('D4R_ZLUDA_')]:
+        del os.environ[key]
+    for item in filter(None, switches.split(',')):
+        k, v = item.split('=', 1)
+        os.environ[k] = v
+    checks = []
+
+    def expect(label, result, code=0):
+        assert result == code, (label, result, code)
+        checks.append(label)
+
+    expect('switch-set mismatch refuses the pack', run_child(library, ptx, D4R_ZLUDA_WMMA='0'), 200)
+    with tempfile.TemporaryDirectory(prefix='d4r-microcuda-check-') as directory:
+        shutil.copytree(pack, directory, dirs_exist_ok=True)
+        for co in (Path(directory) / 'modules').glob('*.co'):
+            b = bytearray(co.read_bytes())
+            b[-1] ^= 1
+            co.write_bytes(b)
+        expect('tampered pack object rejected', run_child(library, ptx, D4R_MICROCUDA_PACK=directory), 200)
+    expect('missing pack', run_child(library, ptx, D4R_MICROCUDA_PACK='/nonexistent-microcuda'), 500)
+
+    lib = c.CDLL(library)
+    A = lambda name, args: api(lib, name, args)
+    init = A('cuInit', [c.c_uint])
+    load = A('cuModuleLoadData', [c.POINTER(ptr), ptr])
+    unload = A('cuModuleUnload', [ptr])
+    get = A('cuModuleGetFunction', [c.POINTER(ptr), ptr, c.c_char_p])
+    launch = A('cuLaunchKernel', [ptr] + [c.c_uint] * 7 + [ptr, c.POINTER(ptr), c.POINTER(ptr)])
+    text = c.create_string_buffer(Path(ptx).read_bytes())
+    module, function = ptr(), ptr()
+    expect('module before init', load(c.byref(module), text), 3)
+    expect('invalid init flags', init(1), 1)
+    expect('init', init(0))
+    proc = A('cuGetProcAddress_v2', [c.c_char_p, c.POINTER(ptr), c.c_int, u64, c.POINTER(c.c_int)])
+    address, status = ptr(), c.c_int()
+    expect('getProc modern alias', proc(b'cuMemAlloc', c.byref(address), 12080, 0, c.byref(status)))
+    assert address.value == c.cast(lib.cuMemAlloc_v2, ptr).value and status.value == 0
+    expect('getProc array API', proc(b'cuArrayCreate', c.byref(address), 12080, 0, c.byref(status)))
+    expect('getProc unknown', proc(b'cuGraphLaunch', c.byref(address), 12080, 0, c.byref(status)), 500)
+    assert address.value is None and status.value == 1
+    expect('getProc perthread flag rejected', proc(b'cuLaunchKernel', c.byref(address), 12080, 2, c.byref(status)), 801)
+    expect('getProc future version rejected', proc(b'cuLaunchKernel', c.byref(address), 13000, 0, c.byref(status)), 801)
+    expect('getProc null output', proc(b'cuInit', None, 12080, 0, None), 1)
+    device = c.c_int()
+    expect('device', A('cuDeviceGet', [c.POINTER(c.c_int), c.c_int])(c.byref(device), 0))
+    context = ptr()
+    expect('context', A('cuCtxCreate_v2', [c.POINTER(ptr), c.c_uint, c.c_int])(c.byref(context), 0, device.value))
+    current = ptr()
+    expect('get current', A('cuCtxGetCurrent', [c.POINTER(ptr)])(c.byref(current)))
+    assert current.value == context.value
+
+    expect('unknown PTX', load(c.byref(module), c.create_string_buffer(b'.version 8.7\n.target sm_89\n.address_size 64\n')), 200)
+    bad = bytearray(Path(ptx).read_bytes())
+    bad[0] ^= 1
+    expect('changed PTX', load(c.byref(module), c.create_string_buffer(bytes(bad))), 200)
+    expect('known PTX', load(c.byref(module), text))
+    expect('unknown function', get(c.byref(function), module, b'unknown'), 500)
+    assert function.value is None
+    expect('native function', get(c.byref(function), module, b'dltss_pwin_enc0_layer'))
+    another = ptr()
+    expect('cached lookup', get(c.byref(another), module, b'dltss_pwin_enc0_layer'))
+    assert another.value == function.value
+    os.environ['D4R_MICROCUDA_NATIVE_DIR'] = '/nonexistent-microcuda'
+    plain_module, plain = ptr(), ptr()
+    expect('second module instance', load(c.byref(plain_module), text))
+    expect('pack function without native override', get(c.byref(plain), plain_module, b'dltss_pwin_enc0_layer'))
+    assert plain.value != function.value
+    expect('second module unload', unload(plain_module))
+    os.environ['D4R_MICROCUDA_NATIVE_DIR'] = native
+    args = c.create_string_buffer(176)
+    n = size(176)
+    extra = (ptr * 5)(3, c.addressof(args), 2, c.addressof(n), None)
+    expect('HIP token rejected as CUDA token', launch(function, 1, 1, 1, 32, 1, 1, 0, None, None, extra), 801)
+    unterminated = (ptr * 8)(1, c.addressof(args), 2, c.addressof(n), 1, c.addressof(args), 2, c.addressof(n))
+    expect('unterminated packed args', launch(function, 1, 1, 1, 32, 1, 1, 0, None, None, unterminated), 1)
+    expect('null function', launch(None, 1, 1, 1, 32, 1, 1, 0, None, None, extra), 400)
+
+    allocate = A('cuMemAlloc_v2', [c.POINTER(u64), size])
+    d2h = A('cuMemcpyDtoH_v2', [ptr, u64, size])
+    source = c.create_string_buffer(bytes(range(256)))
+    result = c.create_string_buffer(256)
+    device_mem = u64()
+    expect('allocation', allocate(c.byref(device_mem), 256))
+    expect('zero-filled allocation readback', d2h(result, device_mem, 256))
+    assert result.raw == bytes(256)
+    stream = ptr()
+    expect('stream create', A('cuStreamCreate', [c.POINTER(ptr), c.c_uint])(c.byref(stream), 1))
+    expect('async H2D', A('cuMemcpyHtoDAsync_v2', [u64, ptr, size, ptr])(device_mem, source, 256, stream))
+    event = ptr()
+    expect('event create', A('cuEventCreate', [c.POINTER(ptr), c.c_uint])(c.byref(event), 0))
+    expect('event record', A('cuEventRecord', [ptr, ptr])(event, stream))
+    expect('event wait', A('cuEventSynchronize', [ptr])(event))
+    expect('event query ready', A('cuEventQuery', [ptr])(event))
+    expect('D2H', d2h(result, device_mem, 256))
+    assert result.raw == source.raw[:256]
+    expect('event destroy', A('cuEventDestroy_v2', [ptr])(event))
+    expect('stream destroy', A('cuStreamDestroy_v2', [ptr])(stream))
+    expect('free', A('cuMemFree_v2', [u64])(device_mem))
+
+    # arrays: CUDA_ARRAY_DESCRIPTOR {size_t Width, Height; int Format; unsigned NumChannels}
+    array_create = A('cuArrayCreate_v2', [c.POINTER(ptr), ptr])
+    array_desc = A('cuArrayGetDescriptor_v2', [ptr, ptr])
+    array_destroy = A('cuArrayDestroy', [ptr])
+    memcpy2d = A('cuMemcpy2D_v2', [ptr])
+    unorm = ptr()
+    expect('UNORM8x4 array', array_create(c.byref(unorm), c.create_string_buffer(struct.pack('<QQiI', 4, 4, 194, 4))))
+    expect('array descriptor unsupported, as ZLUDA', array_desc(c.create_string_buffer(24), unorm), 801)
+    # CUDA_RESOURCE_DESC (144 bytes): resType 0 = ARRAY, hArray at 8
+    resource = c.create_string_buffer(struct.pack('<iiQ', 0, 0, unorm.value) + bytes(128))
+    surface = u64()
+    expect('surface object', A('cuSurfObjectCreate', [c.POINTER(u64), ptr])(c.byref(surface), resource))
+    handle = c.create_string_buffer(96)
+    expect('surface handle readback', d2h(handle, surface, 96))
+    assert struct.unpack_from('<I', handle.raw, 80)[0] == 194 and handle.raw[48:80] == bytes(32)
+    back = c.create_string_buffer(144)
+    expect('surface resource desc', A('cuSurfObjectGetResourceDesc', [ptr, u64])(back, surface))
+    assert back.raw == resource.raw[:144]
+    # CUDA_TEXTURE_DESC (104 bytes): address modes, filter, flags (bit 1 = normalized coordinates)
+    texture = c.create_string_buffer(struct.pack('<iiiiI', 1, 1, 1, 0, 2) + bytes(84))
+    tex = u64()
+    expect('texture object', A('cuTexObjectCreate', [c.POINTER(u64), ptr, ptr, ptr])(c.byref(tex), resource, texture, None))
+    expect('texture destroy', A('cuTexObjectDestroy', [u64])(tex))
+    expect('surface destroy', A('cuSurfObjectDestroy', [u64])(surface))
+    expect('unknown surface destroy', A('cuSurfObjectDestroy', [u64])(surface), 1)
+    expect('array destroy', array_destroy(unorm))
+
+    # packed UNORM 10:10:10:2 through its RGBA32F backing: host -> array -> host
+    packed = ptr()
+    expect('packed 1010102 array', array_create(c.byref(packed), c.create_string_buffer(struct.pack('<QQiI', 4, 2, 80, 4))))
+    words = [(i * 37 % 1024) | (i * 101 % 1024) << 10 | (i * 7 % 1024) << 20 | (i % 4) << 30 for i in range(8)]
+    host_in = c.create_string_buffer(struct.pack('<8I', *words))
+    host_out = c.create_string_buffer(32)
+
+    def copy2d(src_type, src_host, src_array, dst_type, dst_host, dst_array):
+        # CUDA_MEMCPY2D: srcX,srcY,srcType,srcHost,srcDevice,srcArray,srcPitch, dst..., WidthInBytes, Height
+        return memcpy2d(c.create_string_buffer(struct.pack(
+            '<QQi4xQQQQ' 'QQi4xQQQQ' 'QQ', 0, 0, src_type, src_host or 0, 0, src_array or 0, 16,
+            0, 0, dst_type, dst_host or 0, 0, dst_array or 0, 16, 16, 2)))
+    expect('packed upload', copy2d(1, c.addressof(host_in), None, 3, None, packed.value))
+    expect('packed readback', copy2d(3, None, packed.value, 1, c.addressof(host_out), None))
+    assert host_out.raw == host_in.raw[:32], (host_out.raw.hex(), host_in.raw.hex())
+    expect('packed array destroy', array_destroy(packed))
+
+    expect('module unload', unload(module))
+    expect('double unload', unload(module), 400)
+    expect('lookup after unload', get(c.byref(function), module, b'dltss_pwin_enc0_layer'), 400)
+    expect('context destroy', A('cuCtxDestroy_v2', [ptr])(context))
+    print(json.dumps({'passed': len(checks), 'checks': checks}, indent=2))
+
+
+if __name__ == '__main__':
+    main()

tools/microcuda/make_pack.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Offline MicroCUDA module pack: ZLUDA used as an offline compiler.
+
+make_pack.py DLL ZLUDA_CACHE_DB ZLUDA_LIBCUDA OUT_DIR [--switches S] [--arch gfx1101]
+
+ZLUDA keeps every module it compiles in its cache database (keyed by the blake3 of the PTX text, the
+ZLUDA build and its code-generation switches) as a complete AMDGPU code object. This collects, for
+every fatbin in the DLSS DLL, the object ZLUDA compiled for it with one exact ZLUDA build (identified
+like ZLUDA does, by the size and mtime of its libcuda.so) and one switch set, so MicroCUDA can load
+it without the PTX translator. Fatbins without such an object are listed and left out: MicroCUDA
+rejects them instead of running anything else. Native kernel overrides are not part of the pack; the
+bridge serves those as for ZLUDA.
+
+Output: OUT_DIR/modules/<fatbin fnv>.co and OUT_DIR/microcuda-pack.txt, read by the runtime. Fatbins without
+PTX become 'empty' entries: ZLUDA loads those as modules without functions.
+Needs the blake3 Python package. No PTX or NVIDIA code is written; the objects are local build
+products like ZLUDA's cache itself.
+"""
+import argparse
+import collections
+import hashlib
+import json
+import os
+from pathlib import Path
+import sqlite3
+import struct
+import sys
+
+import blake3
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'kernels/tools'))
+from extract_dlss_ptx import fatbins, ptx_entries
+from kernel_manifest import fnv1a64
+
+# EF_AMDGPU_MACH (low byte of the ELF e_flags) of each supported target
+MACH = {'gfx1100': 0x41, 'gfx1101': 0x46, 'gfx1102': 0x47, 'gfx1103': 0x44, 'gfx1200': 0x48, 'gfx1201': 0x4e}
+
+# switches that do not change generated code (see zluda_build_version in ZLUDA's module.rs)
+IGNORED_SWITCHES = ('D4R_ZLUDA_CACHE_HOME=',)
+
+
+def fingerprint(libcuda):
+    s = os.stat(libcuda)
+    return f'{s.st_size:x}.{int(s.st_mtime):x}.{s.st_mtime_ns % 1_000_000_000:x}'
+
+
+def switches_of(version):
+    # "<git sha>+<library fingerprint>[+<sorted D4R_ZLUDA_ switches>]"
+    parts = version.split('+', 2)
+    raw = parts[2] if len(parts) == 3 else ''
+    return ','.join(s for s in raw.split(',') if s and not s.startswith(IGNORED_SWITCHES))
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('dll', type=Path)
+    p.add_argument('db', type=Path)
+    p.add_argument('libcuda', type=Path)
+    p.add_argument('out', type=Path)
+    p.add_argument('--switches', help='code-generation switch set to take when the cache holds several')
+    p.add_argument('--arch', default='gfx1101')
+    a = p.parse_args()
+
+    fp = fingerprint(a.libcuda)
+    db = sqlite3.connect(f'file:{a.db}?mode=ro', uri=True)
+    rows = collections.defaultdict(list)
+    for h, version, attrs, rowid in db.execute(
+            'select hash, zluda_version, backend_key, id from modules where device = ? and zluda_version like ?',
+            (a.arch, f'%+{fp}%')):
+        rows[h].append((switches_of(version), version, attrs, rowid))
+    sets = collections.Counter(s for v in rows.values() for s, *_ in v)
+    if not sets:
+        sys.exit(f'no {a.arch} modules for ZLUDA build {fp} in {a.db}')
+    if a.switches is None:
+        if len(sets) > 1:
+            sys.exit('several switch sets in the cache; pick one with --switches:\n  ' +
+                     '\n  '.join(f'{n:4} {s!r}' for s, n in sets.most_common()))
+        a.switches = next(iter(sets))
+
+    dll = a.dll.read_bytes()
+    (a.out / 'modules').mkdir(parents=True, exist_ok=True)
+    lines, missing, attrs_seen, version_seen = [], [], set(), set()
+    for index, (_, fatbin) in enumerate(fatbins(dll)):
+        ffnv = fnv1a64(fatbin)
+        # ZLUDA compiles the last PTX of a fatbin that it can use; take the last one it compiled
+        ptxs = list(ptx_entries(fatbin))
+        if not ptxs:
+            # no PTX (only cubins for NVIDIA GPUs): ZLUDA's release build loads an empty module, so must we
+            lines.append(f'empty {ffnv:016x} {len(fatbin)}')
+            continue
+        chosen = None
+        for ptx in reversed(ptxs):
+            text = ptx.rstrip(b'\0')
+            for s, version, attrs, rowid in rows.get(blake3.blake3(text).hexdigest(), []):
+                if s == a.switches:
+                    chosen = (text, version, attrs, rowid)
+                    break
+            if chosen:
+                break
+        if chosen is None:
+            missing.append(f'{index}:{ffnv:016x}')
+            continue
+        text, version, attrs, rowid = chosen
+        attrs_seen.add(attrs)
+        version_seen.add(version)
+        binary = db.execute('select binary from modules where id = ?', (rowid,)).fetchone()[0]
+        if binary[:4] != b'\x7fELF' or struct.unpack_from('<I', binary, 0x30)[0] & 0xff != MACH[a.arch]:
+            sys.exit(f'fatbin {index}: cached object is not a {a.arch} code object')
+        name = f'{ffnv:016x}.co'
+        (a.out / 'modules' / name).write_bytes(binary)
+        lines.append(f'module {ffnv:016x} {len(fatbin)} {fnv1a64(text):016x} {len(text)} '
+                     f'{hashlib.sha256(binary).hexdigest()} {name}')
+    if len(attrs_seen) > 1 or len(version_seen) > 1:
+        sys.exit(f'inconsistent compile attributes in the chosen set: {attrs_seen} {version_seen}')
+    fp8 = json.loads(next(iter(attrs_seen))).get('wmma_fp8_native', False) if attrs_seen else False
+    header = [
+        '# d4r MicroCUDA module pack (generated by tools/microcuda/make_pack.py; do not edit)',
+        f'# dll sha256 {hashlib.sha256(dll).hexdigest()}',
+        f'# zluda {next(iter(version_seen))}',
+        f'# attributes {next(iter(attrs_seen))}',
+        f'arch {a.arch}',
+        f'switches {a.switches}',
+        # RDNA4: compiled with native FP8 WMMA (d4r.ini NativeFp8); the runtime refuses a mismatched process
+        f'fp8native {int(fp8)}',
+    ]
+    (a.out / 'microcuda-pack.txt').write_text('\n'.join(header + sorted(lines)) + '\n')
+    print(f'{sum(l.startswith("module") for l in lines)} modules and {sum(l.startswith("empty") for l in lines)} empty modules packed for {a.arch}, switches {a.switches!r}; '
+          f'{len(missing)} fatbins without a compiled object')
+    if missing:
+        (a.out / 'missing.txt').write_text('\n'.join(missing) + '\n')
+
+
+if __name__ == '__main__':
+    main()

tools/microcuda/make_registry.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Offline inventory/registry. No proprietary PTX is emitted into the header.
+
+make_registry.py DLL NATIVE_DIR OUT_DIR [NAME ...]
+Only audited K enc0 export is enabled for now. Inventory covers every PTX.
+Hash identity: existing PTX FNV plus full fatbin FNV+size to avoid runtime LZ4.
+"""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'kernels/tools'))
+from extract_dlss_ptx import fatbins, ptx_entries
+from kernel_manifest import fnv1a64
+
+ENTRY = re.compile(rb'\.entry\s+([\w$]+)\s*\((.*?)\)', re.S)
+AUDITED = {'dltss_pwin_enc0_layer'}
+
+
+def generate(dll, native, out, names):
+    if not set(names) <= AUDITED:
+        raise ValueError('Only K enc0 is ABI/phase audited; extend AUDITED after validation')
+    data = dll.read_bytes()
+    manifest = set()
+    for line in (native / 'd4r-kernels.txt').read_text().splitlines():
+        fields = line.split()
+        if len(fields) == 2 and not line.startswith('#'):
+            manifest.add((fields[0], int(fields[1], 16)))
+    inventory, enabled = [], []
+    for i, (_, fatbin) in enumerate(fatbins(data)):
+        for j, ptx in enumerate(ptx_entries(fatbin)):
+            ptx = ptx.rstrip(b'\0')
+            entries = ENTRY.findall(ptx)
+            phash, fhash = fnv1a64(ptx), fnv1a64(fatbin)
+            row = {'fatbin_index': i, 'ptx_index': j, 'ptx_hash': f'{phash:016x}',
+                   'fatbin_hash': f'{fhash:016x}', 'fatbin_bytes': len(fatbin),
+                   'kernels': []}
+            for raw_name, params in entries:
+                name = raw_name.decode()
+                m = re.fullmatch(rb'\s*\.param\s+\.align\s+(\d+)\s+\.b8\s+\w+\[(\d+)\]\s*', params)
+                abi = {'align': int(m[1]), 'bytes': int(m[2])} if m else None
+                row['kernels'].append({'name': name, 'aggregate_abi': abi,
+                    'native_manifest_match': (name, phash) in manifest,
+                    'native_file': (native / (name + '.hsaco')).is_file()})
+                if name not in names:
+                    continue
+                if abi != {'align': 8, 'bytes': 176}:
+                    raise ValueError(f'{name}: unsupported module/ABI')
+                if (name, phash) not in manifest:
+                    raise ValueError(f'{name}: PTX hash not verified by native manifest')
+                hsaco = native / (name + '.hsaco')
+                notes = subprocess.check_output(['llvm-readobj', '--notes', str(hsaco)], text=True)
+                # Verify both prep and main kernargs before generating a runnable registry.
+                for symbol in [name + '_prep', name]:
+                    blocks = notes.split('  - .args:')[1:]
+                    block = next((b for b in blocks if re.search(r'\.name:\s+' + re.escape(symbol) + r'\s', b)), '')
+                    if not (re.search(r'\.size:\s+176\s', block) and
+                            re.search(r'\.offset:\s+0\s', block) and
+                            re.search(r'\.kernarg_segment_size:\s+176\s', block) and
+                            block.count('.value_kind:') == 1 and
+                            re.search(r'\.value_kind:\s+by_value\s', block) and
+                            re.search(r'\.kernarg_segment_align:\s+8\s', block)):
+                        raise ValueError(f'{symbol}: HSACO argument ABI mismatch')
+                arch = re.search(r'amdhsa.target:\s+amdgcn-amd-amdhsa--(gfx\d+)', notes)
+                if not arch:
+                    raise ValueError('Missing exact GPU target in HSACO')
+                enabled.append((phash, len(ptx), fhash, len(fatbin), name, abi['bytes'],
+                                arch[1], hashlib.sha256(hsaco.read_bytes()).hexdigest()))
+            inventory.append(row)
+    if {r[4] for r in enabled} != set(names):
+        raise ValueError('Requested kernel absent from DLL')
+    out.mkdir(parents=True, exist_ok=True)
+    (out / 'inventory.json').write_text(json.dumps({'dll_sha256': hashlib.sha256(data).hexdigest(),
+                                                   'modules': inventory}, indent=2) + '\n')
+    lines = ['// Generated offline; do not edit. Native bytes/ABI were verified at generation.',
+             'static const RegistryEntry registry[] = {']
+    for ph, ps, fh, fs, name, size, arch, digest in sorted(set(enabled)):
+        lines.append(f'    {{0x{ph:016x}ULL, {ps}, 0x{fh:016x}ULL, {fs}, "{name}", {size}, "{arch}", "{digest}"}},')
+    lines += ['};', '']
+    (out / 'registry.h').write_text('\n'.join(lines))
+    print(f'{len(inventory)} PTX records inventoried; {len(enabled)} registry entries enabled')
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('dll', type=Path)
+    p.add_argument('native', type=Path)
+    p.add_argument('out', type=Path)
+    p.add_argument('names', nargs='*', default=['dltss_pwin_enc0_layer'])
+    a = p.parse_args()
+    generate(a.dll, a.native, a.out, a.names)
+
+
+if __name__ == '__main__':
+    main()

tools/microcuda/merge_pack.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Per-module compile tuning: build a pack from several variant packs of the same DLL, ZLUDA build and target.
+
+merge_pack.py BASE_PACK OUT_PACK [--all VARIANT_PACK] [--choose CHOICES.json]
+
+BASE_PACK fixes the header (switch set the process must run with) and supplies every module not chosen
+otherwise. --all takes every module from one variant pack (to measure that variant); --choose takes a JSON
+map {fatbin_fnv_hex: variant_pack_dir}. Variants must only differ in code-generation choices that keep
+results bit-exact (scheduler strategy, wave size); the caller validates the merged pack's output against the
+reference before using it. Each module line keeps its own SHA256, and a comment records its variant.
+"""
+import argparse
+import json
+from pathlib import Path
+import shutil
+import sys
+
+
+def read(pack):
+    header, modules = [], {}
+    for line in (pack / 'microcuda-pack.txt').read_text().splitlines():
+        if line.startswith(('module ', 'empty ')):
+            modules[line.split()[1]] = line
+        else:
+            header.append(line)
+    return header, modules
+
+
+def field(header, key):
+    return next((l.split(' ', 1)[1] if ' ' in l else '' for l in header if l.split(' ', 1)[0] == key), None)
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('base', type=Path)
+    p.add_argument('out', type=Path)
+    p.add_argument('--all', type=Path)
+    p.add_argument('--choose', type=Path)
+    a = p.parse_args()
+    header, modules = read(a.base)
+    choice = {}
+    if a.all:
+        choice = {f: a.all for f in modules}
+    if a.choose:
+        choice.update({f: Path(v) for f, v in json.loads(a.choose.read_text()).items()})
+    variants = {}
+    if a.out.exists():
+        shutil.rmtree(a.out)
+    (a.out / 'modules').mkdir(parents=True)
+    lines, notes = [], []
+    for f, line in sorted(modules.items()):
+        source = a.base
+        if f in choice and line.startswith('module '):
+            v = choice[f]
+            if v not in variants:
+                vh, vm = read(v)
+                if field(vh, 'arch') != field(header, 'arch') or \
+                        next(l for l in vh if l.startswith('# dll')) != next(l for l in header if l.startswith('# dll')):
+                    sys.exit(f'{v}: different DLL or target')
+                variants[v] = (vh, vm)
+            vh, vm = variants[v]
+            if f in vm:
+                line, source = vm[f], v
+                notes.append(f'# variant {f} {field(vh, "switches")}')
+        if line.startswith('module '):
+            name = line.split()[6]
+            shutil.copyfile(source / 'modules' / name, a.out / 'modules' / name)
+        lines.append(line)
+    (a.out / 'microcuda-pack.txt').write_text('\n'.join(header + notes + lines) + '\n')
+    print(f'{len(lines)} entries, {len(notes)} from variant packs -> {a.out}')
+
+
+if __name__ == '__main__':
+    main()

tools/microcuda/runtime.cpp

@@ -0,0 +1,786 @@
+// Experimental D4R-only Linux CUDA ABI over HIP, without ZLUDA's runtime or PTX translator.
+// Modules come from an offline pack (tools/microcuda/make_pack.py: the code objects ZLUDA compiled
+// for one exact DLSS DLL, ZLUDA build and switch set); anything outside it fails closed. Native
+// kernel overrides, launch handling, arrays and texture/surface objects follow the patched ZLUDA
+// runtime exactly (its module.rs, function.rs, memory.rs, array.rs, surf.rs) so that its code
+// objects and d4r's native kernels see the same ABI. See ZLUDA_REPLACEMENT.md.
+#define __HIP_PLATFORM_AMD__
+#include <hip/hip_runtime_api.h>
+#include <algorithm>
+#include <atomic>
+#include <cmath>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include <openssl/sha.h>
+
+extern char** environ;
+
+using CUresult = int;
+using CUdeviceptr = uint64_t;
+constexpr int SUCCESS=0, INVALID=1, OOM=2, NOT_INIT=3, INVALID_IMAGE=200,
+    INVALID_HANDLE=400, NOT_FOUND=500, NOT_READY=600, UNSUPPORTED=801, UNKNOWN=999;
+
+static int error(hipError_t e) {
+    // HIP errors are not globally interchangeable with CUDA driver errors.
+    switch (e) {
+    case hipSuccess: return SUCCESS;
+    case hipErrorInvalidValue: case hipErrorInvalidConfiguration:
+    case hipErrorInvalidDevicePointer: return INVALID;
+    case hipErrorOutOfMemory: return OOM;
+    case hipErrorNotInitialized: return NOT_INIT;
+    case hipErrorInvalidDevice: return 101;
+    case hipErrorNoDevice: return 100;
+    case hipErrorInvalidImage: return INVALID_IMAGE;
+    case hipErrorInvalidContext: return 201;
+    case hipErrorNoBinaryForGpu: return 209;
+    case hipErrorInvalidHandle: return INVALID_HANDLE;
+    case hipErrorNotFound: case hipErrorInvalidDeviceFunction: return NOT_FOUND;
+    case hipErrorNotReady: return NOT_READY;
+    case hipErrorNotSupported: return UNSUPPORTED;
+    case hipErrorIllegalAddress: return 700;
+    case hipErrorLaunchOutOfResources: return 701;
+    default: return UNKNOWN;
+    }
+}
+static std::atomic<bool> initialized{false};
+static uint64_t fnv(const unsigned char* p, size_t size) {
+    uint64_t h=0xcbf29ce484222325ULL;
+    for(size_t i=0;i<size;++i) h=(h^p[i])*0x100000001b3ULL;
+    return h;
+}
+static void note(const char* format,const char* a,const char* b="") {
+    std::fprintf(stderr,"[d4r-microcuda] "); std::fprintf(stderr,format,a,b); std::fputc('\n',stderr);
+}
+
+// Allocation identity for prep reuse (d4r_prep_identity_at): every cuMemAlloc gets a fresh generation, every
+// host-API write overlapping it bumps its epoch, and every free bumps free_generation (so slot tables keyed by
+// address are reset: NGX recycles addresses after feature recreation). Device-side writes by kernels are not
+// tracked: only weights uploaded by host APIs (NGX's model weights) may use identity reuse.
+struct Allocation { size_t bytes; uint64_t generation; uint64_t epoch; };
+static std::mutex allocations_lock;
+static std::map<uint64_t,Allocation> allocations;
+static uint64_t next_generation=1;
+static std::atomic<uint64_t> free_generation{0};
+static void track_alloc(uint64_t base,size_t bytes) {
+    std::lock_guard<std::mutex> lock(allocations_lock); allocations[base]={bytes,next_generation++,0};
+}
+static void track_free(uint64_t base) {
+    std::lock_guard<std::mutex> lock(allocations_lock); allocations.erase(base);
+    free_generation.fetch_add(1,std::memory_order_release);
+}
+static void track_write(uint64_t dst,size_t bytes) {
+    if(!bytes) return;
+    std::lock_guard<std::mutex> lock(allocations_lock);
+    auto it=allocations.upper_bound(dst);
+    if(it!=allocations.begin()) --it;
+    for(;it!=allocations.end()&&it->first<dst+bytes;++it)
+        if(it->first+it->second.bytes>dst) ++it->second.epoch;
+}
+// -> false if p is not inside a tracked allocation
+static bool identity_of(uint64_t p,uint64_t& generation,uint64_t& epoch) {
+    std::lock_guard<std::mutex> lock(allocations_lock);
+    auto it=allocations.upper_bound(p);
+    if(it==allocations.begin()) return false;
+    --it;
+    if(p-it->first>=it->second.bytes) return false;
+    generation=it->second.generation; epoch=it->second.epoch; return true;
+}
+
+extern "C" {
+int cuInit(unsigned flags) {
+    if(flags) return INVALID;
+    int r=error(hipInit(0)); if(!r) initialized.store(true,std::memory_order_release); return r;
+}
+int cuDeviceGetCount(int* n) { return n?error(hipGetDeviceCount(n)):INVALID; }
+int cuDeviceGet(int* d,int ordinal) { return d?error(hipDeviceGet(d,ordinal)):INVALID; }
+// Contexts are bookkeeping over HIP's per-device state, as in ZLUDA (context.rs): no hipCtx API
+// (deprecated in HIP), just hipSetDevice and a per-thread current-context stack.
+struct Context { int device; };
+static thread_local std::vector<Context*> context_stack;
+static int make_current(Context* c) { return c?error(hipSetDevice(c->device)):SUCCESS; }
+int cuCtxCreate_v2(Context** c,unsigned,int d) {
+    if(!c) return INVALID;
+    int count=0; if(int r=error(hipGetDeviceCount(&count))) return r;
+    if(d<0||d>=count) return 101;
+    *c=new Context{d};
+    if(int r=make_current(*c)) { delete *c; *c=nullptr; return r; }
+    context_stack.push_back(*c); return SUCCESS;
+}
+int cuCtxDestroy_v2(Context* c) {
+    if(!c) return INVALID;
+    context_stack.erase(std::remove(context_stack.begin(),context_stack.end(),c),context_stack.end());
+    delete c; return SUCCESS;
+}
+int cuCtxPushCurrent_v2(Context* c) { if(int r=make_current(c)) return r; context_stack.push_back(c); return SUCCESS; }
+int cuCtxPopCurrent_v2(Context** c) {
+    if(context_stack.empty()) return 201;
+    if(c) *c=context_stack.back();
+    context_stack.pop_back();
+    return context_stack.empty()?SUCCESS:make_current(context_stack.back());
+}
+int cuCtxSetCurrent(Context* c) {
+    if(int r=make_current(c)) return r;
+    if(context_stack.empty()) context_stack.push_back(c); else context_stack.back()=c;
+    if(!c) context_stack.pop_back();
+    return SUCCESS;
+}
+int cuCtxGetCurrent(Context** c) { if(!c) return INVALID; *c=context_stack.empty()?nullptr:context_stack.back(); return SUCCESS; }
+int cuCtxGetDevice(int* d) {
+    if(!d) return INVALID;
+    if(context_stack.empty()||!context_stack.back()) return 201;
+    *d=context_stack.back()->device; return SUCCESS;
+}
+int cuCtxSynchronize() { return error(hipDeviceSynchronize()); }
+int cuDeviceGetName(char* p,int n,int d) { return p&&n>0?error(hipDeviceGetName(p,n,d)):INVALID; }
+int cuDeviceGetUuid(hipUUID* uuid,int d) { return uuid?error(hipDeviceGetUuid(uuid,d)):INVALID; }
+int cuDeviceGetLuid(char* luid,unsigned* mask,int d) {
+    if(!luid||!mask) return INVALID;
+    hipDeviceProp_t props{};
+    int r=error(hipGetDeviceProperties(&props,d));
+    if(!r) { std::memcpy(luid,props.luid,8); *mask=props.luidDeviceNodeMask; }
+    return r;
+}
+int cuDeviceTotalMem_v2(size_t* bytes,int d) { return bytes?error(hipDeviceTotalMem(bytes,d)):INVALID; }
+int cuDeviceComputeCapability(int* major,int* minor,int d) {
+    int ignored; if(!major||!minor) return INVALID;
+    int r=error(hipDeviceGetAttribute(&ignored,hipDeviceAttributeMultiprocessorCount,d));
+    if(!r) { *major=8; *minor=9; } return r; // same NGX sm_89 selection as D4R reference
+}
+int cuDriverGetVersion(int* v) { if(!v) return INVALID; *v=12080; return SUCCESS; }
+int cuDeviceGetAttribute(int* out,int attribute,int d) {
+    if(!out) return INVALID;
+    if(attribute==75||attribute==76) { int ma,mi; int r=cuDeviceComputeCapability(&ma,&mi,d);
+        if(!r) *out=attribute==75?ma:mi;
+        return r; }
+    // Explicit CUDA->HIP enum map. Only the currently audited subset.
+    hipDeviceAttribute_t h;
+    switch(attribute) {
+    case 1: h=hipDeviceAttributeMaxThreadsPerBlock; break;
+    case 2: h=hipDeviceAttributeMaxBlockDimX; break;
+    case 3: h=hipDeviceAttributeMaxBlockDimY; break;
+    case 4: h=hipDeviceAttributeMaxBlockDimZ; break;
+    case 5: h=hipDeviceAttributeMaxGridDimX; break;
+    case 6: h=hipDeviceAttributeMaxGridDimY; break;
+    case 7: h=hipDeviceAttributeMaxGridDimZ; break;
+    case 8: h=hipDeviceAttributeMaxSharedMemoryPerBlock; break;
+    case 10: { int v; int r=error(hipDeviceGetAttribute(&v,hipDeviceAttributeWarpSize,d));
+        if(!r) *out=32;
+        return r; }
+    case 13: h=hipDeviceAttributeClockRate; break;
+    case 16: h=hipDeviceAttributeMultiprocessorCount; break;
+    case 22: h=hipDeviceAttributeMaxTexture2DWidth; break;
+    case 23: h=hipDeviceAttributeMaxTexture2DHeight; break;
+    case 35: { int ignored; int r=error(hipDeviceGetAttribute(&ignored,
+        hipDeviceAttributeMultiprocessorCount,d)); if(!r) *out=0; return r; } // TCC_DRIVER
+    default: return UNSUPPORTED;
+    }
+    return error(hipDeviceGetAttribute(out,h,d));
+}
+// cuMemAlloc zero-fills like ZLUDA (memory.rs alloc_v2): NGX output must not depend on stale VRAM.
+int cuMemAlloc_v2(CUdeviceptr* p,size_t bytes) {
+    if(!p) return INVALID;
+    void* raw=nullptr; int r=error(hipMalloc(&raw,bytes)); if(r) return r;
+    if((r=error(hipMemset(raw,0,bytes)))) { auto ignored=hipFree(raw); (void)ignored; return r; }
+    *p=reinterpret_cast<CUdeviceptr>(raw); track_alloc(*p,bytes); return SUCCESS;
+}
+int cuMemFree_v2(CUdeviceptr p) { track_free(p); return error(hipFree(reinterpret_cast<void*>(p))); }
+int cuMemHostAlloc(void** p,size_t bytes,unsigned flags) { return p?error(hipHostMalloc(p,bytes,flags)):INVALID; }
+int cuMemAllocHost_v2(void** p,size_t bytes) { return cuMemHostAlloc(p,bytes,0); }
+int cuMemFreeHost(void* p) { return error(hipHostFree(p)); }
+int cuMemGetInfo_v2(size_t* free,size_t* total) { return free&&total?error(hipMemGetInfo(free,total)):INVALID; }
+int cuMemcpyHtoD_v2(CUdeviceptr d,const void* s,size_t n) { track_write(d,n); return error(hipMemcpyHtoD(reinterpret_cast<void*>(d),const_cast<void*>(s),n)); }
+int cuMemcpyDtoH_v2(void* d,CUdeviceptr s,size_t n) { return error(hipMemcpyDtoH(d,reinterpret_cast<void*>(s),n)); }
+int cuMemcpyDtoD_v2(CUdeviceptr d,CUdeviceptr s,size_t n) { track_write(d,n); return error(hipMemcpyDtoD(reinterpret_cast<void*>(d),reinterpret_cast<void*>(s),n)); }
+int cuMemcpyHtoDAsync_v2(CUdeviceptr d,const void* s,size_t n,hipStream_t st) { track_write(d,n); return error(hipMemcpyHtoDAsync(reinterpret_cast<void*>(d),const_cast<void*>(s),n,st)); }
+int cuMemcpyDtoHAsync_v2(void* d,CUdeviceptr s,size_t n,hipStream_t st) { return error(hipMemcpyDtoHAsync(d,reinterpret_cast<void*>(s),n,st)); }
+int cuMemcpyDtoDAsync_v2(CUdeviceptr d,CUdeviceptr s,size_t n,hipStream_t st) { track_write(d,n); return error(hipMemcpyDtoDAsync(reinterpret_cast<void*>(d),reinterpret_cast<void*>(s),n,st)); }
+int cuMemsetD8_v2(CUdeviceptr d,unsigned char v,size_t n) { track_write(d,n); return error(hipMemset(reinterpret_cast<void*>(d),v,n)); }
+int cuEventCreate(hipEvent_t* e,unsigned flags) { return e?error(hipEventCreateWithFlags(e,flags)):INVALID; }
+int cuEventDestroy_v2(hipEvent_t e) { return error(hipEventDestroy(e)); }
+int cuEventRecord(hipEvent_t e,hipStream_t s) { return error(hipEventRecord(e,s)); }
+int cuEventQuery(hipEvent_t e) { return error(hipEventQuery(e)); }
+int cuEventSynchronize(hipEvent_t e) { return error(hipEventSynchronize(e)); }
+int cuEventElapsedTime(float* ms,hipEvent_t a,hipEvent_t b) { return ms?error(hipEventElapsedTime(ms,a,b)):INVALID; }
+int cuStreamCreate(hipStream_t* s,unsigned flags) { return s?error(hipStreamCreateWithFlags(s,flags)):INVALID; }
+int cuStreamDestroy_v2(hipStream_t s) { return error(hipStreamDestroy(s)); }
+int cuStreamSynchronize(hipStream_t s) { return error(hipStreamSynchronize(s)); }
+int cuStreamQuery(hipStream_t s) { return error(hipStreamQuery(s)); }
+int cuStreamWaitEvent(hipStream_t s,hipEvent_t e,unsigned flags) { return error(hipStreamWaitEvent(s,e,flags)); }
+// Unimplemented in ZLUDA, which returns NOT_SUPPORTED; the bridge falls back to its own records on that
+// error (descriptors it stored at creation, linear-texture stand-ins), so these must fail the same way.
+int cuDestroyExternalMemory(void*) { return UNSUPPORTED; }
+int cuMipmappedArrayDestroy(void*) { return UNSUPPORTED; }
+int cuGetErrorString(int e,const char** p) {
+    if(!p) return INVALID;
+    switch(e) {
+    case 0: *p="success"; break; case 1: *p="invalid value"; break;
+    case 200: *p="module outside the MicroCUDA pack, or a pack/native file that fails verification"; break;
+    case 801: *p="outside the MicroCUDA subset"; break;
+    default: *p="MicroCUDA error (see numeric CUDA driver code)";
+    } return SUCCESS;
+}
+}
+
+// ---- arrays, 2D copies, texture and surface objects (ZLUDA array.rs, memory.rs, surf.rs) ----
+
+// CUDA formats HIP's legacy descriptor lacks get equivalent backing storage; the original format is
+// remembered per array for surface descriptors and packed copies.
+static std::mutex resources_lock;
+static std::unordered_map<hipArray_t,uint32_t> original_formats;
+static uint32_t original_format(hipArray_t a) {
+    std::lock_guard<std::mutex> lock(resources_lock);
+    auto it=original_formats.find(a); return it==original_formats.end()?0:it->second;
+}
+extern "C" int cuArrayCreate_v2(hipArray_t* handle,const HIP_ARRAY_DESCRIPTOR* desc) {
+    if(!handle||!desc) return INVALID;
+    int format,channels;
+    switch(static_cast<int>(desc->Format)) {
+    case 80: if(desc->NumChannels!=4) return error(hipArrayCreate(handle,desc));
+             format=32; channels=4; break;                 // UNORM 10:10:10:2 -> RGBA32F backing
+    case 192: format=1; channels=1; break; case 193: format=1; channels=2; break;   // UNORM_INT8
+    case 194: format=1; channels=4; break;
+    case 195: format=2; channels=1; break; case 196: format=2; channels=2; break;   // UNORM_INT16
+    case 197: format=2; channels=4; break;
+    case 198: format=8; channels=1; break; case 199: format=8; channels=2; break;   // SNORM_INT8
+    case 200: format=8; channels=4; break;
+    case 201: format=9; channels=1; break; case 202: format=9; channels=2; break;   // SNORM_INT16
+    case 203: format=9; channels=4; break;
+    default: return error(hipArrayCreate(handle,desc));
+    }
+    HIP_ARRAY_DESCRIPTOR backing=*desc;
+    backing.Format=static_cast<hipArray_Format>(format); backing.NumChannels=channels;
+    int r=error(hipArrayCreate(handle,&backing)); if(r) return r;
+    std::lock_guard<std::mutex> lock(resources_lock);
+    original_formats[*handle]=static_cast<uint32_t>(desc->Format); return SUCCESS;
+}
+extern "C" int cuArray3DCreate_v2(hipArray_t* handle,const HIP_ARRAY3D_DESCRIPTOR* desc) {
+    return handle&&desc?error(hipArray3DCreate(handle,desc)):INVALID;
+}
+extern "C" int cuArrayDestroy(hipArray_t a) {
+    int r=error(hipArrayDestroy(a)); if(r) return r;
+    std::lock_guard<std::mutex> lock(resources_lock); original_formats.erase(a); return SUCCESS;
+}
+extern "C" int cuArrayGetDescriptor_v2(void*,hipArray_t) { return UNSUPPORTED; } // as ZLUDA (see above)
+
+static uint32_t packed_component(float v,uint32_t scale) {
+    float c=std::isnan(v)?0.f:std::clamp(v,0.f,1.f);
+    return static_cast<uint32_t>(std::floor(c*static_cast<float>(scale)+0.5f));
+}
+// Packed 10:10:10:2 pixels between public 4-byte words and the RGBA32F backing array.
+static int copy_2d_packed(const hip_Memcpy2D& m,bool src_packed,bool dst_packed) {
+    if(m.WidthInBytes%4||(src_packed&&m.srcXInBytes%4)||(dst_packed&&m.dstXInBytes%4)) return INVALID;
+    const size_t backing_width=m.WidthInBytes*4;
+    if(src_packed&&dst_packed) {
+        hip_Memcpy2D b=m; b.srcXInBytes*=4; b.dstXInBytes*=4; b.WidthInBytes=backing_width;
+        return error(hipMemcpyParam2D(&b));
+    }
+    if((src_packed&&m.dstMemoryType==hipMemoryTypeArray)||(dst_packed&&m.srcMemoryType==hipMemoryTypeArray))
+        return INVALID;
+    const size_t width=m.WidthInBytes;
+    std::vector<float> backing(backing_width/4*m.Height);
+    std::vector<uint8_t> packed(width*m.Height);
+    if(dst_packed) {
+        // external (host or device) packed words -> host staging
+        if(m.srcMemoryType==hipMemoryTypeHost) {
+            if(!m.srcHost||m.srcXInBytes+width>m.srcPitch) return INVALID;
+            for(size_t row=0;row<m.Height;++row)
+                std::memcpy(packed.data()+row*width,static_cast<const uint8_t*>(m.srcHost)+(m.srcY+row)*m.srcPitch+m.srcXInBytes,width);
+        } else if(m.srcMemoryType==hipMemoryTypeDevice||m.srcMemoryType==hipMemoryTypeUnified) {
+            hip_Memcpy2D s=m; s.dstMemoryType=hipMemoryTypeHost; s.dstHost=packed.data();
+            s.dstXInBytes=0; s.dstY=0; s.dstPitch=width;
+            if(int r=error(hipMemcpyParam2D(&s))) return r;
+        } else return UNSUPPORTED;
+        for(size_t p=0;p<packed.size()/4;++p) {
+            uint32_t bits; std::memcpy(&bits,packed.data()+4*p,4);
+            backing[4*p]=static_cast<float>(bits&1023)/1023.f;
+            backing[4*p+1]=static_cast<float>((bits>>10)&1023)/1023.f;
+            backing[4*p+2]=static_cast<float>((bits>>20)&1023)/1023.f;
+            backing[4*p+3]=static_cast<float>(bits>>30)/3.f;
+        }
+        hip_Memcpy2D t=m; t.srcMemoryType=hipMemoryTypeHost; t.srcHost=backing.data();
+        t.srcXInBytes=0; t.srcY=0; t.srcPitch=backing_width; t.dstXInBytes=m.dstXInBytes*4; t.WidthInBytes=backing_width;
+        return error(hipMemcpyParam2D(&t));
+    }
+    hip_Memcpy2D t=m; t.srcXInBytes=m.srcXInBytes*4; t.dstMemoryType=hipMemoryTypeHost; t.dstHost=backing.data();
+    t.dstXInBytes=0; t.dstY=0; t.dstPitch=backing_width; t.WidthInBytes=backing_width;
+    if(int r=error(hipMemcpyParam2D(&t))) return r;
+    for(size_t p=0;p<packed.size()/4;++p) {
+        const float* c=backing.data()+4*p;
+        uint32_t bits=packed_component(c[0],1023)|packed_component(c[1],1023)<<10|
+                      packed_component(c[2],1023)<<20|packed_component(c[3],3)<<30;
+        std::memcpy(packed.data()+4*p,&bits,4);
+    }
+    if(m.dstMemoryType==hipMemoryTypeHost) {
+        if(!m.dstHost||m.dstXInBytes+width>m.dstPitch) return INVALID;
+        for(size_t row=0;row<m.Height;++row)
+            std::memcpy(static_cast<uint8_t*>(m.dstHost)+(m.dstY+row)*m.dstPitch+m.dstXInBytes,packed.data()+row*width,width);
+        return SUCCESS;
+    }
+    if(m.dstMemoryType!=hipMemoryTypeDevice&&m.dstMemoryType!=hipMemoryTypeUnified) return UNSUPPORTED;
+    hip_Memcpy2D s=m; s.srcMemoryType=hipMemoryTypeHost; s.srcHost=packed.data();
+    s.srcXInBytes=0; s.srcY=0; s.srcPitch=width;
+    return error(hipMemcpyParam2D(&s));
+}
+// CUDA_MEMCPY2D has hip_Memcpy2D's layout, but CUDA numbers memory types HOST 1, DEVICE 2, ARRAY 3,
+// UNIFIED 4 where HIP uses 1, 2, 10, 11.
+static bool memory_type(int cuda,hipMemoryType& hip) {
+    switch(cuda) {
+    case 1: hip=hipMemoryTypeHost; return true; case 2: hip=hipMemoryTypeDevice; return true;
+    case 3: hip=hipMemoryTypeArray; return true; case 4: hip=hipMemoryTypeUnified; return true;
+    default: return false;
+    }
+}
+extern "C" int cuMemcpy2D_v2(const hip_Memcpy2D* cuda) {
+    if(!cuda) return INVALID;
+    hip_Memcpy2D m=*cuda;
+    if(!memory_type(static_cast<int>(cuda->srcMemoryType),m.srcMemoryType)||
+       !memory_type(static_cast<int>(cuda->dstMemoryType),m.dstMemoryType)) return UNSUPPORTED;
+    if(m.dstMemoryType==hipMemoryTypeDevice||m.dstMemoryType==hipMemoryTypeUnified)
+        track_write(reinterpret_cast<uint64_t>(m.dstDevice)+m.dstY*m.dstPitch+m.dstXInBytes,
+                    m.Height?(m.Height-1)*m.dstPitch+m.WidthInBytes:0);
+    bool src_packed=m.srcMemoryType==hipMemoryTypeArray&&original_format(m.srcArray)==80;
+    bool dst_packed=m.dstMemoryType==hipMemoryTypeArray&&original_format(m.dstArray)==80;
+    if(!src_packed&&!dst_packed) return error(hipMemcpyParam2D(&m));
+    return copy_2d_packed(m,src_packed,dst_packed);
+}
+
+extern "C" int cuTexObjectCreate(hipTextureObject_t* object,const HIP_RESOURCE_DESC* resource,
+    const HIP_TEXTURE_DESC* texture,const HIP_RESOURCE_VIEW_DESC* view) {
+    return object&&resource&&texture?error(hipTexObjectCreate(object,resource,texture,view)):INVALID;
+}
+extern "C" int cuTexObjectDestroy(hipTextureObject_t object) { return error(hipTexObjectDestroy(object)); }
+extern "C" int cuTexObjectGetResourceDesc(void*,hipTextureObject_t) { return UNSUPPORTED; } // as ZLUDA
+
+// A surface handle is device memory: a copy of HIP's 48-byte image object (so the handle is also a
+// valid texture object for texel fetches), a zeroed 32-byte sampler slot, then the CUDA format word
+// and padding the bridge's output redirect writes into (bytes 84..95).
+constexpr size_t IMAGE_OBJECT_BYTES=48, SAMPLER_OBJECT_BYTES=32,
+    FORMAT_OFFSET=IMAGE_OBJECT_BYTES+SAMPLER_OBJECT_BYTES, SURFACE_HANDLE_BYTES=FORMAT_OFFSET+16;
+struct Surface { hipSurfaceObject_t native; HIP_RESOURCE_DESC desc; };
+static std::unordered_map<uint64_t,Surface> surfaces;
+extern "C" int cuSurfObjectCreate(uint64_t* object,const HIP_RESOURCE_DESC* resource) {
+    if(!object||!resource) return INVALID;
+    uint32_t format=resource->resType==HIP_RESOURCE_TYPE_ARRAY?original_format(resource->res.array.hArray):0;
+    hipSurfaceObject_t native{};
+    int r=error(hipCreateSurfaceObject(&native,reinterpret_cast<const hipResourceDesc*>(resource))); if(r) return r;
+    void* handle=nullptr;
+    if((r=error(hipMalloc(&handle,SURFACE_HANDLE_BYTES)))) { auto ignored=hipDestroySurfaceObject(native); (void)ignored; return r; }
+    uint32_t tail[(SURFACE_HANDLE_BYTES-IMAGE_OBJECT_BYTES)/4]{};
+    tail[(FORMAT_OFFSET-IMAGE_OBJECT_BYTES)/4]=format;
+    r=error(hipMemcpyDtoD(handle,reinterpret_cast<void*>(native),IMAGE_OBJECT_BYTES));
+    if(!r) r=error(hipMemcpyHtoD(static_cast<uint8_t*>(handle)+IMAGE_OBJECT_BYTES,tail,sizeof(tail)));
+    if(r) { auto a=hipFree(handle); auto b=hipDestroySurfaceObject(native); (void)a; (void)b; return r; }
+    std::lock_guard<std::mutex> lock(resources_lock);
+    surfaces[reinterpret_cast<uint64_t>(handle)]={native,*resource};
+    *object=reinterpret_cast<uint64_t>(handle); return SUCCESS;
+}
+extern "C" int cuSurfObjectDestroy(uint64_t object) {
+    Surface s;
+    { std::lock_guard<std::mutex> lock(resources_lock);
+      auto it=surfaces.find(object); if(it==surfaces.end()) return INVALID; s=it->second; surfaces.erase(it); }
+    int native=error(hipDestroySurfaceObject(s.native));
+    int freed=error(hipFree(reinterpret_cast<void*>(object)));
+    return native?native:freed;
+}
+extern "C" int cuSurfObjectGetResourceDesc(HIP_RESOURCE_DESC* desc,uint64_t object) {
+    if(!desc) return INVALID;
+    std::lock_guard<std::mutex> lock(resources_lock);
+    auto it=surfaces.find(object); if(it==surfaces.end()) return INVALID;
+    *desc=it->second.desc; return SUCCESS;
+}
+
+// ---- module pack ----
+
+struct PackEntry { uint64_t fatbin_hash; size_t fatbin_bytes; uint64_t ptx_hash; size_t ptx_bytes;
+                   std::string sha256, file; bool empty=false; };
+struct Pack { bool loaded=false; int status=NOT_FOUND; std::string dir, arch, switches; int fp8native=0;
+              std::vector<PackEntry> entries; };
+static std::mutex pack_lock;
+static Pack pack;
+
+// The code-generation switch set ZLUDA would key its cache with (module.rs zluda_build_version),
+// minus settings that do not change generated code. Pack and process must agree.
+static std::string process_switches() {
+    static const char* ignored[]={"D4R_ZLUDA_NATIVE_DIR","D4R_ZLUDA_DIR","D4R_ZLUDA_DUMP_DIR",
+        "D4R_ZLUDA_DUMP_PTX_DIR","D4R_ZLUDA_LIBCUDA","D4R_ZLUDA_WMMA_FP8_NATIVE","D4R_ZLUDA_CACHE_HOME"};
+    std::vector<std::string> out;
+    for(char** e=environ;*e;++e) {
+        std::string kv=*e; if(kv.rfind("D4R_ZLUDA_",0)) continue;
+        size_t eq=kv.find('='); if(eq==std::string::npos) continue;
+        std::string key=kv.substr(0,eq), value=kv.substr(eq+1);
+        if(std::any_of(std::begin(ignored),std::end(ignored),[&](const char* i) { return key==i; })) continue;
+        size_t b=value.find_first_not_of(" \t\r\n"), e2=value.find_last_not_of(" \t\r\n");
+        out.push_back(key+"="+(b==std::string::npos?"":value.substr(b,e2-b+1)));
+    }
+    std::sort(out.begin(),out.end());
+    std::string joined; for(auto& s:out) { if(!joined.empty()) joined+=','; joined+=s; }
+    return joined;
+}
+static int load_pack() {
+    std::lock_guard<std::mutex> lock(pack_lock);
+    if(pack.loaded) return pack.status;
+    pack.loaded=true;
+    const char* dir=std::getenv("D4R_MICROCUDA_PACK");
+    if(!dir||!*dir) { note("%s","D4R_MICROCUDA_PACK is not set; no module can be loaded"); return pack.status=NOT_FOUND; }
+    pack.dir=dir;
+    FILE* f=std::fopen((pack.dir+"/microcuda-pack.txt").c_str(),"r");
+    if(!f) {
+        // a directory of packs, one per GPU target (<dir>/gfx1101/microcuda-pack.txt, ...): take this GPU's
+        hipDeviceProp_t props{}; int device=0;
+        if(hipGetDevice(&device)==hipSuccess&&hipGetDeviceProperties(&props,device)==hipSuccess) {
+            std::string arch=props.gcnArchName; arch=arch.substr(0,arch.find(':'));
+            pack.dir=std::string(dir)+"/"+arch;
+            f=std::fopen((pack.dir+"/microcuda-pack.txt").c_str(),"r");
+        }
+    }
+    if(!f) { note("cannot read %s/microcuda-pack.txt",dir); return pack.status=NOT_FOUND; }
+    char line[1024];
+    while(std::fgets(line,sizeof(line),f)) {
+        std::string s=line; while(!s.empty()&&(s.back()=='\n'||s.back()=='\r')) s.pop_back();
+        if(s.empty()||s[0]=='#') continue;
+        if(s.rfind("arch ",0)==0) { pack.arch=s.substr(5); continue; }
+        if(s.rfind("switches",0)==0) { pack.switches=s.size()>9?s.substr(9):""; continue; }
+        if(s.rfind("fp8native ",0)==0) { pack.fp8native=std::atoi(s.c_str()+10); continue; }
+        PackEntry e; char sha[65], file[256]; unsigned long long fh,ph; size_t fb,pb;
+        if(std::sscanf(line,"empty %llx %zu",&fh,&fb)==2) {
+            e.fatbin_hash=fh; e.fatbin_bytes=fb; e.ptx_hash=0; e.ptx_bytes=SIZE_MAX; e.empty=true;
+            pack.entries.push_back(e); continue;
+        }
+        if(std::sscanf(line,"module %llx %zu %llx %zu %64s %255s",&fh,&fb,&ph,&pb,sha,file)!=6) continue;
+        e.fatbin_hash=fh; e.fatbin_bytes=fb; e.ptx_hash=ph; e.ptx_bytes=pb; e.sha256=sha; e.file=file;
+        pack.entries.push_back(e);
+    }
+    std::fclose(f);
+    std::string current=process_switches();
+    if(current!=pack.switches) {
+        note("pack was compiled with ZLUDA switches '%s', this process has '%s'; refusing the pack",
+             pack.switches.c_str(),current.c_str());
+        return pack.status=INVALID_IMAGE;
+    }
+    return pack.status=pack.entries.empty()?NOT_FOUND:SUCCESS;
+}
+// RDNA4 objects differ with native FP8 WMMA (ZLUDA keys that by effect: gfx12 with WMMA and NativeFp8 on).
+static bool fp8_matches(const std::string& arch) {
+    auto on=[](const char* k) { const char* v=std::getenv(k); return v&&std::strcmp(v,"1")==0; };
+    bool expected=arch.rfind("gfx12",0)==0&&on("D4R_ZLUDA_WMMA")&&on("D4R_ZLUDA_WMMA_FP8_NATIVE");
+    return expected==(pack.fp8native!=0);
+}
+static bool read_file(const std::string& path,std::vector<unsigned char>& bytes) {
+    FILE* file=std::fopen(path.c_str(),"rb"); if(!file) return false;
+    unsigned char buffer[65536]; size_t count;
+    while((count=std::fread(buffer,1,sizeof(buffer),file))) bytes.insert(bytes.end(),buffer,buffer+count);
+    bool failed=std::ferror(file); std::fclose(file); return !failed;
+}
+static bool digest_matches(const std::vector<unsigned char>& data,const std::string& expected) {
+    unsigned char digest[SHA256_DIGEST_LENGTH]; SHA256(data.data(),data.size(),digest);
+    char hex[65]; for(size_t i=0;i<32;++i) std::sprintf(hex+2*i,"%02x",digest[i]);
+    return expected==hex;
+}
+
+// ---- modules and functions (ZLUDA module.rs / function.rs) ----
+
+struct Post { hipFunction_t function; uint32_t gx, gy, bx, by, bz; };
+struct Function {
+    hipFunction_t main=nullptr, prep=nullptr;
+    uint32_t prep_blocks=0, block_z=0, grid_x=0, prep_slots=0;
+    bool has_prep_key=false; uint32_t prep_key=0;
+    std::atomic<uint64_t> prep_last{0};
+    std::mutex seen_lock; std::vector<uint64_t> seen;
+    std::vector<Post> post;
+    bool native=false;
+    std::atomic<uint64_t> prep_runs{0}, prep_skips{0};
+    // identity policy (d4r_prep_identity_at): prepared images per weights pointer, valid while the pointer's
+    // allocation generation and write epoch are unchanged and no allocation has been freed since the slot
+    // table was last reset. Only on one stream; another stream turns the policy off for this function.
+    uint32_t identity_at=0;
+    hipDeviceptr_t prep_keys=nullptr; size_t prep_keys_bytes=0;
+    std::mutex identity_lock;
+    struct Prepared { uint64_t key, generation, epoch; };
+    std::vector<Prepared> prepared;
+    uint64_t prepared_free_generation=0;
+    hipStream_t identity_stream=nullptr; bool identity_stream_set=false, identity_off=false;
+};
+struct NativeImage { hipModule_t module; std::vector<unsigned char> bytes; };
+struct Module {
+    hipModule_t hip=nullptr;
+    std::vector<unsigned char> image; // retained until HIP module unload
+    std::mutex lock;
+    std::map<std::string,std::unique_ptr<Function>> functions;
+    std::vector<std::unique_ptr<NativeImage>> natives;
+};
+static std::mutex modules_lock;
+static std::vector<Module*> modules;
+static bool registered(Module* m) { return std::find(modules.begin(),modules.end(),m)!=modules.end(); }
+
+static bool global_u32(hipModule_t m,const char* name,uint32_t& value) {
+    hipDeviceptr_t ptr=nullptr; size_t bytes=0;
+    return hipModuleGetGlobal(&ptr,&bytes,m,name)==hipSuccess&&bytes==4&&hipMemcpyDtoH(&value,ptr,4)==hipSuccess;
+}
+// d4r native replacement served by the bridge as DIR/NAME.hsaco (only for PTX it verified); same
+// metadata contract as ZLUDA's native_override. Returns false to use the pack's compile of NAME.
+static bool native_override(Module* m,const std::string& name,Function& f) {
+    const char* dir=std::getenv("D4R_MICROCUDA_NATIVE_DIR"); if(!dir||!*dir) return false;
+    auto image=std::make_unique<NativeImage>();
+    std::string path=std::string(dir)+"/"+name+".hsaco";
+    if(!read_file(path,image->bytes)||image->bytes.empty()) return false;
+    if(hipModuleLoadData(&image->module,image->bytes.data())!=hipSuccess) { note("failed to load %s",path.c_str()); return false; }
+    hipModule_t nm=image->module;
+    auto reject=[&](const char* why) { note("%s: %s; using the pack's kernel",path.c_str(),why);
+        auto ignored=hipModuleUnload(nm); (void)ignored; return false; };
+    if(hipModuleGetFunction(&f.main,nm,name.c_str())!=hipSuccess) return reject("no kernel of that name");
+    if(hipModuleGetFunction(&f.prep,nm,(name+"_prep").c_str())==hipSuccess) {
+        if(!global_u32(nm,"d4r_prep_blocks",f.prep_blocks)||!f.prep_blocks) return reject("prep kernel without d4r_prep_blocks");
+    } else f.prep=nullptr;
+    global_u32(nm,"d4r_block_z",f.block_z);
+    global_u32(nm,"d4r_grid_x",f.grid_x);
+    // d4r_prep_key_at holds offset + 1; the older d4r_prep_key_offset uses 0 for "no key"
+    uint32_t at=0, offset=0;
+    if(global_u32(nm,"d4r_prep_key_at",at)&&at>0) { f.has_prep_key=true; f.prep_key=at-1; }
+    else if(global_u32(nm,"d4r_prep_key_offset",offset)&&offset) { f.has_prep_key=true; f.prep_key=offset; }
+    global_u32(nm,"d4r_prep_key_slots",f.prep_slots);
+    if(f.prep&&global_u32(nm,"d4r_prep_identity_at",f.identity_at)&&f.identity_at) {
+        // the slot table the prep claims into (keys only ever go from 0 to a pointer until reset)
+        if(hipModuleGetGlobal(&f.prep_keys,&f.prep_keys_bytes,nm,"g_prep_keys")!=hipSuccess||
+           f.prep_keys_bytes!=(f.prep_slots+1)*sizeof(uint64_t)||!f.prep_slots) {
+            note("%s: d4r_prep_identity_at without a matching g_prep_keys slot table; preparing every launch",path.c_str());
+            f.identity_at=0;
+        }
+    }
+    for(unsigned k=1;k<=8;++k) {
+        Post p{};
+        if(hipModuleGetFunction(&p.function,nm,(name+"_post"+std::to_string(k)).c_str())!=hipSuccess) break;
+        auto field=[&](const char* s) { uint32_t v=0; global_u32(nm,("d4r_post"+std::to_string(k)+"_"+s).c_str(),v); return v; };
+        p.gx=field("grid_x"); p.gy=field("grid_y");
+        p.bx=field("block_x"); p.by=field("block_y"); p.bz=field("block_z");
+        if(!p.bx) p.bx=32;
+        if(!p.by) p.by=1;
+        if(!p.bz) p.bz=1;
+        f.post.push_back(p);
+    }
+    f.native=true;
+    m->natives.push_back(std::move(image));
+    return true;
+}
+
+static int load_module(Module** out,const void* image) {
+    if(!out||!image) return INVALID;
+    *out=nullptr;
+    if(!initialized.load(std::memory_order_acquire)) return NOT_INIT;
+    if(int r=load_pack()) return r;
+    const auto* p=static_cast<const unsigned char*>(image);
+    uint32_t magic; std::memcpy(&magic,p,4);
+    const PackEntry* entry=nullptr;
+    if(magic==0xba55ed50U) {
+        uint16_t version,header; uint64_t payload;
+        std::memcpy(&version,p+4,2); std::memcpy(&header,p+6,2); std::memcpy(&payload,p+8,8);
+        if(version!=1||header!=16||payload>256*1024*1024) return INVALID_IMAGE;
+        size_t n=header+payload; uint64_t h=0; bool hashed=false;
+        for(const auto& e:pack.entries) if(n==e.fatbin_bytes) {
+            if(!hashed) { h=fnv(p,n); hashed=true; }
+            if(h==e.fatbin_hash) { entry=&e; break; }
+        }
+    } else {
+        // PTX text: identity of the text without trailing NULs, as the bridge's native manifest
+        size_t n=strnlen(reinterpret_cast<const char*>(p),256*1024*1024); uint64_t h=fnv(p,n);
+        for(const auto& e:pack.entries) if(n==e.ptx_bytes&&h==e.ptx_hash) { entry=&e; break; }
+    }
+    if(!entry) return INVALID_IMAGE;
+    hipDeviceProp_t props{}; int device;
+    if(int r=error(hipGetDevice(&device))) return r;
+    if(int r=error(hipGetDeviceProperties(&props,device))) return r;
+    std::string arch=props.gcnArchName; arch=arch.substr(0,arch.find(':'));
+    if(arch!=pack.arch) return 209;
+    if(!fp8_matches(arch)) { note("%s pack was built with NativeFp8=%s; the process setting differs",arch.c_str(),
+                                  pack.fp8native?"1":"0"); return INVALID_IMAGE; }
+    auto m=std::make_unique<Module>();
+    if(entry->empty) { // fatbin without PTX: a module without functions, as ZLUDA loads it
+        std::lock_guard<std::mutex> lock(modules_lock); modules.push_back(m.get()); *out=m.release(); return SUCCESS;
+    }
+    if(!read_file(pack.dir+"/modules/"+entry->file,m->image)||!digest_matches(m->image,entry->sha256)) {
+        note("%s/modules/%s is missing or fails its digest",pack.dir.c_str(),entry->file.c_str());
+        return INVALID_IMAGE;
+    }
+    if(int r=error(hipModuleLoadData(&m->hip,m->image.data()))) return r;
+    std::lock_guard<std::mutex> lock(modules_lock); modules.push_back(m.get()); *out=m.release(); return SUCCESS;
+}
+extern "C" int cuModuleLoadData(Module** out,const void* image) { return load_module(out,image); }
+extern "C" int cuModuleLoadDataEx(Module** m,const void* image,unsigned,int*,void**) {
+    return load_module(m,image); // JIT options are irrelevant to precompiled objects (ZLUDA ignores them too)
+}
+extern "C" int cuModuleLoad(Module** m,const char* path) {
+    if(!m||!path) return INVALID;
+    std::vector<unsigned char> bytes; if(!read_file(path,bytes)) return INVALID;
+    bytes.push_back(0);
+    return load_module(m,bytes.data());
+}
+extern "C" int cuModuleGetFunction(Function** out,Module* m,const char* name) {
+    if(!out||!name) return INVALID;
+    *out=nullptr;
+    { std::lock_guard<std::mutex> lock(modules_lock); if(!registered(m)) return INVALID_HANDLE; }
+    if(!m->hip) return NOT_FOUND; // empty module
+    std::lock_guard<std::mutex> lock(m->lock);
+    auto& slot=m->functions[name];
+    if(!slot) {
+        auto f=std::make_unique<Function>();
+        if(!native_override(m,name,*f)) {
+            f=std::make_unique<Function>();
+            if(int r=error(hipModuleGetFunction(&f->main,m->hip,name))) { m->functions.erase(name); return r; }
+        }
+        slot=std::move(f);
+    }
+    *out=slot.get(); return SUCCESS;
+}
+extern "C" int cuModuleUnload(Module* m) {
+    { std::lock_guard<std::mutex> lock(modules_lock);
+      if(!registered(m)) return INVALID_HANDLE;
+      modules.erase(std::find(modules.begin(),modules.end(),m)); }
+    static const bool stats=std::getenv("D4R_MICROCUDA_STATS")!=nullptr;
+    if(stats) for(auto& [name,f]:m->functions) if(f->prep)
+        std::fprintf(stderr,"[d4r-microcuda] %s: prep runs %llu, skips %llu%s\n",name.c_str(),
+                     (unsigned long long)f->prep_runs.load(),(unsigned long long)f->prep_skips.load(),
+                     f->identity_at?(f->identity_off?" (identity reuse, turned off: several streams)":" (identity reuse)"):"");
+    int r=m->hip?error(hipModuleUnload(m->hip)):SUCCESS;
+    for(auto& n:m->natives) { int nr=error(hipModuleUnload(n->module)); if(!r) r=nr; }
+    delete m; return r;
+}
+extern "C" int d4rMicrocudaGetDispatchStats(Function* f,uint64_t* prep,uint64_t* skips) {
+    if(!f||!prep||!skips) return INVALID;
+    *prep=f->prep_runs.load(std::memory_order_relaxed);
+    *skips=f->prep_skips.load(std::memory_order_relaxed); return SUCCESS;
+}
+
+// Prep under the identity policy; launch(h, grid..., block..., shared) submits on the caller's stream.
+template <class Launch>
+static int identity_prep(Function* f,const unsigned char* block,hipStream_t stream,Launch& launch) {
+    std::lock_guard<std::mutex> lock(f->identity_lock);
+    auto prep=[&] {
+        int r=launch(f->prep,f->prep_blocks,1,1,128,1,1,0);
+        if(!r) f->prep_runs.fetch_add(1,std::memory_order_relaxed);
+        return r;
+    };
+    if(f->identity_off||!block) return prep();
+    if(!f->identity_stream_set) { f->identity_stream=stream; f->identity_stream_set=true; }
+    else if(f->identity_stream!=stream) {
+        // a second stream: slot resets could race with the other stream's reads. Prepare every launch from now on.
+        f->identity_off=true; f->prepared.clear(); return prep();
+    }
+    uint64_t key=0, generation=0, epoch=0;
+    std::memcpy(&key,block+f->identity_at-1,sizeof(key));
+    const uint64_t frees=free_generation.load(std::memory_order_acquire);
+    if(frees!=f->prepared_free_generation) {
+        // an allocation was freed (feature release/recreation): addresses may be recycled. Forget every image
+        // and clear the device slot table on this stream, so no key can match an old image.
+        if(int r=error(hipMemsetAsync(f->prep_keys,0,f->prep_keys_bytes,stream))) return r;
+        f->prepared.clear(); f->prepared_free_generation=frees;
+    }
+    if(!key||!identity_of(key,generation,epoch)) return prep(); // untracked weights: no reuse
+    for(auto& p:f->prepared) if(p.key==key) {
+        if(p.generation==generation&&p.epoch==epoch) { f->prep_skips.fetch_add(1,std::memory_order_relaxed); return SUCCESS; }
+        p.generation=generation; p.epoch=epoch; return prep(); // same slot (claimed by key), new contents
+    }
+    // a new key claims the next free slot; beyond the slots it shares the overflow slot and prepares every launch
+    if(f->prepared.size()<f->prep_slots) f->prepared.push_back({key,generation,epoch});
+    return prep();
+}
+
+// CUDA's packed-argument tokens are POINTER=1, SIZE=2, END=0; HIP's END is 3.
+extern "C" int cuLaunchKernel(Function* f,unsigned gx,unsigned gy,unsigned gz,
+    unsigned bx,unsigned by,unsigned bz,unsigned shared,hipStream_t stream,void** params,void** extra) {
+    if(!f) return INVALID_HANDLE;
+    void* translated[8]{}; void** hip_extra=nullptr; const unsigned char* block=nullptr;
+    if(extra) {
+        size_t i=0, o=0; bool ended=false;
+        while(i<8&&o<8) {
+            uintptr_t token=reinterpret_cast<uintptr_t>(extra[i]);
+            if(token==0) { translated[o]=reinterpret_cast<void*>(uintptr_t{3}); ended=true; break; }
+            if(token!=1&&token!=2) return UNSUPPORTED;
+            if(i+1>=8||o+1>=8) return INVALID;
+            if(token==1) block=static_cast<const unsigned char*>(extra[i+1]);
+            translated[o]=extra[i]; translated[o+1]=extra[i+1]; i+=2; o+=2;
+        }
+        if(!ended) return INVALID;
+        hip_extra=translated;
+    } else if(params) block=static_cast<const unsigned char*>(params[0]); // single-struct kernels
+    auto launch=[&](hipFunction_t h,unsigned x,unsigned y,unsigned z,unsigned tx,unsigned ty,unsigned tz,unsigned sm) {
+        return error(hipModuleLaunchKernel(h,x,y,z,tx,ty,tz,sm,stream,params,hip_extra));
+    };
+    if(f->prep&&f->identity_at) {
+        if(int r=identity_prep(f,block,stream,launch)) return r;
+    } else if(f->prep) {
+        uint64_t key=0;
+        if(f->has_prep_key&&block) std::memcpy(&key,block+f->prep_key,sizeof(key));
+        bool run=true;
+        if(key) {
+            if(!f->prep_slots) run=f->prep_last.load(std::memory_order_relaxed)!=key;
+            else {
+                // one prepared image per key while slots last; keys beyond them prepare every launch
+                std::lock_guard<std::mutex> lock(f->seen_lock);
+                if(std::find(f->seen.begin(),f->seen.end(),key)!=f->seen.end()) run=false;
+                else if(f->seen.size()<f->prep_slots) f->seen.push_back(key);
+            }
+        }
+        if(run) {
+            if(int r=launch(f->prep,f->prep_blocks,1,1,128,1,1,0)) return r;
+            f->prep_last.store(key,std::memory_order_relaxed);
+            f->prep_runs.fetch_add(1,std::memory_order_relaxed);
+        } else f->prep_skips.fetch_add(1,std::memory_order_relaxed);
+    }
+    if(f->block_z) bz=f->block_z;
+    if(f->grid_x) { gx=f->grid_x; gy=1; gz=1; }
+    if(int r=launch(f->main,gx,gy,gz,bx,by,bz,shared)) return r;
+    for(const auto& p:f->post)
+        if(int r=launch(p.function,p.gx?p.gx:gx,p.gy?p.gy:gy,(p.gx||p.gy)?1:gz,p.bx,p.by,p.bz,0)) return r;
+    return SUCCESS;
+}
+
+extern "C" int cuGetProcAddress_v2(const char* name,void** out,int version,uint64_t flags,int* status) {
+    if(!out||!name) return INVALID;
+    *out=nullptr; if(status) *status=1;
+    // No per-thread default-stream ABI or unversioned 32-bit size_t APIs.
+    if(flags>1||version<3020||version>12080) return UNSUPPORTED;
+    struct Symbol { const char* name; void* address; };
+#define SYMBOL(n) {#n,reinterpret_cast<void*>(&n)}
+    static const Symbol symbols[]={
+        SYMBOL(cuInit),SYMBOL(cuDeviceGetCount),SYMBOL(cuDeviceGet),
+        SYMBOL(cuCtxCreate_v2),SYMBOL(cuCtxDestroy_v2),SYMBOL(cuCtxPushCurrent_v2),
+        SYMBOL(cuCtxPopCurrent_v2),SYMBOL(cuCtxSetCurrent),SYMBOL(cuCtxGetCurrent),
+        SYMBOL(cuCtxGetDevice),SYMBOL(cuCtxSynchronize),SYMBOL(cuDriverGetVersion),
+        SYMBOL(cuDeviceGetAttribute),SYMBOL(cuDeviceGetName),SYMBOL(cuDeviceGetUuid),
+        SYMBOL(cuDeviceGetLuid),SYMBOL(cuDeviceTotalMem_v2),SYMBOL(cuDeviceComputeCapability),
+        SYMBOL(cuMemAlloc_v2),SYMBOL(cuMemFree_v2),SYMBOL(cuMemAllocHost_v2),
+        SYMBOL(cuMemHostAlloc),SYMBOL(cuMemFreeHost),SYMBOL(cuMemGetInfo_v2),
+        SYMBOL(cuMemcpyHtoD_v2),SYMBOL(cuMemcpyDtoH_v2),SYMBOL(cuMemcpyDtoD_v2),
+        SYMBOL(cuMemcpyHtoDAsync_v2),SYMBOL(cuMemcpyDtoHAsync_v2),SYMBOL(cuMemcpyDtoDAsync_v2),
+        SYMBOL(cuMemsetD8_v2),SYMBOL(cuMemcpy2D_v2),SYMBOL(cuEventCreate),SYMBOL(cuEventDestroy_v2),
+        SYMBOL(cuEventRecord),SYMBOL(cuEventQuery),SYMBOL(cuEventSynchronize),SYMBOL(cuEventElapsedTime),
+        SYMBOL(cuStreamCreate),SYMBOL(cuStreamDestroy_v2),SYMBOL(cuStreamSynchronize),
+        SYMBOL(cuStreamQuery),SYMBOL(cuStreamWaitEvent),SYMBOL(cuGetErrorString),
+        SYMBOL(cuArrayCreate_v2),SYMBOL(cuArray3DCreate_v2),SYMBOL(cuArrayDestroy),
+        SYMBOL(cuArrayGetDescriptor_v2),SYMBOL(cuMipmappedArrayDestroy),SYMBOL(cuDestroyExternalMemory),
+        SYMBOL(cuTexObjectCreate),SYMBOL(cuTexObjectDestroy),SYMBOL(cuTexObjectGetResourceDesc),
+        SYMBOL(cuSurfObjectCreate),SYMBOL(cuSurfObjectDestroy),SYMBOL(cuSurfObjectGetResourceDesc),
+        SYMBOL(cuModuleLoad),SYMBOL(cuModuleLoadData),SYMBOL(cuModuleLoadDataEx),SYMBOL(cuModuleGetFunction),
+        SYMBOL(cuModuleUnload),SYMBOL(cuLaunchKernel)
+    };
+#undef SYMBOL
+    for(const auto& s:symbols) {
+        size_t n=std::strlen(s.name);
+        bool match=std::strcmp(name,s.name)==0;
+        // GetProcAddress receives CUDA base names and selects the modern ABI.
+        if(n>3&&std::strcmp(s.name+n-3,"_v2")==0)
+            match=match||(std::strlen(name)==n-3&&std::strncmp(name,s.name,n-3)==0);
+        if(match) { *out=s.address; if(status) *status=0; return SUCCESS; }
+    }
+    return NOT_FOUND;
+}
+extern "C" int cuGetProcAddress(const char* name,void** out,int version,uint64_t flags) {
+    return cuGetProcAddress_v2(name,out,version,flags,nullptr);
+}

tools/microcuda/zluda_precompile.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Compile every DLSS fatbin with one ZLUDA build into a cache, for make_pack.py.
+
+LD_LIBRARY_PATH=<ROCm lib dir> zluda_precompile.py DLL ZLUDA_LIBCUDA CACHE_HOME SWITCHES [--jobs N]
+
+SWITCHES is the D4R_ZLUDA_ code-generation set the pack is for, as make_pack.py prints it, e.g.
+'D4R_ZLUDA_IGNORE_DENORMAL=1,D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256,D4R_ZLUDA_WMMA=1,D4R_ZLUDA_WMMA_FP8=1'.
+Each fatbin is loaded with cuModuleLoadData in a worker process (ZLUDA keeps the result in
+CACHE_HOME/zluda/ComputeCache/zluda2.db) and unloaded again; nothing is launched. Already cached
+modules return at once, so an interrupted run can be resumed. Fatbins without PTX fail (ZLUDA has
+nothing to compile) and are left out of the pack. Compiling is CPU heavy and slow for
+the big network modules (a minute or more each): do not run it next to a benchmark.
+"""
+import argparse
+import concurrent.futures
+import ctypes as c
+import os
+from pathlib import Path
+import sys
+import time
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'kernels/tools'))
+from extract_dlss_ptx import fatbins
+
+
+def compile_range(libcuda, dll, indices):
+    lib = c.CDLL(libcuda)
+    for name, args in (('cuInit', [c.c_uint]), ('cuDeviceGet', [c.POINTER(c.c_int), c.c_int]),
+                       ('cuCtxCreate_v2', [c.POINTER(c.c_void_p), c.c_uint, c.c_int]),
+                       ('cuModuleLoadData', [c.POINTER(c.c_void_p), c.c_void_p]),
+                       ('cuModuleUnload', [c.c_void_p])):
+        getattr(lib, name).argtypes = args
+        getattr(lib, name).restype = c.c_int
+    assert lib.cuInit(0) == 0
+    device, context = c.c_int(), c.c_void_p()
+    assert lib.cuDeviceGet(c.byref(device), 0) == 0
+    assert lib.cuCtxCreate_v2(c.byref(context), 0, device.value) == 0
+    images = list(fatbins(Path(dll).read_bytes()))
+    results = []
+    for i in indices:
+        image = c.create_string_buffer(images[i][1])
+        module = c.c_void_p()
+        start = time.time()
+        r = lib.cuModuleLoadData(c.byref(module), image)
+        if r == 0:
+            lib.cuModuleUnload(module)
+        results.append((i, r, time.time() - start))
+    return results
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument('dll', type=Path)
+    p.add_argument('libcuda', type=Path)
+    p.add_argument('cache_home', type=Path)
+    p.add_argument('switches')
+    # ZLUDA's cache is one SQLite file: concurrent writers lose inserts silently (SQLITE_BUSY), so default to 1
+    p.add_argument('--jobs', type=int, default=1)
+    # one process per fatbin: a module the compiler aborts on (e.g. WMMA under D4R_ZLUDA_WAVE64=1) is skipped
+    # instead of ending the run
+    p.add_argument('--isolate', action='store_true')
+    a = p.parse_args()
+    for key in [k for k in os.environ if k.startswith('D4R_ZLUDA_')]:
+        del os.environ[key]
+    for item in filter(None, a.switches.split(',')):
+        key, value = item.split('=', 1)
+        os.environ[key] = value
+    a.cache_home.mkdir(parents=True, exist_ok=True)
+    os.environ['XDG_CACHE_HOME'] = str(a.cache_home.resolve())
+    count = len(list(fatbins(a.dll.read_bytes())))
+    # interleave so every worker gets a mix of small and large modules
+    chunks = [list(range(j, count, a.jobs)) for j in range(a.jobs)]
+    failed = 0
+    if a.isolate:
+        for i in range(count):
+            with concurrent.futures.ProcessPoolExecutor(1) as pool:
+                try:
+                    [(_, r, seconds)] = pool.submit(compile_range, str(a.libcuda), str(a.dll), [i]).result()
+                except concurrent.futures.process.BrokenProcessPool:
+                    r, seconds = 'aborted', 0.0
+            failed += r != 0
+            print(f'fatbin {i:3}: {"ok" if r == 0 else f"CUDA error {r}" if r != "aborted" else "compiler aborted"}'
+                  f' ({seconds:.1f} s)', flush=True)
+    else:
+      with concurrent.futures.ProcessPoolExecutor(a.jobs) as pool:
+        for future in concurrent.futures.as_completed(
+                [pool.submit(compile_range, str(a.libcuda), str(a.dll), chunk) for chunk in chunks]):
+            for i, r, seconds in future.result():
+                failed += r != 0
+                print(f'fatbin {i:3}: {"ok" if r == 0 else f"CUDA error {r}"} ({seconds:.1f} s)', flush=True)
+    print(f'{count - failed}/{count} fatbins compiled into {a.cache_home}/zluda/ComputeCache/zluda2.db')
+
+
+if __name__ == '__main__':
+    main()