engine/rr: native Ray Reconstruction stages with capture/replay and shim integration

- engine/rr: d4r's own shaders for the denoiser frame (exposure, six encoders, five decoders, hkpn output stage) at NGX layouts, plus PTX->GLSL/HIP packing scripts, reference implementations and weight extraction - runtime.h/runtime_rr.inc, game_rr.{h,cpp}: GameDenoiser wired into the engine build (d4r-rr replay binary) and the D3D12 shim's feature-13 path - shim: answer GetFeatureRequirements from d4r's own state only (the old cross-binary CUDA handoff crashed OptiScaler's scan); drop the core's CUDA GetFeatureRequirements import - shim/harness: exact guide capture/replay of a frame's RR inputs - kernels/rr: expanded enc0/engine prototypes; kernels/tools: ptxsim rewrite plus rrsim and an engine test; vulkan: ptx2glsl FP8/cooperative-matrix support, kvk inout buffers and KVK_DISABLE_OPTIMIZATION - bridge: report the surface array in cuSurfObjectCreate traces - packaging: RR settings and the ICU launch overrides in D4R_README
2026-10-12 12:47:53 +02:00
1 parent 30472ae195
commit 50fa65d87b
@@ -10,6 +10,7 @@
 //   lut FILE                    (storage buffer bound after the surfaces)
 //   bufout OFF FILE             (after the dispatch, write the pointer buffer of parameter offset OFF to FILE)
 //   outbuf FILE BYTES           (storage buffer bound after those, written to FILE after the dispatch)
+//   inout FILE OUTFILE          (storage buffer initialised from FILE, bound like outbuf, written to OUTFILE)
 //   bindings N0 N1 ...          (optional: the binding number of each of the above, in that order, for shaders
 //                                whose bindings are not ptx2glsl.py's; e.g. engine/layer_m.comp's L stages)
 #include <vulkan/vulkan.h>
@@ -75,7 +76,10 @@ int main(int argc, char** argv)
    uint32_t np = 8; VkPhysicalDevice pds[8]; vkEnumeratePhysicalDevices(inst, &np, pds);
    VkPhysicalDevice pd = pds[0];
    for (uint32_t i = 0; i < np; ++i) { VkPhysicalDeviceProperties p; vkGetPhysicalDeviceProperties(pds[i], &p); if (p.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) pd = pds[i]; }
-    VkPhysicalDeviceVulkan13Features v13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES};
+    // FP8 cooperative matrices (engine/rr's translated stages, engine/layer_m.comp)
+    VkPhysicalDeviceShaderFloat8FeaturesEXT f8{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT};
+    VkPhysicalDeviceCooperativeMatrixFeaturesKHR cm{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR, &f8};
+    VkPhysicalDeviceVulkan13Features v13{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, &cm};
    VkPhysicalDeviceVulkan12Features v12{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, &v13};
    VkPhysicalDeviceVulkan11Features v11{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES, &v12};
    VkPhysicalDeviceFeatures2 f2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &v11};
@@ -84,7 +88,12 @@ int main(int argc, char** argv)
    uint32_t nq = 8; VkQueueFamilyProperties qf[8]; vkGetPhysicalDeviceQueueFamilyProperties(pd, &nq, qf);
    uint32_t qi = 0; for (uint32_t i = 0; i < nq; ++i) if (qf[i].queueFlags & VK_QUEUE_COMPUTE_BIT) { qi = i; break; }
    float prio = 1; VkDeviceQueueCreateInfo qci{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; qci.queueFamilyIndex = qi; qci.queueCount = 1; qci.pQueuePriorities = &prio;
+    std::vector<const char*> exts;
+    if (cm.cooperativeMatrix) exts.push_back(VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME);
+    if (f8.shaderFloat8) exts.push_back(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME);
+    if (!cm.cooperativeMatrix) v13.pNext = nullptr; else if (!f8.shaderFloat8) cm.pNext = nullptr;
    VkDeviceCreateInfo dci{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &f2}; dci.queueCreateInfoCount = 1; dci.pQueueCreateInfos = &qci;
+    dci.enabledExtensionCount = static_cast<uint32_t>(exts.size()); dci.ppEnabledExtensionNames = exts.data();
    CK(vkCreateDevice(pd, &dci, nullptr, &dev));
    VkQueue q; vkGetDeviceQueue(dev, qi, 0, &q);
    VkCommandPoolCreateInfo cpci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; cpci.queueFamilyIndex = qi; cpci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
@@ -126,6 +135,12 @@ int main(int argc, char** argv)
            std::string f; VkDeviceSize n; ls >> f >> n;
            Buf b = make_buf(n, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); memset(b.map, 0, n); luts.push_back(b); lutOut.push_back(f);
        }
+        else if (k == "inout")
+        {
+            std::string f, o; ls >> f >> o; auto data = slurp(f);
+            Buf b = make_buf(data.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); memcpy(b.map, data.data(), data.size());
+            luts.push_back(b); lutOut.push_back(o);
+        }
        else if (k == "buf")
        {
            int off; std::string f; int64_t delta; ls >> off >> f >> delta;
@@ -197,6 +212,8 @@ int main(int argc, char** argv)
    VkPipelineLayout pl; CK(vkCreatePipelineLayout(dev, &pli, nullptr, &pl));
    VkPipelineShaderStageRequiredSubgroupSizeCreateInfo rs{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO}; rs.requiredSubgroupSize = subgroup;
    VkComputePipelineCreateInfo cpi{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
+    // KVK_DISABLE_OPTIMIZATION=1: VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT (RADV: ACO without its optimizer/scheduler)
+    if (getenv("KVK_DISABLE_OPTIMIZATION")) cpi.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT;
    cpi.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, &rs, VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT, VK_SHADER_STAGE_COMPUTE_BIT, sm, "main", nullptr};
    cpi.layout = pl;
    VkPipeline pipe;
@@ -287,7 +287,7 @@ class T:

    def stmt(self, s):
        if s.startswith('.reg'):
-            m = re.match(r'\.reg\s+\.(\w+)\s+(.*)$', s)
+            m = re.match(r'\.reg\s*\.(\w+)\s*(.*)$', s)
            ty, names = m.groups()
            if '<' in names:
                return
@@ -714,10 +714,7 @@ class T:
        raise Err('store ' + '.'.join(o))

    # ---------------------------------------------------------------- output
-    def emit(self, tex_filter):
-        nb = len(self.blocks)
-        out = []
-        w = out.append
+    def preamble(self, w, tex_filter):
        w('#version 460')
        # scripts/check_redistributable.py refuses files carrying this line: they are NVIDIA's code.
        w(f'// Generated by vulkan/ptx2glsl.py from NVIDIA PTX ({self.kernel}); do not commit or redistribute.')
@@ -741,7 +738,15 @@ class T:
            w(f'layout(set = 0, binding = {b}, scalar) readonly buffer ExpLut {{ uint16_t e[65536]; }} explut;')
            w('uint ex2_lut(uint u) { return uint(explut.e[u & 0xffffu]) | (uint(explut.e[u >> 16]) << 16); }')
        w(f'shared uint sh[{max(self.shared_bytes // 4, 1)}];')
-        w(RUNTIME)
+    def runtime(self):
+        return RUNTIME
+
+    def emit(self, tex_filter):
+        nb = len(self.blocks)
+        out = []
+        w = out.append
+        self.preamble(w, tex_filter)
+        w(self.runtime())
        w('void main()\n{')
        if getattr(self, 'local_bytes', 0):
            w(f'    uint loc[{(self.local_bytes + 3) // 4}];')
@@ -749,7 +754,7 @@ class T:
            names = sorted(n for n, t in self.regs.items() if t == g)
            for i in range(0, len(names), 24):
                init = {'float': '0.0', 'float16_t': '0.0hf', 'f16vec2': 'f16vec2(0.0hf)', 'uint': '0u', 'uint64_t': '0ul', 'bool': 'false'}[g]
-                w(f'    {"precise " if g in ("float", "float16_t", "f16vec2") and not self.fast else ""}{g} ' + ', '.join(f'{n} = {init}' for n in names[i:i + 24]) + ';')
+                w(f'    {"precise " if (g in ("float", "float16_t", "f16vec2") or getattr(self, "precise_all", False) and g != "bool") and not self.fast else ""}{g} ' + ', '.join(f'{n} = {init}' for n in names[i:i + 24]) + ';')
        # control flow graph over the blocks; END is block nb
        succ = []
        for i, blk in enumerate(self.blocks):
@@ -780,26 +785,45 @@ class T:
        headers = sorted({h for _, h in back})
        cont = {h: END + 1 + i for i, h in enumerate(headers)}
        g = {n: [(cont[m] if (n, m) in back else m) for m in succ[n]] for n in nodes}
-        for h in headers:
-            latches = [n for n, m in back if m == h]
-            exits = {m for n in latches for m in succ[n] if (n, m) not in back}
-            if len(exits) != 1:
-                raise Err('loop without a single latch exit')
-            g[cont[h]] = [exits.pop()]
        g[END] = []
-        allnodes = list(g)
-        # immediate post-dominators (iterative, on the acyclic graph g)
-        rpo, seen = [], set()
-        def dfs(n):
-            stack = [(n, iter(g[n]))]; seen.add(n)
-            while stack:
-                x, it = stack[-1]
-                for m in it:
-                    if m not in seen:
-                        seen.add(m); stack.append((m, iter(g[m]))); break
-                else:
-                    rpo.append(x); stack.pop()
-        dfs(0)
+        for h in headers:
+            g[cont[h]] = [END]                      # provisional, replaced below
+        def postorder():
+            rpo, seen = [], set()
+            def dfs(n):
+                stack = [(n, iter(g[n]))]; seen.add(n)
+                while stack:
+                    x, it = stack[-1]
+                    for m in it:
+                        if m not in seen:
+                            seen.add(m); stack.append((m, iter(g[m]))); break
+                    else:
+                        rpo.append(x); stack.pop()
+            dfs(0)
+            return rpo
+        # A loop may leave through its header as well as through its latches. Its body is the set of nodes that
+        # reach a latch without passing the header; the loop region ends at the nearest node that post-dominates
+        # every exit of the body (for a loop with one latch exit this is that exit, as before).
+        preds = {}
+        for n in nodes:
+            for m in succ[n]:
+                preds.setdefault(m, []).append(n)
+        pdom = {END: {END}}
+        for n in postorder():                       # successors first
+            if n != END:
+                pdom[n] = {n} | set.intersection(*(pdom[m] for m in g[n]))
+        for h in headers:
+            body, work = {h}, [n for n, m in back if m == h]
+            while work:
+                n = work.pop()
+                if n not in body:
+                    body.add(n); work += preds.get(n, [])
+            exits = {m for n in body for m in succ[n] if m not in body}
+            if not exits:
+                raise Err('loop without an exit')
+            common = set.intersection(*(pdom[e] for e in exits))
+            g[cont[h]] = [max(common, key=lambda c: len(pdom[c]))]
+        rpo = postorder()
        topo = rpo[::-1]                      # entry first
        index = {n: i for i, n in enumerate(topo)}
        ipdom = {END: END}
@@ -928,21 +952,10 @@ void sust_b(writeonly uimage2D s, int xb, int y, uint v)
 }
 '''

-if __name__ == '__main__':
-    filt = {}
-    args = sys.argv[1:]
-    while '--tex' in args:
-        i = args.index('--tex'); k, v = args[i + 1].split(':'); filt[int(k)] = v; del args[i:i + 2]
-    lut = '--exp-lut' in args
-    if lut:
-        args.remove('--exp-lut')
-    fast = '--fast' in args
-    if fast:
-        args.remove('--fast')
-    src = open(args[0]).read()
-    t = T(src, args[1])
+def build(make, lut, fast, no_half):
+    t = make()
    t.exp_lut = lut
-    if '--no-half-types' not in args:
+    if not no_half:
        t.collect = True
        t.translate()
        parent = {}
@@ -961,8 +974,24 @@ if __name__ == '__main__':
        half = {x for x in set(t.ctx) | set(parent) if x in t.width and groups[find(x)] == {'h'}}
        sys.stderr.write(f'half-typed registers: {len(half)} of {len(t.width)}\n')
        lut2 = t.exp_lut
-        t = T(src, args[1]); t.exp_lut = lut2; t.half = half
+        t = make(); t.exp_lut = lut2; t.half = half
    t.fast = fast
    t.translate()
+    return t
+
+
+if __name__ == '__main__':
+    filt = {}
+    args = sys.argv[1:]
+    while '--tex' in args:
+        i = args.index('--tex'); k, v = args[i + 1].split(':'); filt[int(k)] = v; del args[i:i + 2]
+    lut = '--exp-lut' in args
+    if lut:
+        args.remove('--exp-lut')
+    fast = '--fast' in args
+    if fast:
+        args.remove('--fast')
+    src = open(args[0]).read()
+    t = build(lambda: T(src, args[1]), lut, fast, '--no-half-types' in args)
    sys.stdout.write(t.emit(filt))
    sys.stderr.write(f'blocks={len(t.blocks)} textures={sorted(t.textures)} surfaces={t.surfaces} shared={t.shared_bytes}\n')