Merge remote-tracking branch 'origin/linux' into linux

# Conflicts:
#	engine/rr/ptx2rr.py

SUPPORTED_GAMES.md

@@ -45,3 +45,14 @@ Game-specific setup:
 - **Dying Light: The Beast:** use DLSS inputs; with FSR or XeSS inputs the screen stays black. The game passes its motion vectors as `R16G16B16A16_TYPELESS`, which needs a d4r build from 2026-09-28 or later (older builds reject every frame and show a black screen).
 
 Choose a model with `Model = E`, `Model = K`, or `Model = M` under `[DLSS]` in the game's `d4r/d4r.ini`. Model compatibility does not imply equal performance; see the [Townfall measurements](docs/performance.md) for measured frame rates.
+
+Vulkan games (native Vulkan frontend, engine path; see docs/vulkan.md):
+
+- **Enshrouded** (Vulkan only): preset M verified in game on RX 9070 (gfx1201), GE-Proton11-7, through the game's own DLSS
+  inputs. Install OptiScaler as `dbghelp.dll` (the exe imports it before it creates its Vulkan device; `dxgi.dll` loads
+  too late) with `WINEDLLOVERRIDES="dbghelp=n,b"`, and set in `OptiScaler.ini`: `[Upscalers] VulkanUpscaler=dlss`,
+  `[Spoofing] Vulkan=true`, `VulkanExtensionSpoofing=false`, `Dxgi=false` (a Vulkan game needs no DXGI spoofing; the
+  verified setup ran without it), and `[InitFlags] AutoExposure=true` (the game sends no exposure; needed for K). The game's motion vectors are `R16G16_SNORM`, which the staging path rejects. Launch options as for
+  D3D12 games (`PROTON_FORCE_NVAPI=1 DXVK_NVAPI_GPU_ARCH=AD100`). The lsfg-vk layer made the game's swap chain creation
+  fail under GE-Proton11-7 even without d4r; disable it (`DISABLE_LSFGVK=1`) if the game exits with
+  `Graphics_InitializationFailed`.

docs/vulkan.md

@@ -126,6 +126,43 @@ and the NGX core signatures used by
 Host event signaling follows the
 [Vulkan event synchronization contract](https://docs.vulkan.org/refpages/latest/refpages/source/vkCmdWaitEvents.html).
 
+## Engine path (no staging)
+
+With the engine on (`[Engine] Enabled`, `D4R_ENGINE=1`, which `auto` sets when `d4r\engine` holds models), the
+Vulkan frontend records d4r's own network for presets K, M and L straight into the caller's command buffer on the
+game's own `VkDevice`: the same `GameUpscaler` / `GameUpscalerM` adapters the D3D12 path uses through vkd3d-proton.
+There is no CPU staging, no CUDA evaluation and no host wait; the result belongs to the frame being recorded.
+`D4R_VULKAN_ENGINE=0` keeps the staging path. Anything the engine does not cover returns to the staging path,
+logged once as `native Vulkan engine (K|M|L): <reason>; feature N uses the staging path`.
+
+- Inputs are sampled through the caller's views in `SHADER_READ_ONLY_OPTIMAL` (`D4R_VULKAN_INPUT_GENERAL=1`
+  for `GENERAL`); depth/stencil views get a depth-only view of their own. The output must be an RGBA16F (or,
+  with an RGB10 model, A2B10G10R10) image the engine writes directly in `GENERAL`, because the resource contract
+  gives it `STORAGE` usage only.
+- Each recording ends with a `vkCmdSetEvent`; a released feature's engine objects are destroyed once every
+  recorded event has fired, or at shutdown after the application's device-idle point. Unsubmitted recordings are
+  never waited on.
+- Device features: the engine's shaders use `VK_KHR_cooperative_matrix` (and `VK_EXT_shader_float8` for M and L).
+  `NVSDK_NGX_VULKAN_GetFeatureDeviceExtensionRequirements` now reports both where the adapter has them, so games
+  that honour NGX's requirements enable the extensions. Their feature bits are still up to the game; RADV compiles
+  and runs the engine without them (bit-identical harness output on a plain 1.1 device), but that is outside the
+  Vulkan specification, and a layer that enables the features is the clean fix.
+- Not covered: Ray Reconstruction, bias-colour and transparency masks, and K's game exposure texture in a
+  non-float format. With AutoExposure off and no exposure texture, K is refused; `[InitFlags] AutoExposure=true` in
+  `OptiScaler.ini` covers games that send no exposure (M and L ignore exposure).
+
+Harness checks (RX 9070, gfx1201, GE-Proton11-7, 640x360 -> 1280x720, frame 8, against the staging path):
+K 99.6 % bit-identical, the rest within one half-float ulp; M 52.7 dB and L 54.3 dB PSNR (FP8 rounding) with equal
+means; `dynamic`, `recreate` and `discard` pass for K and M. `D4R_HARNESS_ENGINE_DEVICE=1` creates a 1.3 device with
+the engine's features; `D4R_HARNESS_ALLOW_BLACK=1` tolerates NGX's black reset frame for K on the staging path.
+
+```sh
+D4R_ENGINE=1 D4R_DLSS_PRESET=13 D4R_HARNESS_ENGINE_DEVICE=1 D4R_ENGINE_MODEL_M_DIR=PORTABLE_D4R_DIR/engine/m \
+  scripts/run_vulkan_dlss_harness_proton.sh PROTON_DIR PORTABLE_D4R_DIR build/engine-m.rgba16f 8
+```
+
+In game: Enshrouded (Vulkan only) runs preset M on this path at 1505x847 -> 2560x1440; see SUPPORTED_GAMES.md.
+
 ## Synthetic harness
 
 The harness creates an AMD Vulkan device without NGX-specific device extensions,

engine/compile_rr2.py

@@ -71,11 +71,14 @@ KERNEL['downp'] = 'rr2_downsample_kernel_static_hdr'
 BLOCK['downp'] = '16,16,1'
 BUFFERS['downp'] = [72, 168, 176]
 # with --native: the translation of the per-pixel code around the native block (engine/rr/gen_rr2_enc0.py)
-HYBRID = {'enc0': [], 'dec0_1': ['--dec0_1']}
+HYBRID = {'enc0': [], 'dec0_1': ['--dec0_1', '--tail-waves', os.environ.get('D4R_RR2_TAIL_WAVES', '2')]}
 # windows per workgroup of the hybrid stages (gen_rr2_enc0.py --group; D4R_RR2_GROUP to experiment)
 GROUP = int(os.environ.get('D4R_RR2_GROUP', '1'))
-# encoder 0's gather as a pass of its own (D4R_RR2_SPLIT=0: one shader, the gather on one wave of the block's four)
+# encoder 0's gather as a pass of its own, which pays with the four-wave block (one shader would run the gather on one
-SPLIT = os.environ.get('D4R_RR2_SPLIT', '1') != '0'
+# wave of the four). With the one-wave block (level 3) the gather runs in the block's own shader and hands its
+# features over in shared memory instead (rr2_w.comp RR2_XF_SHARED): no record in the arena, one dispatch less.
+# D4R_RR2_SPLIT=1 / 0 forces the gather pass / the single shader.
+SPLIT = os.environ.get('D4R_RR2_SPLIT', '0' if RELAXED >= 3 and 'enc0' in ONE_WAVE else '1') != '0'
 # The pooled histories as RGBA32F holding q / 1023, as ZLUDA backs them (16 bytes a pixel). D4R_RR2_POOLED=rgb10_a2
 # makes them real UNORM 10:10:10:2 images (0.1 ms faster at 4K), but the hardware's bilinear filter then rounds
 # differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13)
@@ -165,6 +168,10 @@ def main():
         cmd += [f'--buffer={off}' for off in BUFFERS.get(stage, [])]
         cmd += [f'--snorm16={off}' for off in SNORM16.get(stage, [])]
         cmd += [f'--surface={off}={fmt}' for off, fmt in SURFACES.get(stage, {}).items()]
+        # the hybrid stages' and the output passes' modules go through spv_exact_convert.py, so their conversions need
+        # only the lighter guard (ptx2rr.py --exact-convert)
+        exact = (bool(a.native) and stage in HYBRID) or not network
+        cmd += ['--exact-convert'] if exact else []
         subprocess.run(cmd, check=True, stderr=subprocess.DEVNULL)
         info = json.loads(generated.with_suffix('.json').read_text())
         weights = '-'
@@ -191,7 +198,7 @@ def main():
                 subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / 'enc0g.spv')],
                                check=True, stdout=subprocess.DEVNULL)
                 record = int((work / 'enc0_g.record').read_text())
-            if RELAXED >= 3 and stage in ONE_WAVE and split:
+            if RELAXED >= 3 and stage in ONE_WAVE and (split or GROUP == 1):
                 how = how + ['--one-wave']
             subprocess.run([sys.executable, str(HERE / 'rr' / 'gen_rr2_enc0.py'), str(find_module(a.ptx_dir, kernel)),
                             str(generated), str(work / f'{stage}_m.comp'), *HYBRID[stage], *how],
@@ -204,6 +211,9 @@ def main():
         else:
             subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', str(generated), '-o',
                                      str(out / f'{stage}.spv')], check=True, stdout=subprocess.DEVNULL)
+            if exact:
+                subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / f'{stage}.spv')],
+                               check=True, stdout=subprocess.DEVNULL)
         if hybrid:
             pass
         elif network:

engine/game.cpp

@@ -16,7 +16,8 @@ bool gameColorFormat(VkFormat format) {
     }
 }
 bool gameMotionFormat(VkFormat format) {
-    return format == VK_FORMAT_R16G16_SFLOAT || format == VK_FORMAT_R32G32_SFLOAT ||
+    // R16G16_SNORM (Enshrouded): sampled as floats in [-1, 1]; MV.Scale carries them to pixels as for float vectors.
+    return format == VK_FORMAT_R16G16_SFLOAT || format == VK_FORMAT_R32G32_SFLOAT || format == VK_FORMAT_R16G16_SNORM ||
            format == VK_FORMAT_R16G16B16A16_SFLOAT || format == VK_FORMAT_R32G32B32A32_SFLOAT;
 }
 bool gameDepthFormat(VkFormat format, VkImageAspectFlags aspect) {

engine/replay_rr.cpp

@@ -185,7 +185,9 @@ int main(int argc, char** argv) try {
     HostBuffer staging(c, VkDeviceSize(ow) * oh * 8 * 4);
     // GPU time of every frame: REngine's four stamps (start, after exposure, after the network, after the output)
     VkQueryPoolCreateInfo qp{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO};
-    qp.queryType = VK_QUERY_TYPE_TIMESTAMP; qp.queryCount = 4;
+    qp.queryType = VK_QUERY_TYPE_TIMESTAMP; qp.queryCount = 32;
+    // D4R_ENGINE_RR_STAGE_STAMPS=1: the time of every network stage and output group as well (best frame of the repeats)
+    const bool stageTimes = std::getenv("D4R_ENGINE_RR_STAGE_STAMPS") != nullptr;
     VkQueryPool stamps; CK(vkCreateQueryPool(c.device, &qp, nullptr, &stamps));
     VkPhysicalDeviceProperties props; vkGetPhysicalDeviceProperties(c.physical, &props);
     const int repeat = std::getenv("D4R_RR_TIME_REPEAT") ? std::atoi(std::getenv("D4R_RR_TIME_REPEAT")) : 1;
@@ -233,21 +235,36 @@ int main(int argc, char** argv) try {
         engine.setFrame(images, s);
         if (first) c.submit([&](VkCommandBuffer cb) { model.recordUpload(cb); engine.recordInitialize(cb); first = false; });
         double best = 1e30, network = 0, output = 0, exposure = 0;
+        std::vector<double> stageBest(28, 1e30);
         for (int r = 0; r < repeat; ++r) {
             if (r) engine.setFrame(images, s);
             c.submit([&](VkCommandBuffer cb) {
-                vkCmdResetQueryPool(cb, stamps, 0, 4);
+                vkCmdResetQueryPool(cb, stamps, 0, 32);
                 engine.recordFrame(cb, stamps, 0);
             });
             uint64_t t[4];
             CK(vkGetQueryPoolResults(c.device, stamps, 0, 4, sizeof(t), t, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT));
             const double ms = props.limits.timestampPeriod * 1e-6;
+            if (stageTimes) {   // per stage: the minimum over the repeats (stamps 1, then 4 ... as written)
+                uint64_t all[32][2];
+                vkGetQueryPoolResults(c.device, stamps, 0, 32, sizeof(all), all, 16, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
+                uint64_t prev = all[1][0];
+                for (uint32_t q = 4; q < 32 && all[q][1]; ++q) {
+                    stageBest[q - 4] = std::min(stageBest[q - 4], (all[q][0] - prev) * ms);
+                    prev = all[q][0];
+                }
+            }
             if ((t[3] - t[0]) * ms < best) {
                 best = (t[3] - t[0]) * ms; exposure = (t[1] - t[0]) * ms; network = (t[2] - t[1]) * ms; output = (t[3] - t[2]) * ms;
             }
         }
         std::cout << "frame " << f << ": " << best << " ms (exposure " << exposure << ", network " << network
                   << ", output " << output << ")\n";
+        if (stageTimes) {
+            std::cout << "stages (us):";
+            for (double t : stageBest) if (t < 1e29) std::cout << " " << static_cast<int>(t * 1000 + 0.5);
+            std::cout << "\n";
+        }
         for (const auto& d : engine.debugImages()) {
             const size_t bytes = size_t(d.width) * d.height * texelBytes(d.format);
             c.submit([&](VkCommandBuffer cb) {

engine/rr/README.md

@@ -530,6 +530,65 @@ only; the other stages still use `rr2_m.comp`.
 well 4.98 / 5.59. Decoder 0's second kernel is not converted: it hands r and the MLP sum (8 KiB of binary16) to its
 translated tail through shared memory, which a single wave cannot hold without losing the 4 KiB rate.
 
+Shared memory, later (2026-10-18, RX 9070 XT, RADV, 1280x720 -> 2560x1440, `d4r-rr` with `D4R_RR_TIME_REPEAT=40`
+and `D4R_ENGINE_RR_STAGE_STAMPS=1`, five interleaved runs; every image of six temporal frames byte-identical to the
+previous model, also at 1114x626 -> 1080p, 853x480 -> 1440p and with `D4R_RR_TEST_ROBUST2=1`). ACO sizes a shader's
+registers to the waves its shared memory leaves room for (enc1 needed 97 vector registers before scheduling and got
+240, dec0_1 79 and 192), so a stage that declares less shared memory also gets fewer registers and more waves:
+- **Encoder 0 in one shader again at level 3.** With the one-wave block nothing waits while the gather runs, so
+  `gen_rr2_enc0.py --one-wave` without `--split` runs the gather, the block and the tail in one 32-thread group and
+  hands the features over in shared memory: XF shares its 4 KiB with the block's XT and KVS (`rr2_w.comp`
+  `RR2_XF_SHARED`; every feature tile is in registers before the block first writes either). No gather pass, no
+  record in the arena (76 bytes a render pixel, of which 36 MB lay past the arena at 720p). The gather now runs at the
+  block's 6 waves per SIMD instead of 16 as a pass of its own, which takes back most of the saved traffic:
+  enc0g + enc0 798 -> 772 us. `D4R_RR2_SPLIT=1` builds the gather pass.
+- **enc1's merge input shares its 8 KiB with XT and KVS** (`rr2_m.comp` KIND 1): 17 -> 9 KiB of shared memory, 240 ->
+  144 vector registers, 6 -> 10 waves per SIMD: 286 -> 266 us.
+- **dec0_1 hands r and the MLP sum to its tail in XT and KVS' place** (KIND 5, one window per workgroup): written as
+  binary16 pairs of natural channels after a barrier, read in those pairs by the tail; 14 -> 8 KiB, 192 -> 120 vector
+  registers, 8 -> 12 waves per SIMD: 660 -> 623 us.
+- **Lighter conversion guards in the hybrid stages** (enc0, dec0_1; `ptx2rr.py --exact-convert`): their modules go
+  through `spv_exact_convert.py`, whose NoContraction on every conversion already stops the fold of
+  `float(float16_t(x))`, so a `cvt.rn.f16.f32` keeps only the opaque zero on its f32 bits (against the fused
+  multiply-convert, which `precise` alone does not stop: without it ACO emits `v_fma_mixlo_f16` there). In the ISA the
+  same number of `v_cvt_f16_f32` / `v_cvt_f32_f16` as before and no `v_fma_mix*_f16`; 224 / 153 fewer instructions:
+  enc0 747 -> 729 us, dec0_1 598 -> 592. The output passes (post, down, downp, pooled) now go through
+  `spv_exact_convert.py` too and take the same guard. Their ISA already uses `v_fma_mix*_f16` for plain conversions,
+  so the check there is the whole instruction mix: only `v_and_or_b32` / `v_or` / `v_perm` / shifts go (post -16
+  VALU, down -9, downp -73), every `v_fma_f32` / `v_mul_f32` / `v_add_f32` / conversion / `v_exp` / `v_log` count is
+  unchanged (post moves 5 of its 57 mixes from the low to the high half of their registers). Output 673 -> 665 us.
+- **dec0_1's translated tail on two waves** (`gen_rr2_enc0.py --tail-waves 2`). The tail ran on wave 0 while the
+  block's other three waves had nothing left to do; it is a prologue (y = r + MLP * s2 and the shuffles that give each
+  lane its two pixels) and then the same per-pixel code twice. Each of two waves now runs its own copy of the tail with
+  only one pixel's stores (the first six, the last six), and the compiler drops what a copy's stores do not need:
+  every stored value comes from the same instructions on the same values. 120 -> 96 vector registers, 12 -> 16 waves
+  per SIMD; dec0_1 565 -> 474 us. Four waves (each pixel's stores split in two) is slower, 529 us: the per-pixel work
+  the parts share is then done twice. `D4R_RR2_TAIL_WAVES=1` builds the single-wave tail.
+- **One unsigned comparison per axis in the translation's bounds-checked fetch** (`ptx2rr.py` `fetch0`: a negative
+  coordinate is above every size): 231 instructions fewer in enc0, whose gather fetches 102 times, its scalar spills
+  200 -> 176; enc0 745 -> 738 us.
+- **Hardware saturation for the FP8 conversions of f32 values** (`rr_exact.glsl` `FINITE`). RADV lowers a saturating
+  conversion (`saturatedConvertEXT`) as a NaN-preserving clamp per element (`v_minimummaximum_f32`) and then the
+  conversion, because the hardware's FP16_OVFL saturation turns infinities into NaN instead of +-448 (ACO's comment;
+  confirmed on all 2^32 f32 inputs, RX 9070 XT: FP16_OVFL is identical to the clamp for every finite value and NaN,
+  only +-Inf differ). The values converted here never are infinite: sums of FP8 products (each at most 448^2) and
+  binary16 biases stay far below 3.4e38. Written as "an infinity becomes NaN, then saturate", the conversion is the
+  pattern `nir_opt_algebraic` turns into `f2e4m3fn_satfn` (vkd3d-proton's; Mesa >= 25.2, `has_f2e4m3fn_satfn`):
+  the hardware conversion with FP16_OVFL and no clamp. 168 - 232 clamps fewer per network stage (enc0 960 -> 128);
+  frame 3.92 -> 3.79 ms, every stage faster (dec0_1 607 -> 576 us, enc0 752 -> 727, enc1 262 -> 247). The binary16
+  conversions (the decoders' first kernels' input) keep the clamp: NIR moves the infinity test through the widening
+  and the pattern no longer matches. `RR_SAT_CLAMP=1` builds the old form.
+Frame 4.09 -> 3.84 ms. Tried without gain: the decoders' first kernels' expansion in KVS (16 /
+12 / 8 KiB, but the deep levels have too few windows for the waves to matter and the extra barrier cost 4 - 5 us
+each), all twelve tiles of a 1440p frame in one group (sixteen tile slots: the same), the output passes on 32-wide
+waves (0.67 -> 0.82 ms) and unrolled head / branch loops in the 160-channel stages (+10 - 20 us each). Per stage
+(us): enc0 772, enc1 266, enc2 137, dec1_0 246, dec1_1 208, dec0_0 405, dec0_1 623, levels 3 - 5 (eight stages) 395,
+output 670 (post 490, down 180). Levels 4 and 5 have 15 - 77 windows for 64 compute units.
+Eight waves per window for those levels (each token tile's channel tiles split over two waves, the hidden layer's
+two tiles one each, the halves meeting in shared memory; byte-identical) does not pay: enc4 / dec4_0 / dec4_1 46 / 45 /
+36 -> 49 / 45 / 40 us, enc5 35 -> 31, level 3 64 - 66 -> 78 - 86. A workgroup stays on one compute unit, so the idle
+units stay idle and the extra waves only add the shared work and barriers.
+
 ### Presets D and E against a real NVIDIA card, and level 3 for them (2026-10-14)
 
 NVIDIA's own kernels for presets D and E were replayed on an RTX PRO 4500 Blackwell (drivers 595.91 and 580.178: the

engine/rr/gen_rr2_enc0.py

@@ -15,6 +15,10 @@ out:
 With --group N (2 to 4) a workgroup holds N windows along x: waves 0 ... N - 1 run the translated code of one window
 each and the block then runs once per window; the stage's dispatch is 1 / N as wide ("xN" in rr.txt). Measured on
 an RX 9070 XT (2026-10-13) that is slower than one window per workgroup (network +0.4 ms at 1280x720 with N = 4).
+With --one-wave (D4R_RR_RELAXED=3) the block is engine/rr/rr2_w.comp in NVIDIA's own 32-thread group: the gather, the
+block and the tail all run on the one wave, and XF shares its 4 KiB with the block's XT and KVS (rr2_w.comp
+RR2_XF_SHARED), so nothing goes through memory. (The gather was made a pass of its own because on the four-wave block
+it ran on one wave while three waited; on one wave there is nothing to wait for.)
 With --dec0_1 the same for dec0's second kernel (rr2_m.comp KIND 5): nothing before the block (it reads its tokens
 from the arena); after it the last product's D fragments (the MLP sum) and the registers holding r are loaded from
 MSH / RSH, and the translated tail forms y = r + MLP * s2 and computes the kernel's outputs from it.
@@ -177,6 +181,8 @@ def main():
     m = next(i for i, l in enumerate(g) if l.startswith('void main()'))
     # --one-wave (D4R_RR_RELAXED=3): the block as rr2_w.comp, in NVIDIA's own 32-thread group, the tail on the same wave
     one_wave = '--one-wave' in sys.argv
+    if one_wave and (dec or G != 1):
+        raise SystemExit('--one-wave is for encoder 0 with one window per workgroup')
     head = '\n'.join(g[:m])
     if not one_wave:
         head = head.replace('layout(local_size_x = 32, local_size_y = 1, local_size_z = 1) in;',
@@ -289,23 +295,43 @@ def main():
             '    // the gather\'s index arithmetic the tail reads\n' + '\n'.join(recomputed) + '\n' +
             '\n'.join(load + keep) + '\n' + '\n'.join(tail) + '\n    }\n}\n')
         return
-    wx = lambda t: t.replace('gl_WorkGroupID.x', f'({G}u * gl_WorkGroupID.x + gl_SubgroupID)')
+    # --tail-waves N (dec0_1, one window per workgroup): the translated tail on N of the block's waves instead of one.
-    text = (head + f'\n#extension GL_GOOGLE_include_directive : require\n#define RR2_EMBED 1\n#define RR2_GROUP {G}\n' + block + '\n' +
+    # Each wave runs its own copy of the tail with only the N-th part of its stores (in order: the first pixel's,
-            ('uint HDV(uint row, uint col) { return uint(float16BitsToUint16(MSH[gl_SubgroupID * 2048u + row * 32u + col])); }\n'
+    # then the second's); the compiler drops what a copy's stores do not need. Every stored value comes from the
-             'uint RSV(uint row, uint col) { return uint(float16BitsToUint16(RSH[gl_SubgroupID * 2048u + row * 32u + col])); }\n\n' if dec else
+    # same instructions on the same values as on one wave, so the outputs are the same.
+    TW = int(sys.argv[sys.argv.index('--tail-waves') + 1]) if '--tail-waves' in sys.argv else 1
+    if TW > 1 and not (dec and G == 1):
+        raise SystemExit('--tail-waves is for dec0_1 with one window per workgroup')
+    wx = (lambda t: t) if one_wave or TW > 1 else (lambda t: t.replace('gl_WorkGroupID.x', f'({G}u * gl_WorkGroupID.x + gl_SubgroupID)'))
+    STORE = re.compile(r'^\s*(?!if\b|for\b|while\b|switch\b|return\b)[A-Za-z_]\w*\(')
+    stores = [i for i, l in enumerate(tail) if STORE.match(l)]
+    if TW > 1 and len(stores) % TW:
+        raise SystemExit(f'--tail-waves {TW}: {len(stores)} stores do not split evenly')
+    def tail_part(w):
+        mine = set(stores[w * len(stores) // TW:(w + 1) * len(stores) // TW])
+        return [l for i, l in enumerate(tail) if i not in stores or i in mine]
+    tails = ''.join(f'    if (rr2_valid && gl_SubgroupID == {w}u) {{\n' + '\n'.join(load + keep) + '\n' + '\n'.join(tail_part(w)) + '\n    }\n'
+                    for w in range(TW)) if TW > 1 else None
+    valid = (f'gl_SubgroupID < {TW}u && gl_WorkGroupID.x < rr2_gx' if TW > 1 else
+             f'gl_SubgroupID < {G}u && {G}u * gl_WorkGroupID.x + gl_SubgroupID < rr2_gx')
+    text = (head + f'\n#extension GL_GOOGLE_include_directive : require\n#define RR2_EMBED 1\n#define RR2_GROUP {G}\n' +
+            ('#define RR2_XF_SHARED 1\n' if one_wave else '') + block + '\n' +
+            ('uint HDV(uint row, uint col) { return uint(float16BitsToUint16(rr2_msh(row, col))); }\n'
+             'uint RSV(uint row, uint col) { return uint(float16BitsToUint16(rr2_rsh(row, col))); }\n\n' if dec else
              'uint HDV(uint row, uint col) { return col < 4u ? uint(float16BitsToUint16(HD[gl_SubgroupID * 256u + row * 4u + col])) : 0u; }\n\n') +
             'void main()\n{\n' + '\n'.join(body[:nd]) +
             '\n    // this wave\'s window, when the level has it\n'
             '    const uint rr2_gx = (param.w[0] + (param.w[20] & 0xffffu) + 7u) / 8u;\n'
-            f'    const bool rr2_valid = gl_SubgroupID < {G}u && {G}u * gl_WorkGroupID.x + gl_SubgroupID < rr2_gx;\n'
+            f'    const bool rr2_valid = {valid};\n'
             '    if (rr2_valid) {   // NVIDIA\'s warp: the translation before the first product\n' + wx('\n'.join(pre)) + '\n    }\n' +
             ('' if dec else '\n'.join(store)) +
             # (the block runs unconditionally, also for a window past the level's last one: it writes nothing there, and
             # inside a branch the driver compiles it much slower)
             ('\n    barrier();\n    rr2_block();\n    barrier();\n' if G == 1 else
              f'\n    barrier();\n    for (uint w_ = 0u; w_ < {G}u; ++w_) {{\n        rr2_win = w_;\n'
-             '        rr2_block();\n        barrier();\n    }\n') + '    if (rr2_valid) {\n' +
+             '        rr2_block();\n        barrier();\n    }\n') +
-            '\n'.join(load + keep) + '\n' + wx('\n'.join(tail)) + '\n    }\n}\n')
+            (tails + '}\n' if tails else
+             '    if (rr2_valid) {\n' + '\n'.join(load + keep) + '\n' + wx('\n'.join(tail)) + '\n    }\n}\n'))
     Path(out).write_text(text)
     print(f'{out}: {len(pre)} gather lines, {len(cut)} cut, {len(tail)} tail lines, {len(seen)} A fragment sets')
 

engine/rr/ptx2rr.py

@@ -59,6 +59,9 @@ def mem_shape(o):
     return size, count
 
 
+# --exact-convert (the module is passed through engine/rr/spv_exact_convert.py, which keeps every conversion): a
+# cvt.rn.f16.f32 only needs the opaque zero on its f32 bits, against the fused multiply-convert (see cvt)
+CVT_GUARD = '--exact-convert' not in sys.argv
 RR_RUNTIME = r'''
 float16_t H(uint u) { return uint16BitsToFloat16(uint16_t(u)); }
 uint HU(float16_t h) { return uint(float16BitsToUint16(h)); }
@@ -75,7 +78,8 @@ float lg2_ftz(float x) { return log2(abs(x) < 1.17549435e-38 ? 0.0 : x); }
 // PTX .relu: negative results become +0, NaN stays NaN
 f16vec2 relu2(f16vec2 v) { return f16vec2(v.x < 0.0hf ? 0.0hf : v.x, v.y < 0.0hf ? 0.0hf : v.y); }
 float16_t relu1(float16_t v) { return v < 0.0hf ? 0.0hf : v; }
-vec4 fetch0(sampler2D t, ivec2 c) { const ivec2 n = textureSize(t, 0); return (c.x < 0 || c.y < 0 || c.x >= n.x || c.y >= n.y) ? vec4(0.0) : texelFetch(t, c, 0); }
+// (one unsigned comparison per axis: a negative coordinate is above every size)
+vec4 fetch0(sampler2D t, ivec2 c) { const ivec2 n = textureSize(t, 0); return any(greaterThanEqual(uvec2(c), uvec2(n))) ? vec4(0.0) : texelFetch(t, c, 0); }
 // x + copysign(0.5, x) rounded toward zero and then truncated: round half away from zero, exactly
 float round_half_away(float x) { float t = trunc(x); return abs(x - t) >= 0.5 ? t + sign(x) : t; }
 uint PARAM32(uint o) { return param.w[o >> 2]; }
@@ -753,7 +757,7 @@ class RRT(base.T):
         for i, d in enumerate(dst):
             # a binary16 result is the texel rounded to binary16: through H(HU(...) | SUST_Z) so that the driver cannot
             # fold the rounding away when the value is widened again (RR_RUNTIME's note on sust_b)
-            self.A(d, f'H(HU(float16_t(t_[{i}])) | SUST_Z)' if f16 else f't_[{i}]', 'f16' if f16 else 'float')
+            self.A(d, (f'H(HU(float16_t(t_[{i}])) | SUST_Z)' if CVT_GUARD else f'float16_t(t_[{i}])') if f16 else f't_[{i}]', 'f16' if f16 else 'float')
         self.E('}')
 
     def sust(self, op, a):
@@ -814,6 +818,11 @@ class RRT(base.T):
             # conversion into v_fma_mix, one rounding of the exact value, where the kernel rounds to f32 first: an f32
             # result that is exactly a binary16 midpoint then rounds the other way (the output stages' rare one-step
             # differences, e.g. log2(x) * c1 * c2; found 2026-10-13): the OR on the f32 bits keeps the f32 value.
+            # With --exact-convert the conversion itself is decorated NoContraction (spv_exact_convert.py), which keeps the
+            # first: only the f32 OR is needed (checked in the ISA: the same v_cvt_f16_f32 / v_cvt_f32_f16 counts, no
+            # v_fma_mix*_f16; without the f32 OR, ACO fuses some products into v_fma_mixlo_f16 despite `precise`)
+            if not CVT_GUARD:
+                return self.A(a[0], f'float16_t(uintBitsToFloat(floatBitsToUint({self.val(a[1], "f32")}) | SUST_Z))', 'f16')
             return self.A(a[0], f'H(HU(float16_t(uintBitsToFloat(floatBitsToUint({self.val(a[1], "f32")}) | SUST_Z))) | SUST_Z)', 'f16')
         return super().cvt(op, o, a)
 
@@ -910,6 +919,8 @@ class RRT(base.T):
             return self.A(a[0], f'tanh_ocml({self.val(a[1], "f32")})', 'float')
         if op == 'ex2.approx.f16':
             # v_exp_f32 of the widened value, rounded to binary16; the ORs keep that rounding (see cvt)
+            if not CVT_GUARD:
+                return self.A(a[0], f'float16_t(uintBitsToFloat(floatBitsToUint(exp2(float({self.val(a[1], "f16")}))) | SUST_Z))', 'f16')
             return self.A(a[0], f'H(HU(float16_t(uintBitsToFloat(floatBitsToUint(exp2(float({self.val(a[1], "f16")}))) | SUST_Z))) | SUST_Z)', 'f16')
         if op == 'neg.f16':
             return self.A(a[0], f'-{self.val(a[1], "f16")}', 'f16')
@@ -1048,6 +1059,8 @@ def main():
                          'execution) for workgroup --probe-block, into a debug buffer bound after the surfaces')
     ap.add_argument('--probe-block', default='0,0')
     ap.add_argument('--oob-report', action='store_true', help='debugging: count out-of-range buffer accesses')
+    ap.add_argument('--exact-convert', action='store_true',
+                    help='the module goes through spv_exact_convert.py: lighter guards on binary16 conversions (CVT_GUARD)')
     ns = ap.parse_args()
     block = tuple(int(x) for x in ns.block.split(',')) if ns.block else default_block(ns.kernel)
     formats = {int(k): v for k, v in (x.split('=') for x in ns.surface)}

engine/rr/rr2_m.comp

@@ -90,7 +90,7 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena;
 #define E4V(w, b) float(uintBitsToFloate4m3EXT(uint8_t(((w) >> (8u * (b))) & 0xffu)))
 #define RESID(d, a, s, c) const float d = fma(float(a), float(s), float(c));
 #define Q8R(x) q8s(x)
-#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2(float((h)[i_]), float((h)[i_ + 1])); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 #else
 #define RVT float16_t
 #define E4V(w, b) e4((w), (b))
@@ -113,10 +113,33 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena;
 const uint HEAD = W_WQK1 - W_WQK0, BRANCH = W_WD1 - W_WD0;
 #define BRANCHB BRANCH
 
+#if KIND == 1
+// enc1's merge input is in registers (gb) before XT or KVS is first written, so the three share 8 KiB instead of 16
+// (above about 4 KiB per wave this GPU runs the workgroup slower, see KIND 2 below): XT words 0 - 16C-1, KVS after it
+#if 16 * C + 2 * 512 > 64 * 32
+#error rr2_m.comp: XT and KVS no longer fit in GT
+#endif
+shared uint GT[64 * 32];         // the merge input: [token slot][4 sub-positions x 32 FP8 channels]; then XT and KVS
+#define XT GT
+#define KVS GT
+#define KVA(o) (16u * C + (o))
+#elif KIND == 5 && RR2_GROUP == 1
+// dec0's second kernel writes r and the MLP sum for its tail only after the block is done with XT and KVS, so the
+// three share 8 KiB instead of 14: XT words 0 - 16C-1, KVS after it; then r (RSW) at word 0 and the MLP sum (MSW) at
+// 1024, both as binary16 pairs of natural channels 2k, 2k + 1 per token (the tail reads them in those pairs)
+#if 16 * C + 2 * 512 > 2 * 1024
+#error rr2_m.comp: XT and KVS no longer fit under RSW and MSW
+#endif
+shared uint SM[2 * 1024];
+#define XT SM
+#define KVS SM
+#define KVA(o) (16u * C + (o))
+#define RSW(w) SM[w]
+#define MSW(w) SM[1024u + (w)]
+#else
 shared uint XT[16 * C];          // the window's tokens: [token slot][C FP8 channels in slot order]; later the merge rows
 shared uint KVS[2 * 512];        // the current head's QK ([token slot][32]), then V^T ([32][token slot])
-#if KIND == 1
+#define KVA(o) (o)
-shared uint GT[64 * 32];         // the merge input: [token slot][4 sub-positions x 32 FP8 channels]
 #endif
 #if KIND == 2
 // Shared memory is what limits how many waves this GPU keeps running: above about 4 KiB per wave (16 KiB for this
@@ -146,7 +169,15 @@ const uint EBRANCH = W_EWD1 - W_EWD0;
 #define EBRANCHB EBRANCH
 #endif
 #if KIND == 5
+#if RR2_GROUP == 1
+// r and the MLP sum of token row, natural channel col, for the translated tail (gen_rr2_enc0.py)
+float16_t rr2_rsh(uint row, uint col) { return unpackFloat2x16(RSW(row * 16u + (col >> 1u)))[col & 1u]; }
+float16_t rr2_msh(uint row, uint col) { return unpackFloat2x16(MSW(row * 16u + (col >> 1u)))[col & 1u]; }
+#else
 shared float16_t RSH[RR2_GROUP * 64 * 32], MSH[RR2_GROUP * 64 * 32];   // per window, r and the MLP sum per token (raster order) and natural channel
+float16_t rr2_rsh(uint row, uint col) { return RSH[gl_SubgroupID * 2048u + row * 32u + col]; }
+float16_t rr2_msh(uint row, uint col) { return MSH[gl_SubgroupID * 2048u + row * 32u + col]; }
+#endif
 #endif
 #if GUIDE || KIND == 3
 shared float16_t OUTG[64 * 4];   // the guide output per window token (raster order)
@@ -242,6 +273,7 @@ void main()
     {
         B8 gb[8];
         [[unroll]] for (uint kt = 0u; kt < 8u; ++kt) coopMatLoad(gb[kt], GT, wv * 512u + 4u * kt, 32u, CMJ);
+        barrier();   // every wave has its part of GT in registers: XT takes its place
         [[unroll]] for (uint m = 0u; m < MT; ++m) {
             RAC a;
             [[unroll]] for (int i = 0; i < 8; ++i) a[i] = BV(B1, 16u * m + 8u * hl + uint(i));
@@ -342,20 +374,20 @@ void main()
             A8 ka, va;
             Q8(ka, a)
             Q8(va, v)
-            coopMatStore(ka, KVS, n * 128u + 4u * c, 8u, RM);
+            coopMatStore(ka, KVS, KVA(n * 128u + 4u * c), 8u, RM);
-            coopMatStore(va, KVS, 512u + c * 256u + 4u * n, 16u, RM);
+            coopMatStore(va, KVS, KVA(512u + c * 256u + 4u * n), 16u, RM);
         }
         barrier();
         // scores^T (keys x this wave's queries) = QK . QK^T + bias, the cubic softmax down each column
         B8 Qb[2];
-        [[unroll]] for (uint c = 0u; c < 2u; ++c) coopMatLoad(Qb[c], KVS, wv * 128u + 4u * c, 8u, CMJ);
+        [[unroll]] for (uint c = 0u; c < 2u; ++c) coopMatLoad(Qb[c], KVS, KVA(wv * 128u + 4u * c), 8u, CMJ);
         B8 Pb[4];
         {
             float16_t e[4][8];
             [[unroll]] for (uint kt = 0u; kt < 4u; ++kt) {
                 A8 k0, k1;
-                coopMatLoad(k0, KVS, kt * 128u, 8u, RM);
+                coopMatLoad(k0, KVS, KVA(kt * 128u), 8u, RM);
-                coopMatLoad(k1, KVS, kt * 128u + 4u, 8u, RM);
+                coopMatLoad(k1, KVS, KVA(kt * 128u + 4u), 8u, RM);
                 HAC c;
                 coopMatLoad(c, w16.w, (W_BIAS0 + HEAD * hd) / 2u + kt * 1024u + 16u * wv, 64u, RM);
                 K32H(c, k0, Qb[0], k1, Qb[1])
@@ -392,8 +424,8 @@ void main()
             RAC o = RAC(RV(float16_t(0.0)));
             [[unroll]] for (uint ks = 0u; ks < 2u; ++ks) {
                 A8 v0, v1;
-                coopMatLoad(v0, KVS, 512u + c * 256u + 8u * ks, 16u, RM);
+                coopMatLoad(v0, KVS, KVA(512u + c * 256u + 8u * ks), 16u, RM);
-                coopMatLoad(v1, KVS, 512u + c * 256u + 8u * ks + 4u, 16u, RM);
+                coopMatLoad(v1, KVS, KVA(512u + c * 256u + 8u * ks + 4u), 16u, RM);
                 K32(o, v0, Pb[2u * ks], v1, Pb[2u * ks + 1u])
             }
             Q8(Ob[c], o)
@@ -449,12 +481,23 @@ void main()
 
 #if KIND == 5
     // dec0's second kernel: the translated tail forms y = r + MLP * s2 itself and computes the outputs from it
+#if RR2_GROUP == 1
+    barrier();   // every wave is done with XT and KVS
+    // (channel slots j and j + 1, j even, hold the natural channels inter(j) and inter(j) + 1)
+    [[unroll]] for (uint m = 0u; m < MT; ++m)
+        [[unroll]] for (int i = 0; i < 8; i += 2) {
+            const uint w = (t * 32u + inter(16u * m + 8u * hl + uint(i))) >> 1u;
+            RSW(w) = packFloat2x16(f16vec2(float16_t(rv[m][i]), float16_t(rv[m][i + 1])));
+            MSW(w) = packFloat2x16(f16vec2(float16_t(ML[m][i]), float16_t(ML[m][i + 1])));
+        }
+#else
     [[unroll]] for (uint m = 0u; m < MT; ++m)
         [[unroll]] for (int i = 0; i < 8; ++i) {
             const uint at = RR2_WIN * 2048u + t * 32u + inter(16u * m + 8u * hl + uint(i));
             RSH[at] = float16_t(rv[m][i]);
             MSH[at] = float16_t(ML[m][i]);
         }
+#endif
 #else
     // ---------------------------------------------------------------- y = r + MLP * s2: the stage's tokens
     B8 Tb[MT];

engine/rr/rr2_w.comp

@@ -6,8 +6,10 @@
 // Shared memory stays at 4 KiB, above which this GPU runs fewer waves (engine/rr/README.md). The arithmetic is
 // rr2_m.comp's with -DRR_RELAXED=2 -DRR_NATIVE and the arena comes out byte-identical to that build
 // (build/rr/f/nv/invest-20261013/REPORT.md: 453 us against 581 us at 1280x720). compile_rr2.py uses it for dec0_0.
-// KIND 4 is encoder 0's block the same way (the embedding of the gather pass's record, the tokens, the guide head to
+// KIND 4 is encoder 0's block the same way (the embedding of the gather's features, the tokens, the guide head to
-// HD), included with RR2_EMBED and RR2_XF_ARENA by the shader engine/rr/gen_rr2_enc0.py --one-wave generates.
+// HD), included with RR2_EMBED by the shader engine/rr/gen_rr2_enc0.py --one-wave generates: with RR2_XF_SHARED the
+// gather runs in the same shader and leaves the features in XF (the default), with RR2_XF_ARENA they come from the
+// gather pass's record in the arena (--split block).
 #extension GL_KHR_cooperative_matrix : require
 #extension GL_KHR_memory_scope_semantics : require
 #extension GL_EXT_shader_explicit_arithmetic_types : require
@@ -26,8 +28,8 @@ layout(local_size_x = 32) in;
 #if !(KIND == 2 || KIND == 4) || CM != 0 || GUIDE != 0 || C != 32
 #error rr2_w.comp: a 32-channel decoder's first kernel, or encoder 0's block
 #endif
-#if KIND == 4 && !(defined(RR2_EMBED) && defined(RR2_XF_ARENA))
+#if KIND == 4 && !(defined(RR2_EMBED) && (defined(RR2_XF_ARENA) || defined(RR2_XF_SHARED)))
-#error rr2_w.comp: encoder 0's block reads the gather pass's record (gen_rr2_enc0.py --split block --one-wave)
+#error rr2_w.comp: encoder 0's block reads the gather's features (gen_rr2_enc0.py --one-wave)
 #endif
 #define MT (C / 16)
 #ifdef RR2_EMBED
@@ -53,11 +55,21 @@ layout(set = 0, binding = 2, std430) buffer Arena { uint w[]; } arena;
 #define LDB(dst, base, n, kt, nt) coopMatLoad((dst), w8.w, (base) + 256u * ((kt) * ((n) / 16u) + (nt)), 16u, RM)
 #define K32(acc, a0, b0, a1, b1) { (acc) = coopMatMulAdd((a1), (b1), coopMatMulAdd((a0), (b0), (acc))); }
 #define Q8(dst, src) Q8PAIRS(dst, src)
-#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define ACT8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 const uint HEAD = W_WQK1 - W_WQK0, BRANCH = W_WD1 - W_WD0;
 
+#if KIND == 4 && defined(RR2_XF_SHARED)
+// the gather's features (XF, [token slot][64 FP8 feature slots], 4 KiB) are all in registers before the block first
+// writes XT or KVS, so the three share 4 KiB: KVS words 0 - 511, XT words 512 - 1023
+shared uint XF[64 * 16];
+#define KVS XF
+#define XT XF
+#define XTA(o) (512u + (o))
+#else
 shared uint XT[16 * C];
 shared uint KVS[512];          // the expansion (input phase), then per head V^T, then QK
+#define XTA(o) (o)
+#endif
 // Shared memory is what limits how many waves this GPU keeps running: above about 4 KiB per wave the rate falls in
 // proportion (build/rr/f/nv/cost/occ2.py). So the input phase borrows KVS, which the attention only needs later
 // (LOW at word 0, EX at word 512), and the block's unquantised input stays in registers (xr), not in memory.
@@ -68,7 +80,14 @@ shared uint KVS[512];          // the expansion (input phase), then per head V^T
 #endif
 #define EX_AT 0u
 #if KIND == 4
+#ifdef RR2_XF_ARENA
 uint rr2_xf;                     // the window's record from the gather pass: the index of its first arena word
+#define XF_SRC arena.w
+#define XF_AT rr2_xf
+#else
+#define XF_SRC XF
+#define XF_AT 0u
+#endif
 shared float16_t HD[64 * 4];     // the guide head per token (raster order), for the translated tail
 const uint EBRANCH = W_EWD1 - W_EWD0;
 #endif
@@ -103,10 +122,13 @@ void main()
         B8 fb[4][4];
         ACC X[4][MT];
         [[unroll]] for (uint wv = 0u; wv < 4u; ++wv) {
-            [[unroll]] for (uint kt = 0u; kt < 4u; ++kt) coopMatLoad(fb[wv][kt], arena.w, rr2_xf + wv * 256u + 4u * kt, 16u, CMJ);
+            [[unroll]] for (uint kt = 0u; kt < 4u; ++kt) coopMatLoad(fb[wv][kt], XF_SRC, XF_AT + wv * 256u + 4u * kt, 16u, CMJ);
             [[unroll]] for (uint m = 0u; m < MT; ++m)
                 [[unroll]] for (int i = 0; i < 8; ++i) X[wv][m][i] = float(w16.w[W_EBU / 2u + 16u * m + 8u * hl + uint(i)]);
         }
+#ifdef RR2_XF_SHARED
+        barrier();   // XF is read: XT and KVS take its place
+#endif
         [[unroll]] for (uint br = 0u; br < 2u; ++br) {
             A8 ewd[2][4], ewu0[MT], ewu1[MT];
             [[unroll]] for (uint c = 0u; c < 2u; ++c)
@@ -128,7 +150,7 @@ void main()
         [[unroll]] for (uint wv = 0u; wv < 4u; ++wv)
             [[unroll]] for (uint m = 0u; m < MT; ++m) {
                 Q8(qx[wv][m], X[wv][m])
-                coopMatStore(qx[wv][m], XT, wv * 4u * C + 4u * m, C / 4u, CMJ);
+                coopMatStore(qx[wv][m], XT, XTA(wv * 4u * C + 4u * m), C / 4u, CMJ);
             }
     }
 #else
@@ -198,8 +220,8 @@ void main()
                 ACC v = ACC(0.0);
                 [[unroll]] for (uint kt = 0u; kt < MT; kt += 2u) {
                     A8 xa0, xa1;
-                    coopMatLoad(xa0, XT, n * 4u * C + 4u * kt, C / 4u, RM);
+                    coopMatLoad(xa0, XT, XTA(n * 4u * C + 4u * kt), C / 4u, RM);
-                    coopMatLoad(xa1, XT, n * 4u * C + 4u * kt + 4u, C / 4u, RM);
+                    coopMatLoad(xa1, XT, XTA(n * 4u * C + 4u * kt + 4u), C / 4u, RM);
                     K32(v, xa0, wvt[kt], xa1, wvt[kt + 1u])
                 }
                 A8 va;
@@ -219,8 +241,8 @@ void main()
                 ACC a = ACC(0.0);
                 [[unroll]] for (uint kt = 0u; kt < MT; kt += 2u) {
                     B8 xb0, xb1;
-                    coopMatLoad(xb0, XT, n * 4u * C + 4u * kt, C / 4u, CMJ);
+                    coopMatLoad(xb0, XT, XTA(n * 4u * C + 4u * kt), C / 4u, CMJ);
-                    coopMatLoad(xb1, XT, n * 4u * C + 4u * kt + 4u, C / 4u, CMJ);
+                    coopMatLoad(xb1, XT, XTA(n * 4u * C + 4u * kt + 4u), C / 4u, CMJ);
                     K32(a, wk[kt], xb0, wk[kt + 1u], xb1)
                 }
                 A8 ka;
@@ -348,7 +370,7 @@ void main()
         [[unroll]] for (uint m = 0u; m < MT; ++m) {
             floate4m3_t Tb[8];
             [[unroll]] for (int i = 0; i < 8; i += 2) {
-                fe4m3vec2 q; saturatedConvertEXT(q, vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1])));
+                fe4m3vec2 q; saturatedConvertEXT(q, FINITE2(vec2(fma(ML[wv][m][i], f2, rv[wv][m][i]), fma(ML[wv][m][i+1], f2, rv[wv][m][i+1]))));
                 Tb[i] = q.x; Tb[i+1] = q.y;
             }
 #if KIND == 4

engine/rr/rr_exact.glsl

@@ -31,6 +31,19 @@ uint inter(uint j) { return (j & ~31u) | (((j & 31u) >> 2u) << 1u) | (j & 1u) |
 uint uninter(uint c) { return (c & ~31u) | (((c & 15u) >> 1u) << 2u) | (c & 1u) | (((c >> 4u) & 1u) << 1u); }
 int mirror(int v, int n) { v = abs(v); return min(v, 2 * n - 2 - v); }
 
+// FINITE(v): v, but an infinity (which these f32 values never are: sums of FP8 products and binary16 biases stay far
+// below 3.4e38) becomes NaN. Written so, a saturating FP8 conversion is RADV's f2e4m3fn_satfn (nir_opt_algebraic's
+// vkd3d-proton pattern): the hardware conversion with FP16_OVFL, which saturates every finite value the same way,
+// instead of a NaN-preserving clamp per element and then the conversion. RR_SAT_CLAMP=1 keeps the clamp, and so does
+// RR_MIX, whose binary16 accumulators can overflow.
+#if defined(RR_SAT_CLAMP) || defined(RR_MIX)
+#define FINITE(v) (v)
+#define FINITE2(v) (v)
+#else
+#define FINITE(v) (isinf(v) ? uintBitsToFloat(0x7fc00000u) : (v))
+#define FINITE2(v) vec2(FINITE((v).x), FINITE((v).y))
+#endif
+
 #ifdef RR_GFX11
 #include "rr_gfx11.glsl"
 #define F8 uint
@@ -52,7 +65,7 @@ floate4m3_t q8(float16_t h)
 }
 // the same from f32 / f16 values in one saturating hardware conversion (satfinite; for an f32 value that is a binary16
 // number the result equals the f16 conversion's)
-floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
+floate4m3_t q8s(float x) { floate4m3_t r; saturatedConvertEXT(r, FINITE(x)); return r; }
 floate4m3_t q8h(float16_t x) { floate4m3_t r; saturatedConvertEXT(r, x); return r; }
 
 uint code(floate4m3_t v) { return uint(floate4m3BitsToUintEXT(v)); }
@@ -124,13 +137,13 @@ float16_t fma16q(float16_t a, float16_t b, float16_t c, inout bool mid)
 }
 #ifndef RR_GFX11
 // The saturating FP8 conversion of a whole accumulator, two elements per conversion instruction
-#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, vec2((src)[i_], (src)[i_ + 1])); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define Q8PAIRS(dst, src) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(vec2((src)[i_], (src)[i_ + 1]))); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 // q8(act(h)) from the packed table (enc1_pack.act_table; the weight buffer's W8U view at W_ACT)
 #define ACTQ(h) uintBitsToFloate4m3EXT(w8u.w[W_ACT + bh(h)])
 // the activation of a whole accumulator into FP8. RR_NATIVE (D4R_RR_RELAXED=3): in f32, two elements per
 // instruction, straight into FP8 - no binary16 in between and no table (not the reference's roundings)
 #ifdef RR_NATIVE
-#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, x_ * g_); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
+#define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; i_ += 2) { const vec2 x_ = vec2((h)[i_], (h)[i_ + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.0810546875), vec2(0.412109375)), vec2(0.5)); fe4m3vec2 t_; saturatedConvertEXT(t_, FINITE2(x_ * g_)); (dst)[i_] = t_.x; (dst)[i_ + 1] = t_.y; }
 #else
 #define ACTQ8(dst, h) [[unroll]] for (int i_ = 0; i_ < 8; ++i_) (dst)[i_] = ACTQ(float16_t((h)[i_]));
 #endif

engine/runtime_rr2.inc

@@ -638,6 +638,11 @@ struct R2Engine {
         const auto& b = *bound;
         const uint32_t f = frame, cur = f & 1u;
         auto stamp = [&](uint32_t n) { if (stamps) vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, stamps, first + n); };
+        // timing only: D4R_ENGINE_RR_STAGE_STAMPS=1 also stamps after every network stage and output group, from query
+        // first + 4 on (the caller's pool needs 32 queries)
+        static const bool stageStamps = std::getenv("D4R_ENGINE_RR_STAGE_STAMPS") != nullptr;
+        uint32_t extra = 4;
+        auto stageStamp = [&] { if (stageStamps) stamp(extra++); };
         auto step = [&] {
             memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                           VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
@@ -685,6 +690,7 @@ struct R2Engine {
             }
             dispatch(cb, model.stages[stage].pipeline, b.network[stage][cur], (gx + model.stages[stage].group - 1) / model.stages[stage].group, (lh(l) + sy + 7) / 8);
             step();
+            stageStamp();
         }
         stamp(2);
         // timing only: D4R_ENGINE_RR_SKIP=post,down,pooled leaves those passes out (the picture is wrong then)
@@ -706,6 +712,7 @@ struct R2Engine {
                 if (serial) step();
             }
             if (g < groups) step();
+            stageStamp();
         }
         // (the pooling pass reads what the post passes wrote: the last group's down passes do not touch it)
         if (!noPooled) dispatch(cb, model.stages[R2Pooled].pipeline, b.pooled, (qw + 15) / 16, (qh + 15) / 16);

tools/d4r_nvngx_shim.cpp

@@ -3388,6 +3388,8 @@ struct VulkanInterop
     // VK_EXT_shader_float8 (vkd3d-proton enables it when the GPU has it): M's and L's engine layers are FP8
     // cooperative-matrix shaders, which RADV's compiler crashes on without it (RDNA3)
     bool float8 = false;
+    // The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects.
+    bool native = false;
     PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
     PFN_vkCreateBuffer createBuffer = nullptr;
     PFN_vkDestroyBuffer destroyBuffer = nullptr;
@@ -9335,7 +9337,7 @@ static void init_engine_backend()
         WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + L"\\engine\\k").c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
         directory = folder;
     }
-    if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
+    if ((!g_vk.native && (g_vk.interop == nullptr || g_vk.lifetime == nullptr)) || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
         return;
     // A Unix path names the same file through Wine's Z: drive.
     const std::string path = directory[0] == '/' ? "Z:" + directory : directory;
@@ -9364,7 +9366,7 @@ static void init_engine_backend_ldr()
         WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + L"\\engine\\k-ldr").c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
         directory = folder;
     }
-    if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
+    if ((!g_vk.native && (g_vk.interop == nullptr || g_vk.lifetime == nullptr)) || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
         return;
     const std::string path = directory[0] == '/' ? "Z:" + directory : directory;
     try
@@ -9460,7 +9462,7 @@ static void init_engine_backend_m_variant(bool presetL, bool ldr)
         WideCharToMultiByte(CP_ACP, 0, (g_portable.dir + sub).c_str(), -1, folder, sizeof(folder), nullptr, nullptr);
         directory = folder;
     }
-    if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
+    if ((!g_vk.native && (g_vk.interop == nullptr || g_vk.lifetime == nullptr)) || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
     {
         logf("engine backend (%s): Vulkan interop is unavailable", name);
         return;
@@ -10492,7 +10494,7 @@ static std::string rr_model_directory(int slot)
 static void init_engine_backend_rr(int slot)
 {
     const std::string path = rr_model_directory(slot);
-    if (g_vk.interop == nullptr || g_vk.lifetime == nullptr || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
+    if ((!g_vk.native && (g_vk.interop == nullptr || g_vk.lifetime == nullptr)) || g_vk.device == VK_NULL_HANDLE || d4r::vkTable.CreateBuffer == nullptr)
     {
         logf("engine backend (Ray Reconstruction): Vulkan interop is unavailable");
         g_engineR.state[slot] = 3;

tools/d4r_vulkan_frontend.inl

@@ -9,6 +9,12 @@
 struct NativeVkApi
 {
     VkDevice device = VK_NULL_HANDLE;
+    // The game's instance and adapter: d4r's engine (engine/) loads its entry points on this device.
+    VkInstance instance = VK_NULL_HANDLE;
+    VkPhysicalDevice physical = VK_NULL_HANDLE;
+    PFN_vkGetInstanceProcAddr instanceProc = nullptr;
+    PFN_vkCreateImageView createView = nullptr;
+    PFN_vkDestroyImageView destroyView = nullptr;
     VkPhysicalDeviceMemoryProperties memory = {};
     PFN_vkCreateBuffer createBuffer = nullptr;
     PFN_vkDestroyBuffer destroyBuffer = nullptr;
@@ -70,12 +76,31 @@ struct NativeVkFrame
     uint32_t number = 0;
     VkCommandBuffer commandBuffer = VK_NULL_HANDLE;
 };
+// The engine path (D4R_ENGINE): d4r's own network recorded straight into the game's command buffer, no staging.
+// Each recorded evaluation sets an event at its end, so the engine objects are destroyed only after the GPU has
+// finished every submitted recording (or at shutdown, after the application's device idle).
+struct NativeVkEngineView
+{
+    VkImage image = VK_NULL_HANDLE;
+    VkFormat format = VK_FORMAT_UNDEFINED;
+    VkImageAspectFlags aspect = 0;
+    VkImageView view = VK_NULL_HANDLE;
+};
+struct NativeVkEngineState
+{
+    std::unique_ptr<d4r::GameUpscaler> k;
+    std::unique_ptr<d4r::GameUpscalerM> m, mDynamic;
+    std::vector<VkEvent> pending, idle; // set at the end of a recording / reusable after it was set
+    std::vector<NativeVkEngineView> views; // depth-only views the engine samples where the game's view has stencil
+};
 struct NativeVkFeature
 {
     Feature* feature;
     std::vector<NativeVkFrame*> frames;
     std::vector<NativeVkFrame*> reusable;
+    std::shared_ptr<NativeVkEngineState> engine; // null until the engine records a frame
 };
+static std::vector<std::shared_ptr<NativeVkEngineState>> g_nativeVkEngineRetired;
 static std::vector<NativeVkFeature> g_nativeVkFeatures;
 // Release can happen while recordings are still pending. Their GPU objects
 // outlive CUDA feature state and are reclaimed at completion or shutdown.
@@ -566,6 +591,9 @@ static NgxResult native_vk_init(unsigned long long id, const wchar_t* path, VkIn
     if (!deviceProc || !memory) return NGX_FAIL_PLATFORM_ERROR;
     NativeVkApi api;
     api.device = device;
+    api.instance = instance;
+    api.physical = physical;
+    api.instanceProc = instanceProc;
     memory(physical, &api.memory);
     bool ok = true;
     auto load = [&](auto& fn, const char* name) {
@@ -582,6 +610,7 @@ static NgxResult native_vk_init(unsigned long long id, const wchar_t* path, VkIn
     load(api.cmdSetEvent, "vkCmdSetEvent"); load(api.waitEvents, "vkCmdWaitEvents");
     load(api.barrier, "vkCmdPipelineBarrier"); load(api.idle, "vkDeviceWaitIdle");
     load(api.fill, "vkCmdFillBuffer");
+    load(api.createView, "vkCreateImageView"); load(api.destroyView, "vkDestroyImageView");
     if (!ok) return NGX_FAIL_PLATFORM_ERROR;
     // Match NGX's CUDA adapter identity to the supplied Vulkan physical device.
     auto properties2 = reinterpret_cast<PFN_vkGetPhysicalDeviceProperties2>(instanceProc(instance, "vkGetPhysicalDeviceProperties2"));
@@ -701,9 +730,40 @@ D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureInstanceExtensionRequirements(co
     if (p) *p = nullptr;
     return NGX_SUCCESS;
 }
-D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureDeviceExtensionRequirements(VkInstance, VkPhysicalDevice,
+// With the engine on, ask for the extensions its shaders use (VK_KHR_cooperative_matrix, VK_EXT_shader_float8), where
+// the adapter has them: a game enables what NGX asks for. Their feature bits still need a layer (RADV compiles and runs
+// the shaders without them; the harness's D4R_HARNESS_ENGINE_DEVICE enables them properly).
+D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureDeviceExtensionRequirements(VkInstance instance, VkPhysicalDevice physical,
     const void* info, uint32_t* count, VkExtensionProperties** p)
-{ return NVSDK_NGX_VULKAN_GetFeatureInstanceExtensionRequirements(info, count, p); }
+{
+    if (!count) return NGX_FAIL_INVALID_PARAMETER;
+    static VkExtensionProperties wanted[2] = {};
+    static uint32_t wantedCount = 0;
+    static std::once_flag once;
+    // Games ask before NGX init: d4r.ini (which sets D4R_ENGINE) must be applied first, or engine_requested()
+    // caches "off" for the whole process.
+    ensure_portable_config();
+    if (engine_requested() && env_uint("D4R_VULKAN_ENGINE", 1) != 0 && instance && physical)
+        std::call_once(once, [&] {
+            HMODULE module = LoadLibraryA("vulkan-1.dll");
+            PFN_vkGetInstanceProcAddr instanceProc = nullptr;
+            if (!module || !load_export(module, "vkGetInstanceProcAddr", instanceProc)) return;
+            auto enumerate = reinterpret_cast<PFN_vkEnumerateDeviceExtensionProperties>(
+                instanceProc(instance, "vkEnumerateDeviceExtensionProperties"));
+            uint32_t n = 0;
+            if (!enumerate || enumerate(physical, nullptr, &n, nullptr) != VK_SUCCESS) return;
+            std::vector<VkExtensionProperties> available(n);
+            if (enumerate(physical, nullptr, &n, available.data()) != VK_SUCCESS) return;
+            for (const char* name : {"VK_KHR_cooperative_matrix", "VK_EXT_shader_float8"})
+                for (const auto& extension : available)
+                    if (std::strcmp(extension.extensionName, name) == 0) wanted[wantedCount++] = extension;
+            logf("native Vulkan engine: device extension requirements: %u", wantedCount);
+        });
+    *count = wantedCount;
+    if (p) *p = wantedCount ? wanted : nullptr;
+    (void)info;
+    return NGX_SUCCESS;
+}
 D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_GetFeatureRequirements(VkInstance, VkPhysicalDevice, const void*, NgxFeatureRequirement* p)
 { return p ? NVSDK_NGX_D3D12_GetFeatureRequirements(nullptr, nullptr, p) : NGX_FAIL_INVALID_PARAMETER; }
 
@@ -721,6 +781,355 @@ D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_CreateFeature1(VkDevice device, VkCommandB
     return NVSDK_NGX_VULKAN_CreateFeature(cmd, id, p, handle);
 }
 
+static void native_vk_read_params(const Feature* feature, void* p, FrameParams& params)
+{
+    params.jitterX = get_float_or(p, "Jitter.Offset.X", 0); params.jitterY = get_float_or(p, "Jitter.Offset.Y", 0);
+    params.mvScaleX = get_float_or(p, "MV.Scale.X", 1); params.mvScaleY = get_float_or(p, "MV.Scale.Y", 1);
+    params.sharpness = get_float_or(p, "Sharpness", 0); params.reset = get_int_or(p, "Reset", 0);
+    params.preExposure = get_float_or(p, "DLSS.Pre.Exposure", 1); params.exposureScale = get_float_or(p, "DLSS.Exposure.Scale", 1);
+    params.frameTime = get_float_or(p, "FrameTimeDeltaInMsec", 16.6f);
+    params.renderWidth = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Width", feature->width);
+    params.renderHeight = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Height", feature->height);
+    if (!params.renderWidth) params.renderWidth = feature->width;
+    if (!params.renderHeight) params.renderHeight = feature->height;
+    params.colorBaseX = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.X", 0);
+    params.colorBaseY = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.Y", 0);
+    params.depthBaseX = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.X", 0);
+    params.depthBaseY = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.Y", 0);
+    params.mvBaseX = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.X", 0);
+    params.mvBaseY = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.Y", 0);
+    params.outputBaseX = get_uint_or(p, "DLSS.Output.Subrect.Base.X", 0);
+    params.outputBaseY = get_uint_or(p, "DLSS.Output.Subrect.Base.Y", 0);
+    params.invertX = get_int_or(p, "DLSS.Indicator.Invert.X.Axis", 0);
+    params.invertY = get_int_or(p, "DLSS.Indicator.Invert.Y.Axis", 0);
+}
+
+// --- Engine path: d4r's Vulkan network on the game's device, recorded into the game's command buffer ----------
+// D4R_ENGINE (d4r.ini [Engine]) selects it; D4R_VULKAN_ENGINE=0 keeps the staging path. It needs the engine's
+// device features on the game's device (cooperative matrix, and shader float8 for M/L): a game does not enable
+// those itself, so a layer or the harness (D4R_HARNESS_ENGINE_DEVICE) must. Frames the engine does not cover
+// return false and take the CUDA staging path.
+static std::once_flag g_nativeVkEngineOnce;
+static bool g_nativeVkEngineReady = false;
+
+static void native_vk_engine_init()
+{
+    if (!d4r::loadVulkan(g_nativeVk.instanceProc, g_nativeVk.instance, g_nativeVk.device))
+    {
+        logf("native Vulkan engine: Vulkan entry points are unavailable");
+        return;
+    }
+    g_vk.native = true;
+    g_vk.physical = g_nativeVk.physical;
+    g_vk.device = g_nativeVk.device;
+    auto enumerate = reinterpret_cast<PFN_vkEnumerateDeviceExtensionProperties>(
+        g_nativeVk.instanceProc(g_nativeVk.instance, "vkEnumerateDeviceExtensionProperties"));
+    uint32_t count = 0;
+    bool cooperative = false;
+    if (enumerate && enumerate(g_nativeVk.physical, nullptr, &count, nullptr) == VK_SUCCESS)
+    {
+        std::vector<VkExtensionProperties> extensions(count);
+        if (enumerate(g_nativeVk.physical, nullptr, &count, extensions.data()) == VK_SUCCESS)
+            for (const auto& extension : extensions)
+            {
+                g_vk.float8 |= std::strcmp(extension.extensionName, "VK_EXT_shader_float8") == 0;
+                cooperative |= std::strcmp(extension.extensionName, "VK_KHR_cooperative_matrix") == 0;
+            }
+    }
+    if (!cooperative)
+    {
+        logf("native Vulkan engine: the GPU has no VK_KHR_cooperative_matrix");
+        return;
+    }
+    g_nativeVkEngineReady = true;
+    logf("native Vulkan engine: ready on device %p (VK_EXT_shader_float8 %s); the device must have the engine's "
+         "features enabled", static_cast<void*>(g_nativeVk.device), g_vk.float8 ? "present" : "absent");
+}
+
+static bool native_vk_engine_idle(NativeVkEngineState& state)
+{
+    for (auto it = state.pending.begin(); it != state.pending.end();)
+        if (g_nativeVk.eventStatus(g_nativeVk.device, *it) == VK_EVENT_SET &&
+            g_nativeVk.resetEvent(g_nativeVk.device, *it) == VK_SUCCESS)
+        { state.idle.push_back(*it); it = state.pending.erase(it); }
+        else ++it;
+    return state.pending.empty();
+}
+
+static void native_vk_engine_destroy(NativeVkEngineState& state)
+{
+    state.k.reset(); state.m.reset(); state.mDynamic.reset();
+    for (auto& view : state.views) g_nativeVk.destroyView(g_nativeVk.device, view.view, nullptr);
+    for (VkEvent event : state.pending) g_nativeVk.destroyEvent(g_nativeVk.device, event, nullptr);
+    for (VkEvent event : state.idle) g_nativeVk.destroyEvent(g_nativeVk.device, event, nullptr);
+    state.views.clear(); state.pending.clear(); state.idle.clear();
+}
+
+static void native_vk_engine_reap()
+{
+    for (auto it = g_nativeVkEngineRetired.begin(); it != g_nativeVkEngineRetired.end();)
+        if (native_vk_engine_idle(**it)) { native_vk_engine_destroy(**it); it = g_nativeVkEngineRetired.erase(it); }
+        else ++it;
+}
+
+// A sampled view of `resource` the engine can read: the game's own, or a depth-only one for depth/stencil views.
+static bool native_vk_engine_image(NativeVkEngineState& state, const NgxVkResource* resource, bool output,
+                                   d4r::GameImage& image)
+{
+    if (!resource || resource->type != 0) return false;
+    const NgxVkImage& source = resource->resource.image;
+    if (!source.image || !source.view || source.range.levelCount > 1 || source.range.layerCount > 1) return false;
+    image.image = source.image;
+    image.format = source.format;
+    image.view = source.view;
+    image.aspect = VK_IMAGE_ASPECT_COLOR_BIT;
+    static const bool generalInputs = env_uint("D4R_VULKAN_INPUT_GENERAL", 0) != 0;
+    image.layout = output || generalInputs ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+    const bool depth = source.format == VK_FORMAT_D32_SFLOAT || source.format == VK_FORMAT_D32_SFLOAT_S8_UINT ||
+                       source.format == VK_FORMAT_D16_UNORM || source.format == VK_FORMAT_D24_UNORM_S8_UINT ||
+                       source.format == VK_FORMAT_X8_D24_UNORM_PACK32 || source.format == VK_FORMAT_D16_UNORM_S8_UINT;
+    if (!depth) return true;
+    image.aspect = VK_IMAGE_ASPECT_DEPTH_BIT;
+    if (source.range.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT) return true;
+    auto found = std::find_if(state.views.begin(), state.views.end(), [&](const auto& view) {
+        return view.image == source.image && view.format == source.format;
+    });
+    if (found == state.views.end())
+    {
+        if (state.views.size() >= 16) return false;
+        VkImageViewCreateInfo info = {VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
+        info.image = source.image; info.viewType = VK_IMAGE_VIEW_TYPE_2D; info.format = source.format;
+        info.subresourceRange = {VK_IMAGE_ASPECT_DEPTH_BIT, source.range.baseMipLevel, 1, source.range.baseArrayLayer, 1};
+        NativeVkEngineView view{source.image, source.format, VK_IMAGE_ASPECT_DEPTH_BIT};
+        if (g_nativeVk.createView(g_nativeVk.device, &info, nullptr, &view.view) != VK_SUCCESS) return false;
+        state.views.push_back(view);
+        found = state.views.end() - 1;
+    }
+    image.view = found->view;
+    return true;
+}
+
+// True: the engine took this evaluation (result says how it went). False: use the staging path.
+static bool native_vk_engine(NativeVkFeature& owner, VkCommandBuffer cmd, void* p, NgxResult& result)
+{
+    ensure_portable_config();
+    static const bool enabled = engine_requested() && env_uint("D4R_VULKAN_ENGINE", 1) != 0;
+    Feature& feature = *owner.feature;
+    if (!enabled || feature.engineRefused || feature.rayReconstruction) return false;
+    const uint32_t frame = feature.frame + 1;
+    const bool presetM = feature.preset == 13 || feature.preset == 12, presetL = feature.preset == 12;
+    const char* const name = feature.preset == 12 ? "L" : feature.preset == 13 ? "M" : "K";
+    auto refuse = [&](const char* reason) {
+        feature.engineRefused = true;
+        logf("native Vulkan engine (%s): %s; feature %u uses the staging path from frame %u", name, reason,
+             feature.handle.Id, frame);
+        return false;
+    };
+    std::call_once(g_nativeVkEngineOnce, native_vk_engine_init);
+    if (!g_nativeVkEngineReady) return refuse("unavailable");
+    if (!presetM && feature.preset != 11) return refuse("only presets K, L and M are implemented");
+    // NVSDK_NGX_DLSS_Feature_Flags: IsHDR 1, MVLowRes 2, MVJittered 4, DepthInverted 8, AutoExposure 0x40.
+    const bool ldr = (feature.flags & 0x1) == 0;
+    const bool displayMotion = (feature.flags & 0x2) == 0;
+    const bool autoExposure = (feature.flags & 0x40) != 0;
+    d4r::Model* model = nullptr;
+    d4r::MModel* modelM = nullptr;
+    if (presetM)
+    {
+        if (!g_vk.float8) return refuse("the GPU has no VK_EXT_shader_float8 for the FP8 layers");
+        std::call_once(g_engineM.once[presetL][ldr], [presetL, ldr] { init_engine_backend_m_variant(presetL, ldr); });
+        modelM = g_engineM.models[presetL][ldr].get();
+        if (!modelM) return refuse(ldr ? "no LDR model for a game without the HDR flag" : "no model");
+        if (displayMotion) return refuse("needs render-resolution motion vectors");
+    }
+    else
+    {
+        if (ldr) std::call_once(g_engine.ldrOnce, init_engine_backend_ldr);
+        else std::call_once(g_engine.once, init_engine_backend);
+        model = ldr ? g_engine.modelLdr.get() : g_engine.model.get();
+        if (!model) return refuse(ldr ? "no LDR model (engine\\k-ldr) for a game without the HDR flag" : "no model");
+        if (displayMotion && !model->hasDisplayMotion()) return refuse("display-resolution motion needs a newer model");
+        if ((feature.flags & 0x8) == 0 && !model->hasRegularDepth()) return refuse("regular depth needs a newer model");
+        if (!ldr && autoExposure && !model->hasAutoExposure()) return refuse("automatic exposure needs exposure shaders");
+    }
+    FrameParams params;
+    native_vk_read_params(&feature, p, params);
+    void* mask = nullptr; d4r_ngx_get_void(p, "DLSS.Input.Bias.Current.Color.Mask", &mask);
+    void* transparency = nullptr; d4r_ngx_get_void(p, "TransparencyMask", &transparency);
+    if (mask || transparency) return refuse("bias colour and transparency masks are not implemented");
+
+    if (!owner.engine) owner.engine = std::make_shared<NativeVkEngineState>();
+    NativeVkEngineState& state = *owner.engine;
+    const char* names[] = {"Color", "Depth", "MotionVectors", "ExposureTexture", "Output"};
+    const NgxVkResource* resources[5] = {};
+    for (int i = 0; i < 5; ++i)
+    {
+        void* resource = nullptr;
+        d4r_ngx_get_void(p, names[i], &resource);
+        resources[i] = static_cast<const NgxVkResource*>(resource);
+    }
+    d4r::GameImage color, depth, motion, exposure, output;
+    if (!native_vk_engine_image(state, resources[0], false, color) || !native_vk_engine_image(state, resources[1], false, depth) ||
+        !native_vk_engine_image(state, resources[2], false, motion) || !native_vk_engine_image(state, resources[4], true, output))
+        return refuse("a resource is not a single-mip, single-layer 2D image view");
+    const bool gameExposure = !presetM && !ldr && !autoExposure;
+    if (gameExposure && !native_vk_engine_image(state, resources[3], false, exposure))
+        return refuse("exposure without AutoExposure needs an exposure texture");
+    color.x = params.colorBaseX; color.y = params.colorBaseY;
+    depth.x = params.depthBaseX; depth.y = params.depthBaseY;
+    motion.x = params.mvBaseX; motion.y = params.mvBaseY;
+    output.x = params.outputBaseX; output.y = params.outputBaseY;
+    if (output.image == color.image || output.image == depth.image || output.image == motion.image)
+        return refuse("input/output texture aliasing is not supported");
+    if (!d4r::gameColorFormat(color.format) || !d4r::gameMotionFormat(motion.format) ||
+        !d4r::gameDepthFormat(depth.format, depth.aspect) || !d4r::gameOutputFormat(output.format))
+    {
+        logf("native Vulkan engine (%s): formats colour %d, depth %d, motion %d, output %d", name, color.format,
+             depth.format, motion.format, output.format);
+        return refuse("a texture format is not implemented");
+    }
+    // The Vulkan resource contract gives the output STORAGE usage only: the engine's store must write it directly.
+    const bool rgb10 = presetM ? modelM->hasRgb10Output() : model->hasRgb10Output();
+    const bool direct = output.view != VK_NULL_HANDLE && output.layout == VK_IMAGE_LAYOUT_GENERAL &&
+        (output.format == VK_FORMAT_R16G16B16A16_SFLOAT || (rgb10 && output.format == VK_FORMAT_A2B10G10R10_UNORM_PACK32));
+    if (!direct)
+        return refuse("the output must be an RGBA16F (or RGB10A2) storage image the engine writes directly");
+    if (resources[4]->resource.image.width != feature.outWidth || resources[4]->resource.image.height != feature.outHeight)
+        return refuse("the output does not match the feature's output size");
+
+    // An event per recording marks GPU completion; reclaim the ones that have fired.
+    native_vk_engine_idle(state);
+    VkEvent done = VK_NULL_HANDLE;
+    if (!state.idle.empty()) { done = state.idle.back(); state.idle.pop_back(); }
+    else
+    {
+        if (state.pending.size() >= 16) { result = NGX_FAIL_PLATFORM_ERROR; return true; }
+        VkEventCreateInfo info = {VK_STRUCTURE_TYPE_EVENT_CREATE_INFO};
+        if (g_nativeVk.createEvent(g_nativeVk.device, &info, nullptr, &done) != VK_SUCCESS)
+        { result = NGX_FAIL_PLATFORM_ERROR; return true; }
+    }
+    auto releaseEvent = [&] { state.idle.push_back(done); };
+
+    VkMemoryBarrier before = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
+    before.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT;
+    before.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT;
+    VkMemoryBarrier after = {VK_STRUCTURE_TYPE_MEMORY_BARRIER};
+    after.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
+    after.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
+    try
+    {
+        if (presetM)
+        {
+            d4r::GameFrameM game;
+            game.color = color; game.depth = depth; game.motion = motion; game.output = output;
+            if (color.x || color.y || depth.x || depth.y || motion.x || motion.y)
+                return releaseEvent(), refuse("input rectangles must start at the origin");
+            game.jitter[0] = params.jitterX; game.jitter[1] = params.jitterY;
+            game.motionToRender[0] = params.mvScaleX; game.motionToRender[1] = params.mvScaleY;
+            if ((feature.flags & 0x4) != 0 && params.mvScaleX != 0.0f && params.mvScaleY != 0.0f)
+            {
+                game.motionOffset[0] = (params.jitterX - feature.engineJitter[0]) / params.mvScaleX;
+                game.motionOffset[1] = (params.jitterY - feature.engineJitter[1]) / params.mvScaleY;
+            }
+            game.renderWidth = params.renderWidth; game.renderHeight = params.renderHeight;
+            game.reset = params.reset != 0 || !state.m || params.renderWidth != feature.engineMLastWidth ||
+                params.renderHeight != feature.engineMLastHeight;
+            game.depthInverted = (feature.flags & 0x8) != 0;
+            if (!state.m)
+            {
+                const bool dynamic = modelM->hasDynamicResolution();
+                const uint32_t width = dynamic ? std::min(feature.outWidth, std::max(feature.width, params.renderWidth)) : params.renderWidth;
+                const uint32_t height = dynamic ? std::min(feature.outHeight, std::max(feature.height, params.renderHeight)) : params.renderHeight;
+                state.m = std::make_unique<d4r::GameUpscalerM>(*modelM, g_vk.physical, g_vk.device, feature.outWidth,
+                                                               feature.outHeight, width, height);
+                feature.engineMRenderWidth = width; feature.engineMRenderHeight = height;
+                logf("native Vulkan engine (%s): feature %u runs preset %s in the game's command buffer (%ux%u -> %ux%u)",
+                     name, feature.handle.Id, name, params.renderWidth, params.renderHeight, feature.outWidth, feature.outHeight);
+            }
+            d4r::GameUpscalerM* adapter = state.m.get();
+            if (params.renderWidth > feature.engineMRenderWidth || params.renderHeight > feature.engineMRenderHeight)
+            {
+                if (!state.mDynamic)
+                    state.mDynamic = std::make_unique<d4r::GameUpscalerM>(*modelM, g_vk.physical, g_vk.device,
+                        feature.outWidth, feature.outHeight, feature.outWidth, feature.outHeight);
+                adapter = state.mDynamic.get();
+            }
+            d4r::vkTable.CmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                                            0, 1, &before, 0, nullptr, 0, nullptr);
+            adapter->record(cmd, game);
+            feature.engineMLastWidth = params.renderWidth; feature.engineMLastHeight = params.renderHeight;
+            if (frame <= 3 || frame % 600 == 0)
+                logf("native Vulkan engine frame %u recorded, preset %s (jitter %.6f,%.6f, reset %d, mvScale %.3f,%.3f, render %ux%u)",
+                     frame, name, params.jitterX, params.jitterY, game.reset, params.mvScaleX, params.mvScaleY,
+                     params.renderWidth, params.renderHeight);
+        }
+        else
+        {
+            if (gameExposure && (exposure.format != VK_FORMAT_R32_SFLOAT && exposure.format != VK_FORMAT_R16_SFLOAT &&
+                                 exposure.format != VK_FORMAT_R32G32B32A32_SFLOAT && exposure.format != VK_FORMAT_R16G16B16A16_SFLOAT))
+                return releaseEvent(), refuse("the exposure texture must be a float texture");
+            if (!ldr && !(std::isfinite(params.exposureScale) && params.exposureScale > 0.0f))
+                return releaseEvent(), refuse("the exposure scale is not a positive number");
+            if (!ldr && !(std::isfinite(params.preExposure) && params.preExposure > 0.0f))
+                return releaseEvent(), refuse("the pre-exposure is not a positive number");
+            if ((color.x || color.y || depth.x || depth.y || motion.x || motion.y || output.x || output.y) &&
+                !model->hasDirectOrigins())
+                return releaseEvent(), refuse("texture rectangles need a model with direct origins");
+            d4r::GameFrame game;
+            game.color = color; game.depth = depth; game.motion = motion; game.output = output;
+            if (gameExposure) game.exposure = exposure;
+            auto& settings = game.settings;
+            settings.renderWidth = params.renderWidth; settings.renderHeight = params.renderHeight;
+            settings.jitter[0] = params.jitterX; settings.jitter[1] = params.jitterY;
+            settings.displayMotion = displayMotion;
+            const float toOutput = displayMotion ? 1.0f : float(feature.outWidth) / float(params.renderWidth);
+            const float toOutputY = displayMotion ? 1.0f : float(feature.outHeight) / float(params.renderHeight);
+            settings.motionScale[0] = params.mvScaleX * toOutput; settings.motionScale[1] = params.mvScaleY * toOutputY;
+            if ((feature.flags & 0x4) != 0 && params.mvScaleX != 0.0f && params.mvScaleY != 0.0f)
+            {
+                settings.motionOffset[0] = (params.jitterX - feature.engineJitter[0]) / params.mvScaleX;
+                settings.motionOffset[1] = (params.jitterY - feature.engineJitter[1]) / params.mvScaleY;
+            }
+            settings.reset = params.reset != 0 || !state.k;
+            settings.depthInverted = (feature.flags & 0x8) != 0;
+            settings.autoExposure = !ldr && autoExposure;
+            settings.exposureScale = ldr ? 1.0f : params.exposureScale;
+            settings.preExposure = ldr ? 1.0f : params.preExposure;
+            if (!state.k)
+            {
+                state.k = std::make_unique<d4r::GameUpscaler>(*model, g_vk.physical, g_vk.device, feature.outWidth,
+                                                              feature.outHeight, feature.outWidth, feature.outHeight);
+                logf("native Vulkan engine (K): feature %u runs preset K in the game's command buffer (%ux%u -> %ux%u)",
+                     feature.handle.Id, params.renderWidth, params.renderHeight, feature.outWidth, feature.outHeight);
+            }
+            d4r::vkTable.CmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                                            0, 1, &before, 0, nullptr, 0, nullptr);
+            state.k->record(cmd, game);
+            if (frame <= 3 || frame % 600 == 0)
+                logf("native Vulkan engine frame %u recorded, preset K (jitter %.6f,%.6f, reset %d, mvScale %.3f,%.3f, render %ux%u)",
+                     frame, params.jitterX, params.jitterY, settings.reset, params.mvScaleX, params.mvScaleY,
+                     params.renderWidth, params.renderHeight);
+        }
+    }
+    catch (const std::exception& error)
+    {
+        // Something may already be in the command buffer: the caller must not submit a half recording.
+        logf("native Vulkan engine (%s): %s", name, error.what());
+        releaseEvent();
+        feature.engineRefused = true;
+        result = NGX_FAIL_PLATFORM_ERROR;
+        return true;
+    }
+    d4r::vkTable.CmdPipelineBarrier(cmd, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+                                    0, 1, &after, 0, nullptr, 0, nullptr);
+    g_nativeVk.cmdSetEvent(cmd, done, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
+    state.pending.push_back(done);
+    feature.engineJitter[0] = params.jitterX; feature.engineJitter[1] = params.jitterY;
+    feature.frame = frame;
+    result = NGX_SUCCESS;
+    return true;
+}
+
 D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_EvaluateFeature(VkCommandBuffer cmd, const NgxHandle* handle, void* p, void*)
 {
     std::lock_guard<std::mutex> lock(g_nativeVkMutex);
@@ -731,6 +1140,9 @@ D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_EvaluateFeature(VkCommandBuffer cmd, const
     if (!cmd || !p || it == g_nativeVkFeatures.end()) return NGX_FAIL_INVALID_PARAMETER;
     auto& owner = *it;
     if (owner.feature->retiring) return NGX_FAIL_INVALID_PARAMETER;
+    native_vk_engine_reap();
+    if (NgxResult engineResult; owner.frames.empty() && native_vk_engine(owner, cmd, p, engineResult))
+        return engineResult;
     for (auto* frame : owner.frames)
         if (frame->done && frame->result != NGX_SUCCESS) return frame->result;
     native_vk_reap(g_nativeVkRetired);
@@ -781,26 +1193,8 @@ D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_EvaluateFeature(VkCommandBuffer cmd, const
     if (frame->planes[4].image.width != feature->outWidth || frame->planes[4].image.height != feature->outHeight)
     { native_vk_free_frame(frame); return NGX_FAIL_INVALID_PARAMETER; }
     auto& params = frame->params;
-    params.jitterX = get_float_or(p, "Jitter.Offset.X", 0); params.jitterY = get_float_or(p, "Jitter.Offset.Y", 0);
+    native_vk_read_params(feature, p, params);
-    params.mvScaleX = get_float_or(p, "MV.Scale.X", 1); params.mvScaleY = get_float_or(p, "MV.Scale.Y", 1);
-    params.sharpness = get_float_or(p, "Sharpness", 0); params.reset = get_int_or(p, "Reset", 0);
-    params.preExposure = get_float_or(p, "DLSS.Pre.Exposure", 1); params.exposureScale = get_float_or(p, "DLSS.Exposure.Scale", 1);
-    params.frameTime = get_float_or(p, "FrameTimeDeltaInMsec", 16.6f);
-    params.renderWidth = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Width", feature->width);
-    params.renderHeight = get_uint_or(p, "DLSS.Render.Subrect.Dimensions.Height", feature->height);
-    if (!params.renderWidth) params.renderWidth = feature->width;
-    if (!params.renderHeight) params.renderHeight = feature->height;
-    params.colorBaseX = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.X", 0);
-    params.colorBaseY = get_uint_or(p, "DLSS.Input.Color.Subrect.Base.Y", 0);
-    params.depthBaseX = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.X", 0);
-    params.depthBaseY = get_uint_or(p, "DLSS.Input.Depth.Subrect.Base.Y", 0);
-    params.mvBaseX = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.X", 0);
-    params.mvBaseY = get_uint_or(p, "DLSS.Input.MV.Subrect.Base.Y", 0);
-    params.outputBaseX = get_uint_or(p, "DLSS.Output.Subrect.Base.X", 0);
-    params.outputBaseY = get_uint_or(p, "DLSS.Output.Subrect.Base.Y", 0);
     params.hasExposure = frame->planes[3].buffer != VK_NULL_HANDLE;
-    params.invertX = get_int_or(p, "DLSS.Indicator.Invert.X.Axis", 0);
-    params.invertY = get_int_or(p, "DLSS.Indicator.Invert.Y.Axis", 0);
     // Output subrect conversion and auxiliary masks are not implemented yet.
     void* mask = nullptr;
     d4r_ngx_get_void(p, "DLSS.Input.Bias.Current.Color.Mask", &mask);
@@ -876,6 +1270,12 @@ static NgxResult native_vk_release(std::vector<NativeVkFeature>::iterator it)
         if (native_vk_consumed(*frame)) native_vk_free_frame(frame);
         else g_nativeVkRetired.push_back(frame);
     for (auto* frame : it->reusable) native_vk_free_frame(frame);
+    if (it->engine)
+    {
+        if (native_vk_engine_idle(*it->engine)) native_vk_engine_destroy(*it->engine);
+        else g_nativeVkEngineRetired.push_back(it->engine);
+        it->engine.reset();
+    }
     logf("native Vulkan feature %u released after %u completed frames",
          it->feature->handle.Id, it->feature->completedFrames.load());
     it->feature->retiring = true;
@@ -903,6 +1303,8 @@ D4R_EXPORT NgxResult NVSDK_NGX_VULKAN_Shutdown()
         if (g_nativeVk.idle(g_nativeVk.device) != VK_SUCCESS) return NGX_FAIL_PLATFORM_ERROR;
         for (auto* frame : g_nativeVkRetired) native_vk_free_frame(frame);
         g_nativeVkRetired.clear();
+        for (auto& state : g_nativeVkEngineRetired) native_vk_engine_destroy(*state);
+        g_nativeVkEngineRetired.clear();
         for (auto& plane : g_nativeVkBufferPool) native_vk_free_plane(plane, true);
         g_nativeVkBufferPool.clear(); g_nativeVkExternalAllocations = 0;
         g_nativeVkStaging.destroy();

tools/vulkan_dlss_harness.cpp

@@ -28,7 +28,7 @@ using Shutdown = Result (*)();
 
 #define VK_FUNCTIONS(X) \
 X(CreateInstance) X(EnumeratePhysicalDevices) X(GetPhysicalDeviceProperties) X(EnumerateDeviceExtensionProperties) \
-X(GetPhysicalDeviceQueueFamilyProperties) X(GetPhysicalDeviceMemoryProperties) X(CreateDevice) \
+X(GetPhysicalDeviceQueueFamilyProperties) X(GetPhysicalDeviceMemoryProperties) X(GetPhysicalDeviceFeatures2) X(CreateDevice) \
 X(GetDeviceProcAddr) X(GetDeviceQueue) X(CreateCommandPool) X(AllocateCommandBuffers) \
 X(BeginCommandBuffer) X(EndCommandBuffer) X(ResetCommandPool) X(QueueSubmit) X(QueueWaitIdle) X(CreateFence) X(WaitForFences) X(ResetFences) X(DestroyFence) \
 X(CreateImage) X(GetImageMemoryRequirements) X(AllocateMemory) X(BindImageMemory) X(CreateImageView) \
@@ -102,7 +102,10 @@ int main(int argc, char** argv)
 #define LOAD(name) vk##name = reinterpret_cast<PFN_vk##name>(ip(VK_NULL_HANDLE, "vk" #name));
     LOAD(CreateInstance)
 #undef LOAD
-    VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion = VK_API_VERSION_1_1;
+    // D4R_HARNESS_ENGINE_DEVICE=1: a Vulkan 1.3 device with every supported 1.1-1.3 feature plus cooperative matrix
+    // and shader float8, as d4r's engine (and a layer in a game) needs.
+    const bool engineDevice = std::getenv("D4R_HARNESS_ENGINE_DEVICE") != nullptr;
+    VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; app.apiVersion = engineDevice ? VK_API_VERSION_1_3 : VK_API_VERSION_1_1;
     VkInstanceCreateInfo ci = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; ci.pApplicationInfo = &app;
     VkInstance instance;
     check(vkCreateInstance(&ci,nullptr,&instance),"instance");
@@ -142,6 +145,28 @@ int main(int argc, char** argv)
         di.enabledExtensionCount=1; di.ppEnabledExtensionNames=&externalExtension;
         std::puts("enabling external-memory VRAM interop");
     }
+    VkPhysicalDeviceShaderFloat8FeaturesEXT f8 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT8_FEATURES_EXT};
+    VkPhysicalDeviceCooperativeMatrixFeaturesKHR cm = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR, &f8};
+    VkPhysicalDeviceVulkan13Features v13 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES, &cm};
+    VkPhysicalDeviceVulkan12Features v12 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES, &v13};
+    VkPhysicalDeviceVulkan11Features v11 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES, &v12};
+    VkPhysicalDeviceFeatures2 f2 = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &v11};
+    std::vector<const char*> engineExtensions;
+    if (engineDevice) {
+        vkGetPhysicalDeviceFeatures2(physical,&f2);
+        check(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,nullptr),"extensions");
+        std::vector<VkExtensionProperties> extensions(count);
+        check(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,extensions.data()),"extensions");
+        bool float8 = false;
+        for (const auto& e : extensions) float8 |= std::strcmp(e.extensionName,"VK_EXT_shader_float8")==0;
+        if (!float8) { f8.shaderFloat8 = f8.shaderFloat8CooperativeMatrix = VK_FALSE; cm.pNext = nullptr; }
+        if (cm.cooperativeMatrix) engineExtensions.push_back("VK_KHR_cooperative_matrix"); else v13.pNext = nullptr;
+        if (float8) engineExtensions.push_back("VK_EXT_shader_float8");
+        for (uint32_t i = 0; i < di.enabledExtensionCount; ++i) engineExtensions.push_back(di.ppEnabledExtensionNames[i]);
+        di.pNext = &f2; di.enabledExtensionCount = static_cast<uint32_t>(engineExtensions.size());
+        di.ppEnabledExtensionNames = engineExtensions.data();
+        std::printf("engine device: cooperative matrix %u, float8 %u\n", cm.cooperativeMatrix, f8.shaderFloat8);
+    }
     check(vkCreateDevice(physical,&di,nullptr,&device),"device");
     auto dp = vkGetDeviceProcAddr;
 #define LOAD(name) vk##name = reinterpret_cast<PFN_vk##name>(dp(device, "vk" #name));
@@ -247,7 +272,8 @@ int main(int argc, char** argv)
         unsigned nonzero=0, nonfinite=0;
         for (unsigned j=0;j<OW*OH*4;++j) { nonzero += (values[j]&0x7fff)!=0; nonfinite += (values[j]&0x7c00)==0x7c00; }
         std::printf("frame %u: nonzero=%u nonfinite=%u center=%04x,%04x,%04x,%04x\n",n,nonzero,nonfinite,values[((OH/2)*OW+OW/2)*4],values[((OH/2)*OW+OW/2)*4+1],values[((OH/2)*OW+OW/2)*4+2],values[((OH/2)*OW+OW/2)*4+3]);
-        if (!nonzero || nonfinite) return 1;
+        // D4R_HARNESS_ALLOW_BLACK=1: a black frame is reported but not fatal (NGX's K starts black on the reset frame here).
+        if (nonfinite || (!nonzero && !std::getenv("D4R_HARNESS_ALLOW_BLACK"))) return 1;
         if (n+1==frames) { FILE* out=std::fopen(argv[2],"wb"); if (!out) return 1; const auto written=std::fwrite(data,1,OW*OH*8,out); std::fclose(out); if (written!=OW*OH*8) return 1; }
         vkUnmapMemory(device,readMemory);
     }