Engine: native presets K and M in the D3D12 shim, wider game coverage

A standalone Vulkan inference engine (engine/) for DLSS presets K and M that
runs from weights extracted from the user's own nvngx_dlss.dll, with d4r's
own input, network and reconstruction shaders or the native HIP network
layers, recorded into the game's command list by the shim. Opt-in
([Engine] Enabled / D4R_ENGINE=1); anything uncovered falls back to the
CUDA/ZLUDA path with the reason logged.

Byte-identical to NGX's CUDA backend under the phase-capable reference
(1706x960 -> 2560x1440, native HIP network, eleven frames, RX 9070 XT):
- K render- and display-resolution motion vectors, HDR and LDR
- K game-supplied exposure and automatic exposure with any pre-exposure
  and exposure scale (one residual: effective exposure exactly 2 or 4)
- K and M with the DLSS indicator's inverted-axis parameters
- M with any exposure settings (NGX ignores them for M)
K packages need the current compile_k.py (marker D4RO0003).

Also: HIP shared buffers are pooled across feature recreation (ROCm leaks
destroyed imports), hipnet.bin must match the preset, M-only games do not
load the K model, install_d4r_runtime.sh stages k, k-ldr, m, m-ldr with
hip/, the activation table has a generator (gen_activation_m.py), and
package_release.sh can bundle the engine's shaders (never its weights).

Signed-off-by: woutermans <[email protected]>

tests/test_frame_completion.py

@@ -12,6 +12,9 @@ class FrameCompletionTests(unittest.TestCase):
         shim = (ROOT / 'tools/d4r_nvngx_shim.cpp').read_text()
         start = shim.index('static void release_split_frame(Feature* feature, uint32_t frame)\n{')
         end = shim.index('\nstatic void split_watchdog()', start)
+        # release_split_frame signals through this helper, defined after the engine network's exports
+        helper = shim.index('\nstatic VkResult signal_split_semaphore(Feature& feature, uint64_t value)\n{')
+        helper = shim[helper:shim.index('\n}\n', helper) + 3]
         source = r'''
 #include "d4r_frame_completion.h"
 #include <cassert>
@@ -25,7 +28,10 @@ struct Feature {
     D4rFrameCompletion splitCompletion;
     uint64_t splitSignalled = 0;
     int splitSemaphore = 1;
+    bool engineNetTimeline = false; // the Vulkan timeline of this test, not the native network's
+    int engineNet = 0;
 };
+static struct { int (*signalTimeline)(int, uint64_t) = nullptr; } g_engineNet;
 static uint64_t gpuTimeline = 0;
 static bool failSignal = false;
 static int signal(int, const VkSemaphoreSignalInfo* info) {
@@ -36,7 +42,7 @@ static int signal(int, const VkSemaphoreSignalInfo* info) {
 }
 static struct { int device = 1; decltype(&signal) signalSemaphore = signal; } g_vk;
 static void logf(const char*, ...) {}
-''' + shim[start:end] + r'''
+''' + helper + shim[start:end] + r'''
 int main() {
     Feature feature;
     feature.splitCompletion.admit(1); // still running on the CUDA worker

tests/test_optiscaler_packaging.py

@@ -147,6 +147,11 @@ class OptiScalerConfigTests(unittest.TestCase):
                        D4R_BUNDLE_DLSS=str(dlss), D4R_BUNDLE_NGX=str(core), D4R_BUNDLE_TEX=str(inputs / 'tex'))
             version = (repo / 'packaging/VERSION').read_text().strip()
             configs = []
+            # A compiled engine model: only d4r's shaders and marker may reach a zip.
+            for name in ('input_k.spv', 'output_k.spv', 'exposure_k0.spv', 'exposure_k1.spv', 'enc0.spv',
+                         'direct_origins.bin', 'weights.bin', 'offsets.bin', 'lut.bin', 'hipnet.bin', 'model.json'):
+                fixture(inputs / 'engine' / name)
+            env['D4R_BUNDLE_ENGINE'] = str(inputs / 'engine')
             for bundled in ('1', '0'):
                 with self.subTest(bundled=bundled):
                     env['D4R_BUNDLE_NVIDIA'] = bundled
@@ -165,6 +170,10 @@ class OptiScalerConfigTests(unittest.TestCase):
                         self.assertNotIn('d4r/_nvngx.dll', archive.namelist())
                         self.assertEqual('d4r/ngx/_nvngx.dll' in archive.namelist(), bundled == '1')
                         self.assertEqual('d4r/nvngx_dlss.dll' in archive.namelist(), bundled == '1')
+                        engine = sorted(n for n in archive.namelist() if n.startswith('d4r/engine/'))
+                        self.assertEqual(engine, ['d4r/engine/k/' + n for n in (
+                            'README.txt', 'direct_origins.bin', 'enc0.spv', 'exposure_k0.spv', 'exposure_k1.spv',
+                            'input_k.spv', 'output_k.spv')])
                         self.assertEqual(archive.read('d4r/d4r-check.sh'), (ROOT / 'packaging/d4r-check.sh').read_bytes())
             self.assertEqual(configs[0], configs[1])
 

tests/test_shim_config.py

@@ -25,6 +25,29 @@ class ShimConfigTests(unittest.TestCase):
         self.assertEqual(int(portable[1]), 200)
         self.assertEqual(int(runtime[1]), 200)
 
+    def test_engine_is_opt_in_and_configured_by_ini(self):
+        for name in ('packaging/d4r.ini', 'config/d4r.ini.default'):
+            ini = configparser.ConfigParser(inline_comment_prefixes=(';',))
+            ini.read(ROOT / name)
+            self.assertFalse(ini.getboolean('Engine', 'Enabled'), name)
+            self.assertEqual(ini.get('Engine', 'ModelDir'), 'auto', name)
+        source = SHIM.read_text()
+        self.assertIn('portable_set("D4R_ENGINE", ini_flag(ini, "engine", "Enabled", 0) ? "1" : "0");', source)
+        self.assertIn('env_uint("D4R_ENGINE", 0)', source)
+        with tempfile.TemporaryDirectory() as temporary:
+            ini = Path(temporary) / 'd4r.ini'
+            env = {key: value for key, value in os.environ.items() if not key.startswith('D4R_')}
+            cmd = ['python3', str(ROOT / 'scripts/d4r_config.py'), '--config', str(ini)]
+            for text, enabled, model in (('', '0', None), ('[Engine]\nEnabled = true\n', '1', None),
+                                         ('[Engine]\nEnabled = true\nModelDir = /models/k\n', '1', '/models/k')):
+                ini.write_text(text)
+                result = subprocess.run(cmd, env=env, text=True, capture_output=True, check=True)
+                self.assertIn(f'export D4R_ENGINE={enabled}\n', result.stdout)
+                self.assertEqual('export D4R_ENGINE_MODEL_DIR=/models/k\n' in result.stdout, model is not None)
+            ini.write_text('[Engine]\nEnabled = maybe\n')
+            result = subprocess.run(cmd, env=env, text=True, capture_output=True)
+            self.assertEqual(result.returncode, 2)
+
     def test_launcher_marker_poll_defaults_overrides_and_validation(self):
         with tempfile.TemporaryDirectory() as temporary:
             ini = Path(temporary) / 'd4r.ini'
@@ -49,7 +72,8 @@ class ShimConfigTests(unittest.TestCase):
         # and a captured logger, so the test exercises the production message
         # and override behavior without requiring Windows, NGX or a GPU.
         source = SHIM.read_text()
-        start = source.index('    const char* presetNames[] =')
+        creation = source.index('static NgxResult create_feature(')
+        start = source.index('    const char* presetNames[] =', creation)
         end = source.index('    feature->preset = presets[1];', start)
         block = source[start:end]
         runner_source = r'''