Add redistribution guard

scripts/check_redistributable.py refuses NVIDIA binaries (by name and
version resource), NGX SDK headers, PTX and code objects, PTX translations
(marked by ptx2glsl.py/condense.py) and the engine's model data in the
repository, and in each release variant (clean/installer/full) whatever
that variant does not contain. package_release.sh runs it before zipping;
scripts/git-hooks/pre-commit runs it locally.

Signed-off-by: woutermans <[email protected]>

packaging/RELEASE_NOTES.md

@@ -1,6 +1,6 @@
 # Unreleased
 
-- `scripts/check_redistributable.py` refuses NVIDIA's binaries, NGX SDK headers, PTX and code objects, d4r's PTX translations and the engine's model data. `scripts/package_release.sh` runs it on every release before zipping, CI runs it on every push and pull request, and `git config core.hooksPath scripts/git-hooks` runs it before each commit. `LEGAL.md` describes what d4r publishes; `CONTRIBUTING.md` requires signed-off commits.
+- `scripts/check_redistributable.py` refuses NVIDIA's binaries, NGX SDK headers, PTX and code objects, d4r's PTX translations and the engine's model data. `scripts/package_release.sh` runs it on every release before zipping, and `git config core.hooksPath scripts/git-hooks` runs it before each commit.
 - Native engine fixes: the K adapter now passes the game's exposure scale to the exposure shader (it was silently ignored); the HIP network's shared buffers are reused across feature recreation instead of leaking their ROCm mapping (21 → 6 imports over seven K features in the harness); `hipnet.bin` must belong to the preset being created; an M-only game no longer loads the K model; `scripts/install_d4r_runtime.sh` stages `engine/k-ldr` and each model's `hip/` folder.
 - Native engine, preset K: output is now byte-identical to the CUDA reference over an eleven-frame sequence at Quality (1706×960 → 2560×1440, native HIP network, RX 9070 XT), where later frames previously fell to 47–51 dB. The engine now selects NGX's reconstruction table by upscaling ratio instead of using one table for every ratio, and two rounding-order differences in the input and output stages are gone. K models must be rebuilt (`prepare_k.py`, `compile_k.py`); older packages still load and behave as before.
 - Native engine, wider game coverage (K packages must be rebuilt with the current `compile_k.py`, marker `D4RO0003`; older packages keep refusing the new cases with a logged reason):

scripts/check_redistributable.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Refuse NVIDIA property and data read from it where d4r publishes files.
+
+d4r's repository must not contain NVIDIA's software, code compiled or translated from NVIDIA's PTX, or values
+read from NVIDIA's DLSS; each release variant may contain only what scripts/package_release.sh says it contains
+(see NOTICE). This checks a file set for the rest.
+
+usage: check_redistributable.py tree [--staged] [REPO]              files git would commit (all, or the index)
+       check_redistributable.py release clean|installer|full PATH  a staged release folder or its zip
+
+The variants are those of scripts/package_release.sh's D4R_BUNDLE_NVIDIA: `clean` (0) holds nothing of NVIDIA's;
+`installer` (the default) adds texture kernels built from NVIDIA's PTX, and install.sh downloads NVIDIA's DLLs;
+`full` (1) also bundles the two DLLs. Values read from DLSS are refused in every variant.
+Exit status 1 lists every finding; 0 means none were found. This is a safeguard against mistakes, not proof
+that a file set is free of third-party rights.
+"""
+import fnmatch
+import re
+import subprocess
+import sys
+import zipfile
+from pathlib import Path, PurePosixPath
+
+# Files the native engine's tools write from NVIDIA's DLSS (engine/README.md); NOTICE names them.
+MODEL_DATA = {'weights.bin', 'offsets.bin', 'lut.bin', 'hipnet.bin', 'model.json'}
+# Compiled or extracted NVIDIA GPU code.
+NVIDIA_CODE_SUFFIXES = {'.ptx', '.cubin', '.fatbin', '.nvngx'}
+# Kernels kernels/build.sh builds from NVIDIA's PTX (tex/l families and the PTX-translated hiluma kernels).
+# kernels/build.sh k and m build only d4r's own layers (dltss_pwin_*, rrlite_enc1/enc2/enc3/dec1/dec2).
+PTX_KERNELS = ('hiluma_engine_*', 'rrlite_enc0_*', 'rrlite_dec0_*', 'rrlite_post_*', 'rrlite_downsample_*')
+# Headers of NVIDIA's NGX/DLSS SDK, which the probes in tools/ build against from the user's own copy.
+SDK_HEADERS = ('nvsdk_ngx*.h',)
+# The comment d4r's PTX translators put in the first lines of their output.
+TRANSLATION_MARKER = re.compile(rb'^// Generated by \S+ from NVIDIA PTX', re.MULTILINE)
+TRANSLATION_HEADER_LINES = 5
+NVIDIA_DLL = re.compile(r'_?nvngx.*\.dll', re.IGNORECASE)
+SHIM_RELEASE_PATH = 'd4r/nvngx.dll'
+FULL_RELEASE_NVIDIA = {'d4r/nvngx_dlss.dll', 'd4r/ngx/_nvngx.dll'}
+# Large binaries in the tree are not d4r's style (sources, patches, docs); one is most likely captured data.
+TREE_BINARY_LIMIT = 1 << 20
+COMPANY = 'CompanyName'.encode('utf-16-le')
+NVIDIA_UTF16 = 'NVIDIA'.encode('utf-16-le')
+
+
+def nvidia_pe(data):
+    """A Windows binary whose version resource names NVIDIA as its company."""
+    if not data.startswith(b'MZ'):
+        return False
+    at = data.find(COMPANY)
+    while at >= 0:
+        # VS_VERSIONINFO: the key, NUL padding to a 32-bit boundary, then the UTF-16 value.
+        if NVIDIA_UTF16 in data[at + len(COMPANY):at + len(COMPANY) + 64]:
+            return True
+        at = data.find(COMPANY, at + 1)
+    return False
+
+
+def findings_for(path, read, release=None):
+    """Problems with one file. `path` is POSIX and relative; `read()` returns its bytes."""
+    name = PurePosixPath(path).name
+    lower = name.lower()
+    suffix = PurePosixPath(lower).suffix
+    full = release == 'full'
+    problems = []
+    if lower in MODEL_DATA:
+        problems.append('engine model data read from NVIDIA\'s DLSS')
+    if suffix in NVIDIA_CODE_SUFFIXES:
+        problems.append('NVIDIA GPU code')
+    if any(fnmatch.fnmatch(lower, pattern) for pattern in SDK_HEADERS):
+        problems.append('NVIDIA SDK header')
+    if NVIDIA_DLL.fullmatch(name) and path != SHIM_RELEASE_PATH and not (full and path in FULL_RELEASE_NVIDIA):
+        problems.append('NVIDIA NGX/DLSS library')
+    if release is None and suffix in {'.hsaco', '.co', '.spv'}:
+        problems.append('compiled code object (build output)')
+    if release == 'clean' and any(fnmatch.fnmatch(lower, pattern) for pattern in PTX_KERNELS):
+        problems.append('kernel built from NVIDIA\'s PTX')
+    data = read()
+    if TRANSLATION_MARKER.search(b'\n'.join(data[:4096].split(b'\n')[:TRANSLATION_HEADER_LINES])):
+        problems.append('translation of NVIDIA PTX')
+    if nvidia_pe(data) and not (full and path in FULL_RELEASE_NVIDIA):
+        problems.append('Windows binary published by NVIDIA')
+    if release is None and len(data) > TREE_BINARY_LIMIT and b'\0' in data[:8192]:
+        problems.append(f'binary file over {TREE_BINARY_LIMIT >> 20} MiB')
+    return problems
+
+
+def git(repo, *args):
+    return subprocess.run(['git', '-C', str(repo), *args], check=True, capture_output=True).stdout
+
+
+def tree_files(repo, staged):
+    if staged:
+        names = git(repo, 'diff', '--cached', '--name-only', '--diff-filter=ACMR', '-z')
+        reader = lambda path: lambda: git(repo, 'show', f':{path}')
+    else:
+        names = git(repo, 'ls-files', '-z', '--cached', '--others', '--exclude-standard')
+        reader = lambda path: lambda: (repo / path).read_bytes()
+    for path in names.decode().split('\0'):
+        # Gitlinks (external/MinHook) and files deleted from the work tree have nothing to read.
+        if path and (staged or (repo / path).is_file()):
+            yield path, reader(path)
+
+
+def release_files(root):
+    if root.is_file():
+        archive = zipfile.ZipFile(root)
+        for info in archive.infolist():
+            if not info.is_dir():
+                yield info.filename, (lambda info=info: archive.read(info))
+        return
+    for file in sorted(root.rglob('*')):
+        if file.is_file():
+            yield file.relative_to(root).as_posix(), file.read_bytes
+
+
+def main(argv):
+    if len(argv) >= 1 and argv[0] == 'tree':
+        staged = '--staged' in argv[1:]
+        rest = [a for a in argv[1:] if a != '--staged']
+        if len(rest) > 1:
+            sys.exit(__doc__)
+        repo = Path(rest[0] if rest else '.').resolve()
+        repo = Path(git(repo, 'rev-parse', '--show-toplevel').decode().strip())
+        files, release, where = tree_files(repo, staged), None, 'the index' if staged else str(repo)
+    elif len(argv) == 3 and argv[0] == 'release' and argv[1] in ('clean', 'installer', 'full'):
+        root = Path(argv[2])
+        if not root.exists():
+            sys.exit(f'{root} does not exist')
+        files, release, where = release_files(root), argv[1], str(root)
+    else:
+        sys.exit(__doc__)
+    found = 0
+    for path, read in files:
+        for problem in findings_for(path, read, release):
+            print(f'{path}: {problem}', file=sys.stderr)
+            found += 1
+    if found:
+        print(f'{found} file(s) in {where} must not be published (see NOTICE)', file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

scripts/git-hooks/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Refuses commits that add NVIDIA's files, code built from NVIDIA's PTX or values read from DLSS
+# (scripts/check_redistributable.py; see NOTICE). Enable once per clone:
+#   git config core.hooksPath scripts/git-hooks
+exec python3 "$(git rev-parse --show-toplevel)/scripts/check_redistributable.py" tree --staged

scripts/package_release.sh

@@ -258,6 +258,10 @@ if [[ -n "${D4R_BUILD_INFO:-}" ]]; then
 fi
 sed -e "s/@VERSION@/$VERSION/g" "$ROOT/packaging/D4R_README.txt" | variant | sed 's/$/\r/' > "$STAGE/D4R_README.txt"
 
+# Last line of defence for the published zip: only what this variant contains, never values read from DLSS.
+python3 "$ROOT/scripts/check_redistributable.py" release "$VARIANT" "$STAGE" ||
+  { echo "refusing to build $NAME.zip" >&2; exit 2; }
+
 # One timestamp for every file (SOURCE_DATE_EPOCH, default the last commit): ZLUDA's kernel cache is keyed on
 # its library's size and mtime, so every extraction of the zip shares one cache.
 EPOCH="${SOURCE_DATE_EPOCH:-$(git -C "$ROOT" log -1 --format=%ct)}"

tests/test_optiscaler_packaging.py

@@ -88,7 +88,8 @@ class OptiScalerConfigTests(unittest.TestCase):
             inputs = Path(temporary) / 'inputs'
             inputs.mkdir()
             (repo / 'scripts').mkdir(parents=True)
-            for name in ('package_release.sh', 'configure_optiscaler.py', 'check_glibc_compat.sh'):
+            for name in ('package_release.sh', 'configure_optiscaler.py', 'check_glibc_compat.sh',
+                         'check_redistributable.py'):
                 shutil.copy2(ROOT / 'scripts' / name, repo / 'scripts' / name)
             shutil.copytree(ROOT / 'packaging', repo / 'packaging')
             for name in ('LICENSE', 'NOTICE'):

tests/test_redistributable.py

@@ -0,0 +1,126 @@
+"""scripts/check_redistributable.py: what may reach the repository and each release variant."""
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[1]
+CHECK = ROOT / 'scripts/check_redistributable.py'
+# The version-resource layout of an NVIDIA DLL: UTF-16 key, padding, UTF-16 value.
+NVIDIA_PE = (b'MZ' + b'\0' * 62 + 'CompanyName'.encode('utf-16-le') + b'\0\0'
+             + 'NVIDIA Corporation'.encode('utf-16-le'))
+
+
+def run(*args, cwd=None):
+    return subprocess.run([sys.executable, str(CHECK), *args], cwd=cwd, text=True, capture_output=True)
+
+
+def write(path, data=b'fixture'):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(data)
+
+
+class TreeTests(unittest.TestCase):
+    def setUp(self):
+        self.temporary = tempfile.TemporaryDirectory()
+        self.repo = Path(self.temporary.name)
+        env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
+        self.git = lambda *a: subprocess.run(['git', '-C', str(self.repo), *a], check=True, env=env,
+                                             capture_output=True)
+        self.git('init', '-q')
+        write(self.repo / 'engine/runtime.cpp', b'int main() {}\n')
+        write(self.repo / '.gitignore', b'/build/\n')
+        self.git('add', '-A')
+
+    def tearDown(self):
+        self.temporary.cleanup()
+
+    def test_ignored_files_pass_and_unignored_model_data_fails(self):
+        write(self.repo / 'build/k-model/weights.bin')
+        self.assertEqual(run('tree', str(self.repo)).returncode, 0)
+        write(self.repo / 'engine/k-model/weights.bin')
+        result = run('tree', str(self.repo))
+        self.assertEqual(result.returncode, 1)
+        self.assertIn('engine/k-model/weights.bin: engine model data', result.stderr)
+
+    def test_renamed_nvidia_binary_and_translation_are_found(self):
+        write(self.repo / 'tools/helper.bin', NVIDIA_PE)
+        write(self.repo / 'kernels/k/out.hip',
+              b'// Generated by kernels/native/ptx2hip.py from NVIDIA PTX (k); 0 roundf idioms.\n')
+        result = run('tree', str(self.repo))
+        self.assertEqual(result.returncode, 1)
+        self.assertIn('tools/helper.bin: Windows binary published by NVIDIA', result.stderr)
+        self.assertIn('kernels/k/out.hip: translation of NVIDIA PTX', result.stderr)
+
+    def test_staged_mode_checks_the_index_not_the_work_tree(self):
+        write(self.repo / 'tools/helper.dll', NVIDIA_PE)
+        self.git('add', 'tools/helper.dll')
+        write(self.repo / 'tools/helper.dll', b'replaced after staging')
+        result = run('tree', '--staged', str(self.repo))
+        self.assertEqual(result.returncode, 1)
+        self.assertIn('tools/helper.dll: Windows binary published by NVIDIA', result.stderr)
+        self.git('rm', '-q', '-f', '--cached', 'tools/helper.dll')
+        self.assertEqual(run('tree', '--staged', str(self.repo)).returncode, 0)
+
+
+class ReleaseTests(unittest.TestCase):
+    def stage(self, root, extra):
+        write(root / 'd4r/nvngx.dll')  # d4r's shim under NVIDIA's name
+        write(root / 'd4r/kernels/gfx1101/dltss_pwin_enc1_layer.hsaco')
+        write(root / 'd4r/kernels/gfx1101/rrlite_enc1_4x4.hsaco')
+        write(root / 'd4r/engine/k/input_k.spv')
+        for name, data in extra.items():
+            write(root / name, data)
+
+    def check(self, variant, extra, archive=False):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary) / 'stage'
+            self.stage(root, extra)
+            target = root
+            if archive:
+                target = Path(temporary) / 'release.zip'
+                with zipfile.ZipFile(target, 'w') as output:
+                    for file in root.rglob('*'):
+                        output.write(file, file.relative_to(root).as_posix())
+            return run('release', variant, str(target))
+
+    def test_own_files_pass_every_variant(self):
+        for variant in ('clean', 'installer', 'full'):
+            for archive in (False, True):
+                with self.subTest(variant=variant, archive=archive):
+                    result = self.check(variant, {}, archive)
+                    self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_each_variant_holds_only_its_nvidia_files(self):
+        dlls = {'d4r/nvngx_dlss.dll': NVIDIA_PE, 'd4r/ngx/_nvngx.dll': NVIDIA_PE}
+        kernels = {'d4r/kernels/gfx1101/rrlite_post_3_1_mvlo_hdr.hsaco': b'code',
+                   'd4r/kernels/gfx1101/hiluma_engine_input_depthinv_mvlo_hdr_v2_rel.hsaco': b'code'}
+        for archive in (False, True):
+            with self.subTest(archive=archive):
+                clean = self.check('clean', {**dlls, **kernels}, archive)
+                self.assertEqual(clean.returncode, 1)
+                for name in {**dlls, **kernels}:
+                    self.assertIn(name + ':', clean.stderr)
+                self.assertEqual(self.check('installer', kernels, archive).returncode, 0)
+                installer = self.check('installer', dlls, archive)
+                self.assertEqual(installer.returncode, 1)
+                for name in dlls:
+                    self.assertIn(name + ':', installer.stderr)
+                self.assertEqual(self.check('full', {**dlls, **kernels}, archive).returncode, 0)
+
+    def test_nvidia_dll_elsewhere_and_dlss_values_fail_every_variant(self):
+        refused = {'d4r/extra/nvngx_dlss.dll': NVIDIA_PE, 'd4r/engine/k/weights.bin': b'w',
+                   'd4r/engine/k/model.json': b'{}', 'd4r/kernels/gfx1101/layer.ptx': b'.version 8.7'}
+        for variant in ('clean', 'installer', 'full'):
+            with self.subTest(variant=variant):
+                result = self.check(variant, refused)
+                self.assertEqual(result.returncode, 1)
+                for name in refused:
+                    self.assertIn(name + ':', result.stderr)
+
+
+if __name__ == '__main__':
+    unittest.main()