Merge remote-tracking branch 'origin/linux' into linux
# Conflicts: # engine/rr/ptx2rr.py # install.sh # packaging/build_engine_models.sh
This commit is contained in:
rozaxlia committed
commit
c0fed5e5cb
50 files changed
+1627
-238
No files matched your search
@@ -70,6 +70,10 @@ that archive's d4r/README.txt: setup runs automatically when the ASI loads.
|
||||
@installer compatibility tools (or set D4R_PROTON=/path/to/GE-Proton11-3). It takes a minute or two once;
|
||||
@installer the models are kept in ~/.cache/d4r/engine-models for the next game. bash install.sh --help lists
|
||||
@installer options for scripted installs.
|
||||
@installer RR D/E also have separate gfx11 models; devices without compatible matrix support retain CUDA
|
||||
@installer translation. Builds report progress and time out after 900 seconds per command (override with
|
||||
@installer D4R_MODEL_TIMEOUT). Failed captures and logs are retained under ~/.cache/d4r/model-builds;
|
||||
@installer D4R_MODEL_LOG_DIR changes that location. Model checksums are verified before cache reuse.
|
||||
@installer 3. In Steam, open the game's Properties:
|
||||
@clean 3. In Steam, open the game's Properties:
|
||||
@full 2. In Steam, open the game's Properties:
|
||||
|
||||
@@ -84,8 +84,28 @@ export PATH="$KIT/bin:$PATH" # the bundled glslangValidator
|
||||
glslangValidator --version > /dev/null 2>&1 || fail "glslangValidator ($KIT/bin) does not run on this system"
|
||||
sha() { sha256sum -- "$1" | cut -d' ' -f1; }
|
||||
|
||||
WORK="$(mktemp -d "${TMPDIR:-/tmp}/d4r-models.XXXXXX")"
|
||||
trap 'kill $(jobs -p) 2> /dev/null || true; rm -rf -- "$WORK"' EXIT
|
||||
LOG_ROOT="${D4R_MODEL_LOG_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/d4r/model-builds}"
|
||||
mkdir -p -- "$LOG_ROOT"
|
||||
BUILD_LOGS="$(mktemp -d "$LOG_ROOT/build.XXXXXX")"
|
||||
WORK="$BUILD_LOGS/work"; mkdir -p "$WORK"
|
||||
MODEL_TOOLS="$KIT/model_tools.py"
|
||||
[[ -f "$MODEL_TOOLS" ]] || fail "missing $MODEL_TOOLS; update the model builder tools"
|
||||
cleanup() {
|
||||
local status=$?
|
||||
kill $(jobs -p) 2> /dev/null || true
|
||||
if (( status == 0 )); then
|
||||
while IFS= read -r -d '' log; do
|
||||
relative="${log#"$WORK/"}"
|
||||
mkdir -p -- "$BUILD_LOGS/$(dirname -- "$relative")"
|
||||
cp -- "$log" "$BUILD_LOGS/$relative"
|
||||
done < <(find "$WORK" -name '*.log' -type f -print0)
|
||||
rm -rf -- "$WORK"
|
||||
else
|
||||
printf 'build-engine-models: failed build and captures preserved at %s\n' "$WORK" >&2
|
||||
fi
|
||||
printf 'Model build diagnostics: %s\n' "$BUILD_LOGS"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
@@ -95,6 +115,7 @@ place() { # place NAME DIR FILE...
|
||||
shift 2
|
||||
mkdir -p "$RUNTIME/engine" "$WORK/place/$name"
|
||||
for f in "$@"; do cp -- "$src/$f" "$WORK/place/$name/"; done
|
||||
python3 "$MODEL_TOOLS" manifest "$WORK/place/$name"
|
||||
rm -rf -- "$dst.new"; mv -- "$WORK/place/$name" "$dst.new"
|
||||
rm -rf -- "$dst.old"; [[ ! -e "$dst" ]] || mv -- "$dst" "$dst.old"
|
||||
mv -- "$dst.new" "$dst"; rm -rf -- "$dst.old"
|
||||
@@ -106,22 +127,28 @@ model_files() { # rr.txt and the files it names, for model directory $1
|
||||
}
|
||||
run() { # run LOG COMMAND...: the command's output in WORK/LOG, shown when it fails
|
||||
local log="$WORK/$1"; shift
|
||||
"$@" > "$log" 2>&1 || { tail -n 20 "$log" >&2; fail "failed: $(basename -- "$2") (log above)"; }
|
||||
python3 "$MODEL_TOOLS" run --log "$log" --timeout "${D4R_MODEL_TIMEOUT:-900}" -- "$@" ||
|
||||
fail "failed: $1 (see $log)"
|
||||
}
|
||||
|
||||
if (( DO_RR )); then
|
||||
DLL="$RUNTIME/nvngx_dlssd.dll"
|
||||
[[ -f "$DLL" && "$(sha "$DLL")" == "$DLSSD_SHA256" ]] || fail "Ray Reconstruction models need nvngx_dlssd.dll 310.9.1 in $RUNTIME (run install.sh)"
|
||||
echo 'Building the Ray Reconstruction models (presets D, E and F) from nvngx_dlssd.dll...'
|
||||
# Build both portable arithmetic variants; runtime selects by device capabilities. Do not let the build
|
||||
# host (or an inherited developer switch) silently determine what another game/GPU receives from the cache.
|
||||
for p in D E; do
|
||||
l=$(tr DE de <<< "$p")
|
||||
bash "$KIT/engine/rr/build_native_model.sh" --dll "$DLL" "$p" "$WORK/rr-$l" > "$WORK/rr-$l.log" 2>&1 ||
|
||||
{ tail -n 20 "$WORK/rr-$l.log" >&2; fail "preset $p model failed"; }
|
||||
place "rr-$l" "$WORK/rr-$l" $(model_files "$WORK/rr-$l") exposure_rr0.spv exposure_rr1.spv
|
||||
for arithmetic in fp8 gfx11; do
|
||||
l=$(tr DE de <<< "$p"); name="rr-$l"; gfx11=0
|
||||
if [[ "$arithmetic" == gfx11 ]]; then name+="-gfx11"; gfx11=1; fi
|
||||
run "$name.log" env D4R_RR_GFX11="$gfx11" bash "$KIT/engine/rr/build_native_model.sh" \
|
||||
--dll "$DLL" "$p" "$WORK/$name"
|
||||
place "$name" "$WORK/$name" $(model_files "$WORK/$name") exposure_rr0.spv exposure_rr1.spv
|
||||
done
|
||||
done
|
||||
run ptx-f.log python3 "$KIT/kernels/tools/extract_dlss_ptx.py" "$DLL" "$WORK/ptx-f"
|
||||
run rr-f.log python3 "$KIT/engine/compile_rr2.py" --dll "$DLL" "$WORK/rr-f" --ptx-dir "$WORK/ptx-f"
|
||||
place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv
|
||||
place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv exposure_f2.spv
|
||||
fi
|
||||
|
||||
if (( DO_SR )); then
|
||||
@@ -149,16 +176,17 @@ if (( DO_SR )); then
|
||||
local w="$WORK/$1"
|
||||
mkdir -p "$w/replay"
|
||||
printf ' rendering one frame with model %s through d4r (GE-Proton %s)...\n' "$1" "$(basename "$PROTON")"
|
||||
(cd "$APP" && env STEAM_COMPAT_CLIENT_INSTALL_PATH="$STEAM_ROOT" STEAM_COMPAT_DATA_PATH="$WORK/prefix" \
|
||||
(cd "$APP" && python3 "$MODEL_TOOLS" run --log "$w/run.log" --timeout "${D4R_MODEL_TIMEOUT:-900}" -- \
|
||||
env STEAM_COMPAT_CLIENT_INSTALL_PATH="$STEAM_ROOT" STEAM_COMPAT_DATA_PATH="$WORK/prefix" \
|
||||
PROTON_FORCE_NVAPI=1 DXVK_NVAPI_GPU_ARCH=AD100 WINEDLLOVERRIDES='d3d12=n,b;d3d12core=n,b;d4r12core=n,b' \
|
||||
D4R_ENGINE=0 D4R_DLSS_PRESET="$2" D4R_HARNESS_CREATE_FLAGS="${4:-0x4B}" D4R_SHIM_LOG="$(winpath "$w/shim.log")" \
|
||||
D4R_CUDA_TRACE="$(winpath "$w/cuda.log")" D4R_CUDA_VERBOSE=1 \
|
||||
D4R_CUDA_REPLAY_DUMP_DIR="$w/replay" D4R_CUDA_REPLAY_DUMP_LIMIT=60 \
|
||||
"$PROTON/proton" run "$APP/d3d12_dlss_harness.exe" "$(winpath "$RUNTIME/nvngx.dll")" "$(winpath "$w/out.raw")" \
|
||||
1 640 360 1280 720) > "$w/run.log" 2>&1 || true
|
||||
if ! ls "$w/replay" | grep -q "$3"; then
|
||||
tail -n 15 "$w/run.log" >&2
|
||||
1 640 360 1280 720) || fail "model $1: capture process failed; diagnostics in $w"
|
||||
if ! python3 "$MODEL_TOOLS" capture "$w" "$1"; then
|
||||
[[ ! -f "$w/shim.log" ]] || tail -n 15 "$w/shim.log" >&2
|
||||
fail "model $1: the test frame did not run through DLSS (logs above)"
|
||||
fail "model $1: incomplete or invalid test frame (diagnostics in $w)"
|
||||
fi
|
||||
}
|
||||
mkdir -p "$WORK/prefix"
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded model-build commands and integrity checks (also shipped in d4r/tools)."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
MANIFEST = 'files.sha256.json'
|
||||
MODELS = ('rr-d', 'rr-e', 'rr-f', 'rr-d-gfx11', 'rr-e-gfx11', 'k', 'k-ldr', 'm', 'm-ldr', 'l', 'l-ldr')
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def manifest(directory, write=False):
|
||||
directory = Path(directory)
|
||||
files = {str(p.relative_to(directory)): digest(p) for p in sorted(directory.rglob('*'))
|
||||
if p.is_file() and p.name != MANIFEST}
|
||||
if not files or not any(n in files for n in ('model.json', 'rr.txt')):
|
||||
raise ValueError(f'{directory}: missing model metadata')
|
||||
if write:
|
||||
(directory / MANIFEST).write_text(json.dumps(files, indent=2, sort_keys=True) + '\n')
|
||||
elif files != json.loads((directory / MANIFEST).read_text()):
|
||||
raise ValueError(f'{directory}: missing, changed or unexpected model files; rebuild the models')
|
||||
|
||||
|
||||
def capture(directory, name):
|
||||
"""A completed, finite frame and all files referenced by each network-layer capture."""
|
||||
import numpy as np
|
||||
directory = Path(directory)
|
||||
output = directory / 'out.raw'
|
||||
if output.stat().st_size != 1280 * 720 * 8:
|
||||
raise ValueError('capture did not produce a complete RGBA16F frame')
|
||||
pixels = np.fromfile(output, dtype=np.float16).reshape(-1, 4)
|
||||
if not np.isfinite(pixels).all() or not np.any(pixels[:, :3]):
|
||||
raise ValueError('capture produced a nonfinite or empty frame')
|
||||
if name == 'k':
|
||||
kernels = ['dltss_pwin_' + s + '_layer' for s in
|
||||
('enc0', 'enc1', 'enc2', 'enc3', 'enc4', 'dec5', 'dec4', 'dec3', 'dec2', 'dec1', 'dec0')]
|
||||
else:
|
||||
kernels = ['rrlite_' + s + '_4x4' for s in ('enc1', 'enc2', 'enc3', 'enc4', 'enc5', 'dec4', 'dec3', 'dec2', 'dec1')]
|
||||
kernels += ['rrlite_enc0_4x4_mvlo_hdr' + ('_folded' if name == 'm' else ''),
|
||||
'rrlite_dec0_4x4' + ('_folded' if name == 'm' else '')]
|
||||
for kernel in kernels:
|
||||
matches = sorted((directory / 'replay').glob('replay-*-' + kernel))
|
||||
if not matches:
|
||||
raise ValueError(f'missing captured layer {kernel}')
|
||||
launch = matches[0]
|
||||
rows = [line.split() for line in (launch / 'manifest.txt').read_text().splitlines()]
|
||||
if ['kernel', kernel] not in rows:
|
||||
raise ValueError(f'{kernel}: incorrect capture metadata')
|
||||
args = next(int(r[1]) for r in rows if r[0] == 'args')
|
||||
if args <= 0 or (launch / 'args.bin').stat().st_size != args:
|
||||
raise ValueError(f'{kernel}: truncated arguments')
|
||||
allocations = {r[1]: int(r[3]) for r in rows if r[0] == 'alloc'}
|
||||
if not allocations:
|
||||
raise ValueError(f'{kernel}: no weight allocations captured')
|
||||
for index, size in allocations.items():
|
||||
if size <= 0 or (launch / f'alloc-{index}.bin').stat().st_size != size:
|
||||
raise ValueError(f'{kernel}: truncated allocation {index}')
|
||||
for r in rows:
|
||||
if r[0] == 'pointer' and (r[2] not in allocations or not 0 <= int(r[3]) < allocations[r[2]]):
|
||||
raise ValueError(f'{kernel}: invalid captured pointer')
|
||||
|
||||
|
||||
def run(log, timeout, command):
|
||||
if command and command[0] == '--':
|
||||
command = command[1:]
|
||||
if not command or not 0 < timeout <= 86400:
|
||||
raise ValueError('command required; timeout must be between 0 and 86400 seconds')
|
||||
log = Path(log)
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
started = time.monotonic()
|
||||
print(f' starting {log.stem} (deadline {timeout:g}s; log {log})', flush=True)
|
||||
with log.open('wb') as output:
|
||||
child = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
def stop():
|
||||
for sig in (signal.SIGTERM, signal.SIGKILL):
|
||||
try:
|
||||
os.killpg(child.pid, sig)
|
||||
except ProcessLookupError:
|
||||
break
|
||||
try:
|
||||
child.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
# Kill remaining members too, even when the group leader has exited.
|
||||
if sig == signal.SIGTERM:
|
||||
try:
|
||||
os.killpg(child.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
break
|
||||
previous = {}
|
||||
def interrupted(sig, frame):
|
||||
raise KeyboardInterrupt
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
previous[sig] = signal.signal(sig, interrupted)
|
||||
result = None
|
||||
try:
|
||||
heartbeat = 0
|
||||
while child.poll() is None:
|
||||
elapsed = time.monotonic() - started
|
||||
if elapsed >= timeout:
|
||||
stop()
|
||||
print(f' {log.stem}: timed out after {elapsed:.0f}s', file=sys.stderr)
|
||||
result = 124
|
||||
break
|
||||
if elapsed >= heartbeat + 15:
|
||||
print(f' {log.stem}: still running ({elapsed:.0f}s, pid {child.pid}); log {log}', flush=True)
|
||||
heartbeat = elapsed
|
||||
time.sleep(min(.25, max(.01, timeout - elapsed)))
|
||||
if result is None:
|
||||
result = child.returncode
|
||||
except KeyboardInterrupt:
|
||||
stop()
|
||||
result = 130
|
||||
finally:
|
||||
for sig, handler in previous.items():
|
||||
signal.signal(sig, handler)
|
||||
if result:
|
||||
print('\n'.join(log.read_text(errors='replace').splitlines()[-30:]), file=sys.stderr)
|
||||
print(f' {log.stem}: exit={result}, elapsed={time.monotonic() - started:.1f}s; log {log}', flush=True)
|
||||
return result if result >= 0 else 128 - result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest='action', required=True)
|
||||
for action in ('manifest', 'validate'):
|
||||
action_parser = sub.add_parser(action)
|
||||
action_parser.add_argument('directory', type=Path)
|
||||
if action == 'validate':
|
||||
action_parser.add_argument('models', nargs='*', choices=MODELS)
|
||||
p = sub.add_parser('capture')
|
||||
p.add_argument('directory', type=Path)
|
||||
p.add_argument('name', choices=('k', 'm', 'l'))
|
||||
p = sub.add_parser('run')
|
||||
p.add_argument('--log', required=True)
|
||||
p.add_argument('--timeout', type=float, default=900)
|
||||
p.add_argument('command', nargs=argparse.REMAINDER)
|
||||
a = parser.parse_args()
|
||||
try:
|
||||
if a.action == 'run':
|
||||
return run(a.log, a.timeout, a.command)
|
||||
if a.action == 'capture':
|
||||
capture(a.directory, a.name)
|
||||
elif a.action == 'manifest':
|
||||
manifest(a.directory, write=True)
|
||||
else:
|
||||
for name in (a.models or MODELS):
|
||||
manifest(a.directory / name)
|
||||
except (OSError, ValueError, KeyError, StopIteration) as error:
|
||||
print(f'model-tools: {error}', file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user