ayo
f71a483a00
Add default-off PreferAccuracy with conservative native variants for every RDNA3/RDNA4 target and a separate JIT cache. Preserve fast defaults and reject mixed accuracy sets. Add a pinned Debian build environment, rebuild all release components against GLIBC 2.41, and include toolchain provenance and release notes for the accuracy changes and the preceding artifact/synchronization fixes. Refs #6; CPU-overhead confirmation for #7 remains pending.
233 lines
13 KiB
Bash
Executable File
233 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Builds the drag-in release: a zip whose contents are extracted into the folder that holds a game's
|
|
# main .exe, like an OptiScaler release. packaging/D4R_README.txt describes the result.
|
|
#
|
|
# usage: scripts/package_release.sh [OUT_DIR] (default: dist/)
|
|
#
|
|
# Inputs (environment):
|
|
# D4R_OPTISCALER OptiScaler release archive (.7z or .zip) or extracted folder (required; 0.9.4 tested)
|
|
# D4R_DLSS_DLLS colon-separated nvngx_dlss.dll files whose kernel code the native kernels accept
|
|
# (required; only hashes of their PTX go into the release)
|
|
# D4R_ZLUDA_DIR ZLUDA build with patches/zluda applied (libnvcuda.so) default ~/.cache/d4r-zluda-current
|
|
# D4R_VKD3D_DIR d4r-patched vkd3d-proton (d3d12.dll, d3d12core.dll) default ~/.cache/d4r-vkd3d-d4r
|
|
# D4R_ROCM_DIR ROCm with clang, for the kernels default /opt/rocm
|
|
# D4R_ROCM_RUNTIME ROCm runtime bundled as d4r/rocm (scripts/fetch_rocm_runtime.sh)
|
|
# default ~/.cache/d4r-rocm-runtime
|
|
# D4R_MAX_GLIBC highest GLIBC ABI the Linux libraries may require default 2.41
|
|
# D4R_GPU_ARCHS GPU targets to build kernels for default gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201
|
|
# (RDNA4 targets also get a <target>-fp8 folder: the kernels for d4r.ini NativeFp8 = on)
|
|
# D4R_OPTISCALER_LICENSE OptiScaler's LICENSE (GPL-3.0) text; default: the system's SPDX copy
|
|
# D4R_VKD3D_SRC vkd3d-proton source checkout, for its license files default ~/.cache/d4r-vkd3d-proton
|
|
# D4R_ZLUDA_SRC ZLUDA source checkout, for its license files default: D4R_ZLUDA_DIR's ../d4r-zluda-upstream
|
|
# SOURCE_DATE_EPOCH timestamp given to every packaged file default: the last commit's
|
|
# D4R_BUILD_INFO optional build-toolchain/provenance text to include in d4r/source/
|
|
# D4R_PACKAGE_KERNEL_JOBS independent target builds at once (default 1)
|
|
# The zip also contains NVIDIA's files and the texture kernels built from NVIDIA's PTX; redistributing
|
|
# those is up to whoever publishes it (they are not covered by d4r's license):
|
|
# D4R_BUNDLE_DLSS nvngx_dlss.dll to include D4R_BUNDLE_NGX _nvngx.dll to include
|
|
# D4R_BUNDLE_TEX directory with texture-kernel code objects (kernels/build.sh tex), one subdirectory
|
|
# per target folder (gfx1101, gfx1201, gfx1201-fp8, ...), or a flat gfx1101 directory for
|
|
# older builds
|
|
# Accuracy texture sets go in accuracy/<target>/, built with D4R_PREFER_ACCURACY=1.
|
|
# D4R_ZLUDA_EMIT d4r_emit from the patched ZLUDA build; required in full builds when an accuracy
|
|
# texture set is not supplied in D4R_BUNDLE_TEX/accuracy/<target>.
|
|
# D4R_BUNDLE_NVIDIA=0 leaves them out (d4r-VERSION-nonvidia.zip; users then add the two DLLs themselves).
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
OUT="$(realpath -m "${1:-$ROOT/dist}")"
|
|
VERSION="$(cat "$ROOT/packaging/VERSION")"
|
|
VARIANT=full
|
|
[[ "${D4R_BUNDLE_NVIDIA:-1}" == 0 ]] && VARIANT=clean
|
|
NAME="d4r-$VERSION"
|
|
[[ "$VARIANT" == clean ]] && NAME="$NAME-nonvidia"
|
|
STAGE="$OUT/$NAME"
|
|
ZLUDA="${D4R_ZLUDA_DIR:-$HOME/.cache/d4r-zluda-current}"
|
|
VKD3D="${D4R_VKD3D_DIR:-$HOME/.cache/d4r-vkd3d-d4r}"
|
|
ROCM="${D4R_ROCM_DIR:-/opt/rocm}"
|
|
ROCM_RUNTIME="${D4R_ROCM_RUNTIME:-$HOME/.cache/d4r-rocm-runtime}"
|
|
ARCHS="${D4R_GPU_ARCHS:-gfx1100 gfx1101 gfx1102 gfx1103 gfx1200 gfx1201}"
|
|
: "${D4R_OPTISCALER:?set D4R_OPTISCALER to the OptiScaler release archive or folder}"
|
|
: "${D4R_DLSS_DLLS:?set D4R_DLSS_DLLS to the nvngx_dlss.dll files the kernel manifest accepts}"
|
|
for f in "$ZLUDA/libnvcuda.so" "$VKD3D/d3d12.dll" "$VKD3D/d3d12core.dll" "$ROCM_RUNTIME/lib/libamdhip64.so.7"; do
|
|
[[ -f "$f" ]] || { echo "missing $f (the ROCm runtime comes from scripts/fetch_rocm_runtime.sh)" >&2; exit 2; }
|
|
done
|
|
|
|
# D4R_SKIP_BUILD=1 packages the shim and bridge already in build/ (e.g. the binaries that were tested)
|
|
if [[ "${D4R_SKIP_BUILD:-0}" != 1 ]]; then
|
|
"$ROOT/scripts/build_d4r_nvngx_shim.sh" >/dev/null
|
|
"$ROOT/scripts/build_wine_nvcuda_bridge.sh" >/dev/null
|
|
fi
|
|
|
|
rm -rf "$STAGE"
|
|
mkdir -p "$STAGE/d4r/zluda" "$STAGE/d4r/rocm" "$STAGE/d4r/ngx" "$STAGE/d4r/kernels" "$STAGE/d4r/licenses" "$STAGE/d4r/source"
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
|
|
# OptiScaler, as dxgi.dll with its INI preconfigured for d4r
|
|
if [[ -d "$D4R_OPTISCALER" ]]; then
|
|
OPTI="$D4R_OPTISCALER"
|
|
else
|
|
OPTI="$TMP/optiscaler"
|
|
mkdir -p "$OPTI"
|
|
case "$D4R_OPTISCALER" in
|
|
*.7z) 7z x -y -o"$OPTI" "$D4R_OPTISCALER" >/dev/null ;;
|
|
*.zip) unzip -q -o "$D4R_OPTISCALER" -d "$OPTI" ;;
|
|
*) echo "D4R_OPTISCALER must be a .7z, a .zip or a folder" >&2; exit 2 ;;
|
|
esac
|
|
fi
|
|
cp "$OPTI/OptiScaler.dll" "$STAGE/dxgi.dll"
|
|
python3 - "$OPTI/OptiScaler.ini" "$ROOT/packaging/optiscaler.settings" "$STAGE/OptiScaler.ini" <<'PY'
|
|
import re, sys
|
|
lines = open(sys.argv[1], encoding="utf-8", newline="").read().split("\n") # keeps the file's line endings
|
|
for setting in open(sys.argv[2], encoding="utf-8"):
|
|
setting = setting.strip()
|
|
if not setting or setting.startswith("#"):
|
|
continue
|
|
path, value = setting.split("=", 1)
|
|
section, key = path.split(".", 1)
|
|
start = next((i for i, l in enumerate(lines) if l.strip() == f"[{section}]"), None)
|
|
if start is None:
|
|
sys.exit(f"OptiScaler.ini has no [{section}]")
|
|
end = next((i for i in range(start + 1, len(lines)) if lines[i].startswith("[")), len(lines))
|
|
for i in range(start + 1, end):
|
|
if re.match(rf"{re.escape(key)}\s*=", lines[i]):
|
|
lines[i] = f"{key}={value}" + ("\r" if lines[i].endswith("\r") else "")
|
|
break
|
|
else:
|
|
sys.exit(f"OptiScaler.ini has no {key} in [{section}]")
|
|
open(sys.argv[3], "w", encoding="utf-8", newline="").write("\n".join(lines))
|
|
PY
|
|
|
|
# d4r-patched vkd3d-proton: same-frame DLSS results
|
|
cp "$VKD3D/d3d12.dll" "$VKD3D/d3d12core.dll" "$STAGE/"
|
|
|
|
# d4r itself
|
|
cp "$ROOT/build/d4r_nvngx.dll" "$STAGE/d4r/nvngx.dll"
|
|
cp "$ROOT/build/wine-nvcuda/x86_64-unix/nvcuda.dll.so" "$STAGE/d4r/nvcuda.dll"
|
|
cp "$ZLUDA/libnvcuda.so" "$STAGE/d4r/zluda/libcuda.so"
|
|
cp -r "$ROCM_RUNTIME/lib" "$STAGE/d4r/rocm/lib"
|
|
"$ROOT/scripts/check_glibc_compat.sh" "${D4R_MAX_GLIBC:-2.41}" \
|
|
"$STAGE/d4r/nvcuda.dll" "$STAGE/d4r/zluda/libcuda.so" "$STAGE/d4r/rocm/lib/"*
|
|
cp "$ROOT/packaging/d4r.ini" "$STAGE/d4r/d4r.ini"
|
|
cp "$ROOT/packaging/d4r-check.sh" "$STAGE/d4r/d4r-check.sh"
|
|
if [[ "$VARIANT" == full ]]; then
|
|
: "${D4R_BUNDLE_DLSS:?set D4R_BUNDLE_DLSS}" "${D4R_BUNDLE_NGX:?set D4R_BUNDLE_NGX}" "${D4R_BUNDLE_TEX:?set D4R_BUNDLE_TEX}"
|
|
cp "$D4R_BUNDLE_DLSS" "$STAGE/d4r/nvngx_dlss.dll"
|
|
cp "$D4R_BUNDLE_NGX" "$STAGE/d4r/ngx/_nvngx.dll"
|
|
else
|
|
printf 'Put NVIDIA'"'"'s NGX runtime, _nvngx.dll, in this folder (see D4R_README.txt).\r\n' > "$STAGE/d4r/ngx/README.txt"
|
|
fi
|
|
IFS=: read -r -a DLLS <<< "$D4R_DLSS_DLLS"
|
|
# one folder per target; RDNA4 targets also get <target>-fp8 (native FP8 WMMA, d4r.ini NativeFp8), which the
|
|
# bridge serves instead when that setting is on
|
|
folders=()
|
|
for arch in $ARCHS; do
|
|
folders+=("$arch")
|
|
[[ "$arch" == gfx12* ]] && folders+=("$arch-fp8")
|
|
done
|
|
build_target() {
|
|
local folder="$1" arch fp8 tex_dir accurate accurate_tex f
|
|
arch="${folder%-fp8}"
|
|
fp8=0
|
|
[[ "$folder" == *-fp8 ]] && fp8=1
|
|
D4R_PREFER_ACCURACY=0 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" k "$STAGE/d4r/kernels/$folder" >/dev/null
|
|
D4R_PREFER_ACCURACY=0 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" m "$STAGE/d4r/kernels/$folder" >/dev/null
|
|
rm -f "$STAGE/d4r/kernels/$folder"/*.resolution.txt # empty LTO notes from clang's -save-temps
|
|
if [[ "$VARIANT" == full ]]; then
|
|
tex_dir=""
|
|
if [[ -d "$D4R_BUNDLE_TEX/$folder" ]]; then
|
|
tex_dir="$D4R_BUNDLE_TEX/$folder"
|
|
elif [[ "$folder" == gfx1101 ]]; then
|
|
tex_dir="$D4R_BUNDLE_TEX" # older, flat gfx1101 texture-kernel builds
|
|
fi
|
|
if [[ -n "$tex_dir" ]]; then
|
|
for f in "$tex_dir"/*.hsaco; do # texture kernels only; the layers above are built from source
|
|
[[ -f "$f" ]] || continue
|
|
[[ -e "$STAGE/d4r/kernels/$folder/$(basename "$f")" ]] || cp "$f" "$STAGE/d4r/kernels/$folder/"
|
|
done
|
|
fi
|
|
fi
|
|
python3 "$ROOT/kernels/tools/kernel_manifest.py" "$STAGE/d4r/kernels/$folder" "${DLLS[@]}"
|
|
|
|
# Every release target also gets conservative network and texture variants. Do not copy fast
|
|
# texture objects into this set: their compiler policy is fixed inside the binary.
|
|
accurate="$STAGE/d4r/kernels/accuracy/$folder"
|
|
D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" k "$accurate" >/dev/null
|
|
D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" "$ROOT/kernels/build.sh" m "$accurate" >/dev/null
|
|
if [[ "$VARIANT" == full ]]; then
|
|
accurate_tex="$D4R_BUNDLE_TEX/accuracy/$folder"
|
|
if [[ -f "$accurate_tex/d4r-accuracy.txt" && "$(cat "$accurate_tex/d4r-accuracy.txt")" == 1 ]]; then
|
|
for f in "$accurate_tex"/*.hsaco; do
|
|
[[ -f "$f" ]] || continue
|
|
[[ -e "$accurate/$(basename "$f")" ]] || cp "$f" "$accurate/"
|
|
done
|
|
else
|
|
: "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT or supply D4R_BUNDLE_TEX/accuracy/$folder}"
|
|
D4R_PREFER_ACCURACY=1 D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$arch" D4R_NATIVE_FP8="$fp8" \
|
|
D4R_DLSS_DLL="$D4R_BUNDLE_DLSS" "$ROOT/kernels/build.sh" tex "$accurate" >/dev/null
|
|
fi
|
|
fi
|
|
if [[ "$VARIANT" == full ]]; then
|
|
for f in "$STAGE/d4r/kernels/$folder"/*.hsaco; do
|
|
[[ -f "$accurate/$(basename "$f")" ]] || {
|
|
echo "missing accuracy variant of $(basename "$f") for $folder" >&2; exit 2;
|
|
}
|
|
done
|
|
fi
|
|
rm -f "$accurate"/*.resolution.txt
|
|
python3 "$ROOT/kernels/tools/kernel_manifest.py" "$accurate" "${DLLS[@]}"
|
|
}
|
|
kernel_jobs="${D4R_PACKAGE_KERNEL_JOBS:-1}"
|
|
[[ "$kernel_jobs" =~ ^[1-9][0-9]*$ ]] || { echo "D4R_PACKAGE_KERNEL_JOBS must be positive" >&2; exit 2; }
|
|
pending=()
|
|
kernel_failed=0
|
|
for folder in "${folders[@]}"; do
|
|
build_target "$folder" &
|
|
pending+=("$!")
|
|
if (( ${#pending[@]} >= kernel_jobs )); then
|
|
wait "${pending[0]}" || kernel_failed=1
|
|
pending=("${pending[@]:1}")
|
|
fi
|
|
done
|
|
for pid in "${pending[@]}"; do wait "$pid" || kernel_failed=1; done
|
|
[[ "$kernel_failed" == 0 ]] || { echo "native target build failed; no release ZIP created" >&2; exit 1; }
|
|
|
|
# licenses and sources
|
|
ZLUDA_SRC="${D4R_ZLUDA_SRC:-$(dirname "$ZLUDA")/d4r-zluda-upstream}"
|
|
VKD3D_SRC="${D4R_VKD3D_SRC:-$HOME/.cache/d4r-vkd3d-proton}"
|
|
L="$STAGE/d4r/licenses"
|
|
cp "$ROOT/LICENSE" "$L/d4r-LICENSE.txt"
|
|
cp "$ROOT/NOTICE" "$L/d4r-NOTICE.txt"
|
|
cp "$ZLUDA_SRC/LICENSE-APACHE" "$L/ZLUDA-LICENSE-APACHE.txt"
|
|
cp "$ZLUDA_SRC/LICENSE-MIT" "$L/ZLUDA-LICENSE-MIT.txt"
|
|
cp "$ZLUDA_SRC/ext/llvm-project/llvm/LICENSE.TXT" "$L/LLVM-LICENSE.txt"
|
|
cp "$VKD3D_SRC/LICENSE" "$L/vkd3d-proton-LICENSE.txt"
|
|
cp "$VKD3D_SRC/COPYING" "$L/vkd3d-proton-COPYING.txt"
|
|
cp "${D4R_OPTISCALER_LICENSE:-/usr/share/licenses/spdx/GPL-3.0-only.txt}" "$L/OptiScaler-LICENSE-GPL-3.0.txt"
|
|
for f in "$ROCM_RUNTIME"/licenses/*; do cp "$f" "$L/ROCm-$(basename "$f")"; done
|
|
mkdir -p "$STAGE/d4r/source/patches"
|
|
cp -r "$ROOT/patches/zluda" "$ROOT/patches/vkd3d-proton" "$STAGE/d4r/source/patches/"
|
|
ZLUDA_COMMIT="$(git -C "$ZLUDA_SRC" rev-parse HEAD 2>/dev/null || echo unknown)"
|
|
VKD3D_COMMIT="$(git -C "$VKD3D_SRC" rev-parse HEAD 2>/dev/null || echo unknown)"
|
|
# "@clean " / "@full " lines belong to one variant only
|
|
variant() { sed -n -e "s/^@$VARIANT //" -e '/^@[a-z]* /d' -e p; }
|
|
file_version() { [[ -f "$1" ]] && strings -el "$1" | grep -A1 '^FileVersion$' | sed -n 2p | tr ',' '.' | tr -d ' '; }
|
|
sed -e "s/@VERSION@/$VERSION/g" -e "s/@ZLUDA_COMMIT@/$ZLUDA_COMMIT/g" -e "s/@VKD3D_COMMIT@/$VKD3D_COMMIT/g" \
|
|
-e "s/@DLSS_VERSION@/$(file_version "$STAGE/d4r/nvngx_dlss.dll")/g" -e "s/@NGX_VERSION@/$(file_version "$STAGE/d4r/ngx/_nvngx.dll")/g" \
|
|
"$ROOT/packaging/SOURCES.txt" | variant > "$STAGE/d4r/source/SOURCES.txt"
|
|
if [[ -n "${D4R_BUILD_INFO:-}" ]]; then
|
|
cp "$D4R_BUILD_INFO" "$STAGE/d4r/source/BUILD_INFO.txt"
|
|
fi
|
|
sed -e "s/@VERSION@/$VERSION/g" "$ROOT/packaging/D4R_README.txt" | variant | sed 's/$/\r/' > "$STAGE/D4R_README.txt"
|
|
|
|
# One timestamp for every file (SOURCE_DATE_EPOCH, default the last commit): ZLUDA's kernel cache is keyed on
|
|
# its library's size and mtime, so every extraction of the zip shares one cache.
|
|
EPOCH="${SOURCE_DATE_EPOCH:-$(git -C "$ROOT" log -1 --format=%ct)}"
|
|
find "$STAGE" -exec touch -h -d "@$EPOCH" {} +
|
|
# The zip holds the folder's contents, so it can be extracted straight into the game folder.
|
|
rm -f "$OUT/$NAME.zip"
|
|
(cd "$STAGE" && find . -type f | LC_ALL=C sort | sed 's|^\./||' | zip -q -X -9 "$OUT/$NAME.zip" -@)
|
|
(cd "$OUT" && sha256sum "$NAME.zip" > "$NAME.zip.sha256")
|
|
printf 'Built %s (%s)\n' "$OUT/$NAME.zip" "$(du -h "$OUT/$NAME.zip" | cut -f1)"
|