Add accuracy mode and GLIBC 2.41 release builds
Add default-off PreferAccuracy with conservative native variants for every RDNA3/RDNA4 target and a separate JIT cache. Preserve fast defaults and reject mixed accuracy sets. Add a pinned Debian build environment, rebuild all release components against GLIBC 2.41, and include toolchain provenance and release notes for the accuracy changes and the preceding artifact/synchronization fixes. Refs #6; CPU-overhead confirmation for #7 remains pending.
This commit is contained in:
ayo committed
1 parent
b2b3781aca
commit
f71a483a00
25 files changed
+641
-105
No files matched your search
+26
-4
@@ -10,15 +10,22 @@
|
||||
# D4R_NATIVE_FP8 1 (RDNA4 only): the variant for ZLUDA's native FP8 WMMA (d4r.ini [Kernels] NativeFp8):
|
||||
# M's layers on FP8 WMMA with NVIDIA's e4m3 activations, texture tails matching ZLUDA's
|
||||
# D4R_ZLUDA_WMMA_FP8_NATIVE lowering. K is the same in both variants.
|
||||
# D4R_PREFER_ACCURACY 1: preserve FP8 quantization, per-MMA f16 rounding and denormal handling;
|
||||
# build a separate set (default output: kernels/out/native/accuracy).
|
||||
# Texture kernels (tex) are NVIDIA's PTX with parts replaced, so they additionally need:
|
||||
# D4R_DLSS_DLL nvngx_dlss.dll 310.7 (its PTX is extracted into kernels/extracted/, never committed)
|
||||
# D4R_ZLUDA_EMIT ZLUDA's d4r_emit (patches/zluda applied; `cargo build --release -p ptx --example d4r_emit`),
|
||||
# which compiles PTX offline for D4R_GPU_ARCH (no GPU of that kind needed)
|
||||
# D4R_DLSS_PTX_DIR optional pre-extracted PTX directory for parallel per-target builds
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WHAT="${1:-all}"
|
||||
OUT="$(realpath -m "${2:-$HERE/out/native}")"
|
||||
ACCURACY="${D4R_PREFER_ACCURACY:-0}"
|
||||
[[ "$ACCURACY" == 0 || "$ACCURACY" == 1 ]] || { echo "D4R_PREFER_ACCURACY must be 0 or 1" >&2; exit 2; }
|
||||
DEFAULT_OUT="$HERE/out/native"
|
||||
[[ "$ACCURACY" == 1 ]] && DEFAULT_OUT+="/accuracy"
|
||||
OUT="$(realpath -m "${2:-$DEFAULT_OUT}")"
|
||||
ROCM="${D4R_ROCM_DIR:-/opt/rocm}"
|
||||
ARCH="${D4R_GPU_ARCH:-gfx1101}"
|
||||
FP8="${D4R_NATIVE_FP8:-0}"
|
||||
@@ -30,6 +37,14 @@ esac
|
||||
CLANG="$ROCM/lib/llvm/bin/clang++"
|
||||
[[ -x "$CLANG" ]] || { echo "clang++ not found in $ROCM/lib/llvm/bin (set D4R_ROCM_DIR)" >&2; exit 2; }
|
||||
mkdir -p "$OUT"
|
||||
case "$WHAT" in all|k|m|tex) ;; *) echo "unknown kernel family: $WHAT" >&2; exit 2 ;; esac
|
||||
# Never certify a directory containing older fast binaries as an accuracy set.
|
||||
if [[ "$ACCURACY" == 1 ]] && compgen -G "$OUT/*.hsaco" >/dev/null &&
|
||||
{ [[ ! -f "$OUT/d4r-accuracy.txt" ]] || [[ "$(cat "$OUT/d4r-accuracy.txt")" != 1 ]]; }; then
|
||||
echo "accuracy kernels need an empty output directory or an existing accuracy set: $OUT" >&2
|
||||
exit 2
|
||||
fi
|
||||
rm -f "$OUT/d4r-accuracy.txt"
|
||||
|
||||
# HIP source -> raw code object (the kernel name inside matches the DLSS kernel it replaces)
|
||||
build_hip() {
|
||||
@@ -39,6 +54,7 @@ build_hip() {
|
||||
tmp="$(mktemp -d)"
|
||||
# per-kernel compiler flags from a "// d4r-build-flags: ..." line in the source
|
||||
flags="$(sed -n 's|^// d4r-build-flags: *||p' "$src")"
|
||||
[[ "$ACCURACY" == 1 ]] && extra+=" -DD4R_ACCURACY"
|
||||
# shellcheck disable=SC2086
|
||||
(cd "$tmp" && "$CLANG" -x hip --offload-arch="$ARCH" --offload-device-only -O3 $flags $extra -I"$ROCM/include" \
|
||||
--rocm-path="$ROCM" --rocm-device-lib-path="$ROCM/amdgcn/bitcode" -I"$(dirname "$src")" \
|
||||
@@ -59,8 +75,12 @@ if [[ "$WHAT" == all || "$WHAT" == tex ]]; then
|
||||
echo "== texture kernels (NVIDIA PTX with native parts)"
|
||||
: "${D4R_DLSS_DLL:?set D4R_DLSS_DLL to nvngx_dlss.dll (310.7)}"
|
||||
: "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT to d4r_emit from a ZLUDA build with patches/zluda}"
|
||||
PTX_DIR="$HERE/extracted/ptx"
|
||||
python3 "$HERE/tools/extract_dlss_ptx.py" "$D4R_DLSS_DLL" "$PTX_DIR"
|
||||
PTX_DIR="${D4R_DLSS_PTX_DIR:-$HERE/extracted/ptx}"
|
||||
if [[ -z "${D4R_DLSS_PTX_DIR:-}" ]]; then
|
||||
python3 "$HERE/tools/extract_dlss_ptx.py" "$D4R_DLSS_DLL" "$PTX_DIR"
|
||||
elif [[ ! -d "$PTX_DIR" ]]; then
|
||||
echo "missing extracted PTX directory: $PTX_DIR" >&2; exit 2
|
||||
fi
|
||||
# kernel : source of its native part [: w64 = compiled as wave64 (two CUDA warps per wave; RDNA3 issues
|
||||
# FP32 work for all 64 lanes at once). Only for kernels without MMAs; measured faster, bit-identical:
|
||||
# post 0.89 -> 0.77 ms, hiluma output 0.905 -> 0.887 ms (downsample was slower as wave64).]
|
||||
@@ -84,8 +104,10 @@ if [[ "$WHAT" == all || "$WHAT" == tex ]]; then
|
||||
done
|
||||
for spec in "${specs[@]}"; do
|
||||
IFS=: read -r kernel src mode <<< "$spec"
|
||||
D4R_ZLUDA_WAVE64="$([[ "$mode" == w64 ]] && echo 1 || echo 0)" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" \
|
||||
D4R_PREFER_ACCURACY="$ACCURACY" \
|
||||
D4R_ZLUDA_WAVE64="$([[ "$mode" == w64 && "$ACCURACY" == 0 ]] && echo 1 || echo 0)" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" \
|
||||
D4R_TEX_FP8="$FP8" D4R_DLSS_PTX_DIR="$PTX_DIR" "$HERE/tex/build_tex.sh" "$kernel" "$src" "$OUT"
|
||||
done
|
||||
fi
|
||||
[[ "$ACCURACY" == 1 ]] && printf '1\n' > "$OUT/d4r-accuracy.txt"
|
||||
echo "native kernels in $OUT"
|
||||
@@ -10,6 +10,9 @@
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include "../common/wmma_layout.h"
|
||||
#if defined(D4R_ACCURACY) && defined(PWIN_F32ACC)
|
||||
#undef PWIN_F32ACC
|
||||
#endif
|
||||
#pragma clang fp contract(off)
|
||||
|
||||
typedef _Float16 half_t;
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include "../common/wmma_layout.h"
|
||||
#ifdef D4R_ACCURACY
|
||||
#define SWIN_EXACT
|
||||
#endif
|
||||
#if defined(D4R_FP8_WMMA) && D4R_WMMA_LAYOUT != 12 && defined(__AMDGCN__)
|
||||
#error "D4R_FP8_WMMA needs the gfx12 WMMA layout"
|
||||
#endif
|
||||
|
||||
@@ -11,12 +11,14 @@ D="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
R="${D4R_ROCM_DIR:-/opt/rocm}"
|
||||
ARCH="${D4R_GPU_ARCH:-gfx1101}"
|
||||
FP8="${D4R_TEX_FP8:-0}"
|
||||
ACCURACY="${D4R_PREFER_ACCURACY:-0}"
|
||||
LLVM="$R/lib/llvm/bin"
|
||||
B="$(mktemp -d)"
|
||||
trap 'rm -rf "$B"' EXIT
|
||||
# D4R_TEX_CFLAGS: extra flags for SRC.hip (tests, e.g. -DD4R_WMMA_LAYOUT=12 for the gfx12-layout shim on gfx11)
|
||||
read -r -a FLAGS <<< "${D4R_TEX_CFLAGS:-}"
|
||||
[[ "$FP8" == 1 ]] && FLAGS+=(-DD4R_TEX_FP8)
|
||||
[[ "$ACCURACY" == 1 ]] && FLAGS+=(-DD4R_ACCURACY)
|
||||
# SRC.hip -> bitcode without target attributes, so ZLUDA's own target settings apply after linking
|
||||
"$LLVM/clang++" -x hip -std=c++20 -nogpuinc -nogpulib -O3 -mno-wavefrontsize64 --offload-device-only \
|
||||
--offload-arch="$ARCH" -fgpu-rdc -emit-llvm -c -Xclang -fdenormal-fp-math=dynamic -DD4R_KERNEL_NAME="$K" \
|
||||
@@ -24,10 +26,12 @@ read -r -a FLAGS <<< "${D4R_TEX_CFLAGS:-}"
|
||||
"$LLVM/llvm-dis" "$B/raw.bc" -o "$B/raw.ll"
|
||||
sed -E -e '/@llvm.used/d' -e '/wchar_size/d' -e '/llvm.module.flags/d' -e '/__hip_cuid/d' -e 's/optnone//g' \
|
||||
-e "s/\"target-cpu\"=\"$ARCH\"//g" -e 's/"target-features"="[^"]+"//g' "$B/raw.ll" | "$LLVM/llvm-as" -o "$B/extra.bc" -
|
||||
python3 "$D/make_ptx.py" "$K" "$B/$K.ptx" >/dev/null
|
||||
D4R_PREFER_ACCURACY="$ACCURACY" python3 "$D/make_ptx.py" "$K" "$B/$K.ptx" >/dev/null
|
||||
# the code-generation switches of the runtime (d4r.ini [Kernels]), which ZLUDA's module cache keys on
|
||||
env LD_LIBRARY_PATH="$R/lib" D4R_ZLUDA_EXTRA_BC="$B/extra.bc" D4R_ZLUDA_WMMA=1 D4R_ZLUDA_WMMA_FP8=1 \
|
||||
D4R_ZLUDA_WMMA_FP8_NATIVE="$FP8" D4R_ZLUDA_IGNORE_DENORMAL=1 D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 \
|
||||
D4R_ZLUDA_WAVE64="${D4R_ZLUDA_WAVE64:-0}" "$D4R_ZLUDA_EMIT" "$B/$K.ptx" "$B/dump" "$ARCH" >/dev/null
|
||||
D4R_ZLUDA_WMMA_FP8_NATIVE="$FP8" D4R_ZLUDA_IGNORE_DENORMAL="$((1 - ACCURACY))" \
|
||||
D4R_ZLUDA_WMMA_F32ACC=0 D4R_ZLUDA_FAST_MATH=0 D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 \
|
||||
D4R_ZLUDA_WAVE64="$([[ "$ACCURACY" == 1 ]] && echo 0 || echo "${D4R_ZLUDA_WAVE64:-0}")" \
|
||||
"$D4R_ZLUDA_EMIT" "$B/$K.ptx" "$B/dump" "$ARCH" >/dev/null
|
||||
cp "$B/dump/module.hsaco" "$OUT/$K.hsaco"
|
||||
printf '%-52s %s\n' "$K" "$(grep -E '^\s+\.(vgpr_count|private_segment_fixed_size|wavefront_size):' "$B/dump/asm.s" | tr -s ' ' | paste -sd' ')"
|
||||
@@ -14,7 +14,7 @@
|
||||
// Kernels that ZLUDA compiles in strict-FP mode (constrained intrinsics) cannot take inlined FP code from
|
||||
// here (the AMDGPU backend fails on some constrained operations): build them with -DD4R_SUST_OUTLINE,
|
||||
// which keeps the FP bodies of the surface stores in noinline leaf functions (no calls, no stack).
|
||||
#ifdef D4R_SUST_OUTLINE
|
||||
#if defined(D4R_SUST_OUTLINE) || defined(D4R_ACCURACY)
|
||||
#define FP_BODY __attribute__((device, noinline)) static
|
||||
#else
|
||||
#define FP_BODY __attribute__((device, always_inline)) static inline
|
||||
|
||||
Reference in new issue
Block a user