Narrow the RDNA4 accuracy set's cost for preset M
The FP8 Swin layers encode with a clamp to +-448 (NaN-propagating packed minimum/maximum) followed by v_cvt_pk_fp8_f32, in both the fast and the accuracy build: equal to half_to_e4m3 for every non-NaN input, NaN encodes as 0xff. The accuracy build rounds each k32 step with v_fma_mix (K32_MIX). On gfx12 the accuracy builds of M's post and downsample kernels are wave64 with denormal handling kept. RX 9070 XT, 1280x720 -> 3840x2160, D3D12 harness: the accuracy set goes from 3.54 ms to 3.05 ms per evaluation (3.01 -> 2.59 ms under load). Its Swin layers still equal swin_model.py on sampled blocks and its image is byte-identical to the previous accuracy set over 8 temporal frames.
This commit is contained in:
woutermans committed
1 parent
ba77946a4a
commit
ffa686e079
4 files changed
+30
-29
No files matched your search
+10
-2
@@ -18,6 +18,7 @@
|
||||
# which compiles PTX offline for D4R_GPU_ARCH (no GPU of that kind needed)
|
||||
# D4R_DLSS_PTX_DIR optional pre-extracted PTX directory for parallel per-target builds
|
||||
set -euo pipefail
|
||||
shopt -s extglob
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WHAT="${1:-all}"
|
||||
@@ -131,8 +132,15 @@ if [[ "$WHAT" == all || "$WHAT" == tex || "$WHAT" == l ]]; then
|
||||
for spec in "${specs[@]}"; do
|
||||
IFS=: read -r kernel src mode <<< "$spec"
|
||||
[[ "$ACCURACY" == 0 && "$kernel" == "$native_out" ]] && continue
|
||||
D4R_PREFER_ACCURACY="$ACCURACY" \
|
||||
D4R_ZLUDA_WAVE64="$([[ "$mode" == w64 && "$ACCURACY" == 0 ]] && echo 1 || echo 0)" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" \
|
||||
# accuracy sets are wave32, except M's post and downsample kernels on RDNA4: their wave64 builds keep the
|
||||
# denormal handling and gave the same image on an RX 9070 XT (post 0.79 -> 0.67 ms, downsample 0.43 -> 0.39 ms)
|
||||
w64=0
|
||||
if [[ "$mode" == w64 ]]; then
|
||||
[[ "$ACCURACY" == 0 ]] && w64=1
|
||||
[[ "$ARCH" == gfx12* && "$kernel" == rrlite_@(post|downsample)_* ]] && w64=1
|
||||
fi
|
||||
D4R_PREFER_ACCURACY="$ACCURACY" D4R_TEX_ACCURACY_WAVE64="$w64" \
|
||||
D4R_ZLUDA_WAVE64="$w64" D4R_ROCM_DIR="$ROCM" D4R_GPU_ARCH="$ARCH" \
|
||||
D4R_TEX_FP8="$FP8" D4R_DLSS_PTX_DIR="$PTX_DIR" "$HERE/tex/build_tex.sh" "$kernel" "$src" "$OUT"
|
||||
done
|
||||
if [[ "$WHAT" != l && "$ACCURACY" == 0 ]]; then
|
||||
|
||||
+15
-23
@@ -34,6 +34,12 @@
|
||||
#if defined(D4R_FP8_WMMA) && defined(__GFX12__) && !defined(SWIN_NO_A8)
|
||||
#define SWIN_A8 1
|
||||
#endif
|
||||
// The exact SWIN_A8 build rounds each k32 step with v_fma_mix (f16(P + C) in one rounding) instead of
|
||||
// converting the accumulator to f32 and back: the same codes as the conversion form on the recorded launches
|
||||
// and as tools/swin_model.py, at about half the cost. K32_NO_MIX keeps the conversion form.
|
||||
#if defined(SWIN_A8) && defined(SWIN_EXACT) && !defined(K32_NO_MIX) && !defined(K32_MIX)
|
||||
#define K32_MIX
|
||||
#endif
|
||||
|
||||
#pragma clang fp contract(off)
|
||||
|
||||
@@ -207,31 +213,17 @@ __device__ __forceinline__ uint32_t codes8x2(hv2 q)
|
||||
}
|
||||
|
||||
#ifdef SWIN_A8
|
||||
// e4m3 codes of two halves (RNE, satfinite, NaN -> 0x7f | sign: equal to half_to_e4m3 for all inputs) in bytes 0
|
||||
// and 1. The instruction rounds like NVIDIA's conversion but encodes overflow as 0x7f, so magnitudes are
|
||||
// clamped to 448 first and NaNs enter as infinities.
|
||||
__device__ __forceinline__ uint32_t codes2(hv2 h)
|
||||
{
|
||||
const u16x2 u = __builtin_bit_cast(u16x2, h);
|
||||
const u16x2 m = u & (unsigned short)0x7fff;
|
||||
const u16x2 nan = __builtin_bit_cast(u16x2, (i16x2)(__builtin_bit_cast(i16x2, (u16x2)((unsigned short)0x7c00 - m)) >> 15));
|
||||
const u16x2 k = __builtin_elementwise_max(__builtin_elementwise_min(m, (u16x2){0x5f00, 0x5f00}), (u16x2)(nan & (unsigned short)0x7c00));
|
||||
const hv2 c = __builtin_bit_cast(hv2, (u16x2)(k | (u & (unsigned short)0x8000)));
|
||||
return (uint32_t)__builtin_amdgcn_cvt_pk_fp8_f32((float)c[0], (float)c[1], 0, false) & 0xffffu;
|
||||
}
|
||||
// e4m3 codes of four halves in the bytes of one word. Exact builds use codes2. The fast build converts
|
||||
// first and then turns the instruction's overflow code into the saturated one in all four bytes at once
|
||||
// (low seven bits 0x7f -> 0x7e); it equals half_to_e4m3 for every non-NaN input, and a NaN encodes as the
|
||||
// negative saturated value 0xfe instead of 0x7f | sign.
|
||||
// e4m3 codes of four halves (RNE, satfinite) in the bytes of one word. The instruction rounds like NVIDIA's
|
||||
// conversion but encodes overflow as 0x7f, so the values are clamped to +-448 first, with the NaN-propagating
|
||||
// minimum / maximum. Equal to half_to_e4m3 for every non-NaN input; a NaN encodes as the NaN code 0xff (the
|
||||
// reference keeps its sign: 0x7f or 0xff).
|
||||
__device__ __forceinline__ uint32_t codes4(hv2 a, hv2 b)
|
||||
{
|
||||
#ifdef SWIN_EXACT
|
||||
return codes2(a) | (codes2(b) << 16);
|
||||
#else
|
||||
uint32_t w = (uint32_t)__builtin_amdgcn_cvt_pk_fp8_f32((float)a[0], (float)a[1], 0, false);
|
||||
w = (uint32_t)__builtin_amdgcn_cvt_pk_fp8_f32((float)b[0], (float)b[1], (int)w, true);
|
||||
return w - ((((w & 0x7f7f7f7fu) + 0x01010101u) >> 7) & 0x01010101u);
|
||||
#endif
|
||||
const hv2 lim = {(half_t)448.0f, (half_t)448.0f};
|
||||
a = __builtin_elementwise_maximum(__builtin_elementwise_minimum(a, lim), -lim);
|
||||
b = __builtin_elementwise_maximum(__builtin_elementwise_minimum(b, lim), -lim);
|
||||
const uint32_t w = (uint32_t)__builtin_amdgcn_cvt_pk_fp8_f32((float)a[0], (float)a[1], 0, false);
|
||||
return (uint32_t)__builtin_amdgcn_cvt_pk_fp8_f32((float)b[0], (float)b[1], (int)w, true);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
# part of it calls functions from SRC.hip, is compiled by ZLUDA with SRC's bitcode linked in
|
||||
# (D4R_ZLUDA_EXTRA_BC) and the resulting code object is saved as OUT_DIR/KERNEL.hsaco.
|
||||
# Called by kernels/build.sh, which sets D4R_ROCM_DIR, D4R_GPU_ARCH, D4R_DLSS_PTX_DIR, D4R_ZLUDA_EMIT,
|
||||
# D4R_ZLUDA_WAVE64 (1 = wave64 code object) and D4R_TEX_FP8 (1 = the gfx12 native-FP8 variant).
|
||||
# D4R_ZLUDA_WAVE64 (1 = wave64 code object; accuracy builds only with D4R_TEX_ACCURACY_WAVE64=1) and
|
||||
# D4R_TEX_FP8 (1 = the gfx12 native-FP8 variant).
|
||||
# ZLUDA compiles offline for D4R_GPU_ARCH with its d4r_emit tool, so any target builds on any machine.
|
||||
set -euo pipefail
|
||||
K=$1 SRC=$2 OUT=$3
|
||||
@@ -31,7 +32,7 @@ D4R_PREFER_ACCURACY="$ACCURACY" python3 "$D/make_ptx.py" "$K" "$B/$K.ptx" >/dev/
|
||||
env LD_LIBRARY_PATH="$R/lib" D4R_ZLUDA_EXTRA_BC="$B/extra.bc" D4R_ZLUDA_WMMA=1 D4R_ZLUDA_WMMA_FP8=1 \
|
||||
D4R_ZLUDA_WMMA_FP8_NATIVE="$FP8" D4R_ZLUDA_IGNORE_DENORMAL="$((1 - ACCURACY))" \
|
||||
D4R_ZLUDA_WMMA_F32ACC=0 D4R_ZLUDA_FAST_MATH=0 D4R_ZLUDA_IMPLICIT_MAX_BLOCK=256 \
|
||||
D4R_ZLUDA_WAVE64="$([[ "$ACCURACY" == 1 ]] && echo 0 || echo "${D4R_ZLUDA_WAVE64:-0}")" \
|
||||
D4R_ZLUDA_WAVE64="$([[ "$ACCURACY" == 1 && "${D4R_TEX_ACCURACY_WAVE64:-0}" != 1 ]] && echo 0 || echo "${D4R_ZLUDA_WAVE64:-0}")" \
|
||||
"$D4R_ZLUDA_EMIT" "$B/$K.ptx" "$B/dump" "$ARCH" >/dev/null
|
||||
cp "$B/dump/module.hsaco" "$OUT/$K.hsaco"
|
||||
printf '%-52s %s\n' "$K" "$(grep -E '^\s+\.(vgpr_count|private_segment_fixed_size|wavefront_size):' "$B/dump/asm.s" | tr -s ' ' | paste -sd' ')"
|
||||
Reference in new issue
Block a user