Merge upstream lifecycle fixes into native Windows RDNA4 port

kernels/build.sh

@@ -2,7 +2,7 @@
 # Builds the native replacements for DLSS kernels into one directory that ZLUDA serves them from
 # (D4R_ZLUDA_NATIVE_DIR, or NativeKernelDirFast in d4r.ini). See docs/native-kernels.md.
 #
-# usage: kernels/build.sh [all|k|m|tex] [OUT_DIR]      (default: all, kernels/out/native)
+# usage: kernels/build.sh [all|k|l|m|tex] [OUT_DIR]      (default: all, kernels/out/native)
 #
 #   D4R_ROCM_DIR    ROCm installation with clang and the HIP device libraries (default /opt/rocm)
 #   D4R_GPU_ARCH    target GPU (default gfx1101): an RDNA3 (gfx110x) or RDNA4 (gfx120x) target; the kernels
@@ -37,7 +37,7 @@ esac
 CLANG="$ROCM/lib/llvm/bin/clang++"
 [[ -x "$CLANG" ]] || { echo "clang++ not found in $ROCM/lib/llvm/bin (set D4R_ROCM_DIR)" >&2; exit 2; }
 mkdir -p "$OUT"
-case "$WHAT" in all|k|m|tex) ;; *) echo "unknown kernel family: $WHAT" >&2; exit 2 ;; esac
+case "$WHAT" in all|k|l|m|tex) ;; *) echo "unknown kernel family: $WHAT" >&2; exit 2 ;; esac
 # Never certify a directory containing older fast binaries as an accuracy set.
 if [[ "$ACCURACY" == 1 ]] && compgen -G "$OUT/*.hsaco" >/dev/null &&
     { [[ ! -f "$OUT/d4r-accuracy.txt" ]] || [[ "$(cat "$OUT/d4r-accuracy.txt")" != 1 ]]; }; then
@@ -67,11 +67,11 @@ if [[ "$WHAT" == all || "$WHAT" == k ]]; then
     echo "== DLSS 4 (preset K) transformer layers"
     for src in "$HERE"/k/dltss_pwin_*.hip; do build_hip "$src"; done
 fi
-if [[ "$WHAT" == all || "$WHAT" == m ]]; then
-    echo "== DLSS 4.5 (preset M) Swin layers"
+if [[ "$WHAT" == all || "$WHAT" == m || "$WHAT" == l ]]; then
+    echo "== DLSS 4.5 (presets L/M) shared Swin layers"
     for src in "$HERE"/m/rrlite_*.hip; do build_hip "$src" "$([[ "$FP8" == 1 ]] && echo -DD4R_FP8_WMMA)"; done
 fi
-if [[ "$WHAT" == all || "$WHAT" == tex ]]; then
+if [[ "$WHAT" == all || "$WHAT" == tex || "$WHAT" == l ]]; then
     echo "== texture kernels (NVIDIA PTX with native parts)"
     : "${D4R_DLSS_DLL:?set D4R_DLSS_DLL to nvngx_dlss.dll (310.7)}"
     : "${D4R_ZLUDA_EMIT:?set D4R_ZLUDA_EMIT to d4r_emit from a ZLUDA build with patches/zluda}"
@@ -86,18 +86,30 @@ if [[ "$WHAT" == all || "$WHAT" == tex ]]; then
     # post 0.89 -> 0.77 ms, hiluma output 0.905 -> 0.887 ms (downsample was slower as wave64).]
     # Every flag combination DLSS selects (motion vectors hi/lo, HDR/LDR, depth inverted/regular, ...) gets
     # the same treatment, so games other than the one measured run the native versions too.
-    specs=(rrlite_dec0_4x4_folded:dec0_head)
-    for mv in mvhi mvlo; do
-        for range in hdr ldr; do
-            specs+=("rrlite_enc0_4x4_${mv}_${range}_folded:enc0_tail")
-            for v in 3_1 3_2; do specs+=("rrlite_post_${v}_${mv}_${range}_folded:sust_only:w64"); done
-            for depth in depthinv depthreg; do
-                for kind in "" _max; do
-                    [[ -z "$kind" ]] && specs+=("hiluma_engine_output_${depth}_${mv}_${range}_v1_rel:sust_only:w64")
-                    specs+=("hiluma_engine_output_${depth}_${mv}_${range}${kind}_v2_rel:sust_only:w64")
+    specs=()
+    if [[ "$WHAT" != l ]]; then
+        specs+=(rrlite_dec0_4x4_folded:dec0_head)
+        for mv in mvhi mvlo; do
+            for range in hdr ldr; do
+                specs+=("rrlite_enc0_4x4_${mv}_${range}_folded:enc0_tail")
+                for v in 3_1 3_2; do specs+=("rrlite_post_${v}_${mv}_${range}_folded:sust_only:w64"); done
+                for depth in depthinv depthreg; do
+                    for kind in "" _max; do
+                        [[ -z "$kind" ]] && specs+=("hiluma_engine_output_${depth}_${mv}_${range}_v1_rel:sust_only:w64")
+                        specs+=("hiluma_engine_output_${depth}_${mv}_${range}${kind}_v2_rel:sust_only:w64")
+                    done
                 done
             done
         done
+    fi
+    # L's unfolded variants keep their complete neural arithmetic. The folded M
+    # GEMM heads/tails are incompatible; substitute only the surface stores.
+    specs+=(rrlite_dec0_4x4:sust_only)
+    for mv in mvhi mvlo; do
+        for range in hdr ldr; do
+            specs+=("rrlite_enc0_4x4_${mv}_${range}:sust_only")
+            for v in 3_1 3_2; do specs+=("rrlite_post_${v}_${mv}_${range}:sust_only:w64"); done
+        done
     done
     for mode in static dynamic; do
         for range in hdr ldr; do specs+=("rrlite_downsample_kernel_${mode}_${range}:sust_only"); done

kernels/support/motion_dilate.hip

@@ -0,0 +1,64 @@
+#include "../../tools/d4r_motion_dilation.h"
+#pragma clang fp contract(off)
+
+__attribute__((device)) static int clamp_index(int x, uint32_t limit)
+{
+    return x < 0 ? 0 : x >= (int)limit ? (int)limit - 1 : x;
+}
+
+extern "C" __attribute__((global, amdgpu_flat_work_group_size(128, 128)))
+void d4r_motion_dilate(D4rMotionDilationParams p)
+{
+    const uint32_t id = __builtin_amdgcn_workgroup_id_x() * 128 + __builtin_amdgcn_workitem_id_x();
+    if ((uint64_t)id >= (uint64_t)p.motion_width * p.motion_height)
+        return;
+    const uint32_t x = id % p.motion_width, y = id / p.motion_width;
+    const int cx = clamp_index((int)__builtin_floorf((x + 0.5f) * p.render_width / p.motion_width + p.jitter_x), p.render_width);
+    const int cy = clamp_index((int)__builtin_floorf((y + 0.5f) * p.render_height / p.motion_height + p.jitter_y), p.render_height);
+    const uint8_t* depth = (const uint8_t*)p.depth;
+    const uint8_t* motion = (const uint8_t*)p.motion;
+    uint8_t* output = (uint8_t*)p.output;
+    const float current = *(const float*)(depth + (uint64_t)cy * p.depth_pitch + 4 * cx);
+    float best = current;
+    int bx = cx, by = cy;
+    for (int dy = -(int)p.radius; dy <= (int)p.radius; ++dy)
+        for (int dx = -(int)p.radius; dx <= (int)p.radius; ++dx)
+        {
+            const int nx = clamp_index(cx + dx, p.render_width), ny = clamp_index(cy + dy, p.render_height);
+            const float candidate = *(const float*)(depth + (uint64_t)ny * p.depth_pitch + 4 * nx);
+            if (p.reversed_depth ? candidate > best : candidate < best)
+                best = candidate, bx = nx, by = ny;
+        }
+    // Sky gaps need wider coverage than opaque edges. Select the closest
+    // foreground pixel within eight render pixels, rather than the closest
+    // depth (which can borrow a distant roof/chimney's velocity).
+    const bool sky = p.reversed_depth ? current <= 1.e-7f : current >= 1.0f - 1.e-7f;
+    bool sky_filled = false;
+    if (sky)
+    {
+        int best_distance = 65;
+        for (int dy = -8; dy <= 8; ++dy)
+            for (int dx = -8; dx <= 8; ++dx)
+            {
+                const int nx = cx + dx, ny = cy + dy;
+                const int distance = dx * dx + dy * dy;
+                if (nx < 0 || ny < 0 || nx >= (int)p.render_width || ny >= (int)p.render_height ||
+                    distance > 64 || distance > best_distance)
+                    continue;
+                const float candidate = *(const float*)(depth + (uint64_t)ny * p.depth_pitch + 4 * nx);
+                const bool foreground = p.reversed_depth ? candidate > 1.e-7f : candidate < 1.0f - 1.e-7f;
+                // Stable coordinate tie-break matches nearest-foreground fill.
+                if (foreground && (distance < best_distance || nx < bx || (nx == bx && ny < by)))
+                    best_distance = distance, best = candidate, bx = nx, by = ny, sky_filled = true;
+            }
+    }
+    uint32_t value = *(const uint32_t*)(motion + (uint64_t)y * p.motion_pitch + 4 * x);
+    const bool nearer = p.reversed_depth ? best > current * 1.02f + 1.e-7f : best < current * 0.98f - 1.e-7f;
+    if (sky_filled || nearer)
+    {
+        const int mx = clamp_index((int)__builtin_floorf((bx + 0.5f - p.jitter_x) * p.motion_width / p.render_width), p.motion_width);
+        const int my = clamp_index((int)__builtin_floorf((by + 0.5f - p.jitter_y) * p.motion_height / p.render_height), p.motion_height);
+        value = *(const uint32_t*)(motion + (uint64_t)my * p.motion_pitch + 4 * mx);
+    }
+    *(uint32_t*)(output + (uint64_t)y * p.motion_pitch + 4 * x) = value;
+}

kernels/tex/make_ptx.py

@@ -211,7 +211,9 @@ def replace_rz_round(lines):
 ENC0_REF = "rrlite_enc0_4x4_mvhi_hdr_folded"
 # the other flag combinations of the kernels whose only native part is the surface-store rewrite
 SUST_ONLY_RE = re.compile(r"^(hiluma_engine_output_depth(inv|reg)_mv(hi|lo)_(hdr|ldr)(_max)?_v[12]_rel|"
-                          r"rrlite_post_3_[12]_mv(hi|lo)_(hdr|ldr)_folded|rrlite_downsample_kernel_(static|dynamic)_(hdr|ldr))$")
+                          r"rrlite_post_3_[12]_mv(hi|lo)_(hdr|ldr)(_folded)?|"
+                          r"rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)|rrlite_dec0_4x4|"
+                          r"rrlite_downsample_kernel_(static|dynamic)_(hdr|ldr))$")
 ENC0_RE = re.compile(r"^rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)_folded$")