Merge upstream RDNA4 layouts and validate native Windows K integration

tools/d4r_nvngx_shim.cpp

@@ -535,6 +535,8 @@ static void load_portable_config()
         portable_set_unix("D4R_ZLUDA_NATIVE_DIR", g_portable.unixDir + "/kernels");
     portable_set_unix("D4R_ZLUDA_WMMA", ini_flag(ini, "kernels", "Wmma", 1) ? "1" : "0");
     portable_set_unix("D4R_ZLUDA_WMMA_FP8", ini_flag(ini, "kernels", "Fp8Wmma", 1) ? "1" : "0");
+    // RDNA4's native FP8 WMMA; ZLUDA and the bridge ignore it on other GPUs
+    portable_set_unix("D4R_ZLUDA_WMMA_FP8_NATIVE", ini_flag(ini, "kernels", "NativeFp8", 1) ? "1" : "0");
     portable_set_unix("D4R_ZLUDA_IGNORE_DENORMAL", ini_flag(ini, "kernels", "IgnoreDenormals", 1) ? "1" : "0");
     const std::string maxBlock = ini_value(ini, "kernels", "ImplicitMaxBlock");
     portable_set_unix("D4R_ZLUDA_IMPLICIT_MAX_BLOCK", maxBlock.empty() ? "256" : maxBlock);
@@ -3774,6 +3776,12 @@ static void run_evaluation(Feature* feature, int slotIndex, uint32_t frame, Fram
                               g.cu.setArrayRedirect(feature->output.array, target, static_cast<uint32_t>(rowBytes)) == 0;
         if (!redirect && feature->outputRedirected)
             g.cu.setArrayRedirect(feature->output.array, 0, 0);
+        static bool logged = false;
+        if (redirect && !logged)
+        {
+            logf("frame %u: direct output (the native output kernel writes the shared buffer; no array copy)", frame);
+            logged = true;
+        }
         feature->outputRedirected = redirect;
     }
     const int eventStartResult = timing.enabled && feature->profileEventsReady
@@ -4250,11 +4258,12 @@ D4R_EXPORT NgxResult NVSDK_NGX_D3D12_CreateFeature(ID3D12GraphicsCommandList*, u
             preset = value;
     }
     feature->preset = presets[1];
-    // Direct output needs the native output kernel of the preset in use (verified: K = 11; M's result is
-    // not written through the redirected stores). Only a forced preset qualifies.
+    // Direct output needs the native output kernel of the preset in use (verified: K = 11 through
+    // hiluma_engine_output, M = 13 through rrlite_downsample_kernel, its last kernel). Only a forced
+    // preset qualifies.
     if (!forced.empty())
     {
-        char allowed[128] = "11";
+        char allowed[128] = "11,13";
         if (const std::string list = env_string("D4R_SHIM_OUTPUT_DIRECT_PRESETS"); !list.empty())
             snprintf(allowed, sizeof(allowed), "%s", list.c_str());
         for (char* token = strtok(allowed, ","); token != nullptr; token = strtok(nullptr, ","))

tools/windows/gfx12_wmma_probe.cpp

@@ -37,11 +37,13 @@ int main(int argc, char** argv)
             HipApi& hip; hipModule_t module;
             ~ModuleCleanup() { if (module) (void)hip.hipModuleUnload(module); }
         } cleanup{hip, module};
-        hipFunction_t raw_kernel = nullptr, adapter_kernel = nullptr, swap_kernel = nullptr;
+        hipFunction_t raw_kernel = nullptr, adapter_kernel = nullptr, upstream_kernel = nullptr, swap_kernel = nullptr;
         hip.check(hip.hipModuleGetFunction(&raw_kernel, module, "d4r_wmma_gfx1201"),
             "hipModuleGetFunction(WMMA)");
         hip.check(hip.hipModuleGetFunction(&adapter_kernel, module, "d4r_wmma_legacy_contract"),
             "hipModuleGetFunction(WMMA adapter)");
+        hip.check(hip.hipModuleGetFunction(&upstream_kernel, module, "d4r_wmma_upstream_contract"),
+            "hipModuleGetFunction(WMMA upstream)");
         hip.check(hip.hipModuleGetFunction(&swap_kernel, module, "d4r_wmma_swap_test"),
             "hipModuleGetFunction(WMMA swap)");
         constexpr size_t elements = 16u * 16u;
@@ -86,11 +88,13 @@ int main(int argc, char** argv)
             void* output_arg = dout.pointer;
             unsigned steps_arg = steps;
             void* params[] = {&a_arg, &b_arg, &c_arg, &output_arg, &steps_arg};
-            for (unsigned mode = 0; mode < 2u; ++mode) {
+            const hipFunction_t kernels[] = {raw_kernel, adapter_kernel, upstream_kernel};
+            const char* modes[] = {"raw", "legacy_adapter", "upstream_layout"};
+            for (unsigned mode = 0; mode < 3u; ++mode) {
             output.fill(std::numeric_limits<float>::quiet_NaN());
             hip.check(hip.hipMemcpy(dout.pointer, output.data(), sizeof(output), hipMemcpyHostToDevice), "hipMemcpy(output sentinel)");
-            hip.check(hip.hipModuleLaunchKernel(mode ? adapter_kernel : raw_kernel, 1, 1, 1, 32, 1, 1, 0, nullptr,
-                params, nullptr), mode ? "hipModuleLaunchKernel(WMMA adapter)" : "hipModuleLaunchKernel(WMMA raw)");
+            hip.check(hip.hipModuleLaunchKernel(kernels[mode], 1, 1, 1, 32, 1, 1, 0, nullptr,
+                params, nullptr), modes[mode]);
             hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(WMMA)");
             hip.check(hip.hipMemcpy(output.data(), dout.pointer, sizeof(output), hipMemcpyDeviceToHost), "hipMemcpy(output)");
             for (unsigned row = 0; row < 16u; ++row) {
@@ -107,7 +111,7 @@ int main(int argc, char** argv)
                     const float relative = absolute / std::max(1.0f, std::fabs(expected));
                     if (!std::isfinite(got) || absolute > 0.0001f) {
                         std::fprintf(stderr, "FAIL WMMA mode=%s iteration=%u row=%u col=%u steps=%u expected=%.9g actual=%.9g abs=%.9g\n",
-                            mode ? "adapter" : "raw", iteration, row, col, steps, expected, got, absolute);
+                            modes[mode], iteration, row, col, steps, expected, got, absolute);
                         return 5;
                     }
                     max_absolute_error = std::max(max_absolute_error, absolute);
@@ -116,7 +120,7 @@ int main(int argc, char** argv)
             }
             }
         }
-        std::printf("PASS GFX12_WMMA architecture=gfx1201 modes=raw,legacy_adapter iterations=%u max_abs=%.9g max_rel=%.9g\n",
+        std::printf("PASS GFX12_WMMA architecture=gfx1201 modes=raw,legacy_adapter,upstream_layout iterations=%u max_abs=%.9g max_rel=%.9g\n",
             args.iterations, max_absolute_error, max_relative_error);
         return 0;
     } catch (const std::exception& error) {

tools/windows/wmma_gfx1201.hip

@@ -4,6 +4,7 @@
 using h8 = _Float16 __attribute__((ext_vector_type(8)));
 using f8 = float __attribute__((ext_vector_type(8)));
 #include "../../kernels/common/wmma_backend.h"
+#include "../../kernels/common/wmma_layout.h"
 
 extern "C" __attribute__((global)) void d4r_wmma_swap_test(float* output)
 {
@@ -65,3 +66,29 @@ extern "C" __attribute__((global)) void d4r_wmma_legacy_contract(
     for (unsigned int i = 0; i < 8u; ++i)
         output[(2u * i + group) * 16u + token] = result[i];
 }
+
+// Upstream's native per-target layout, including accumulator -> next-operand
+// packing. It must match the same scalar reference as the independent probe.
+extern "C" __attribute__((global)) void d4r_wmma_upstream_contract(
+    const unsigned short* a_bits, const unsigned short* b_bits,
+    const float* initial_c, float* output, unsigned int steps)
+{
+    const unsigned int lane = __builtin_amdgcn_workitem_id_x();
+    if (lane >= 32u) return;
+    const unsigned int token = lane & 15u;
+    const _Float16* a = reinterpret_cast<const _Float16*>(a_bits);
+    const _Float16* b = reinterpret_cast<const _Float16*>(b_bits);
+    const wm_op a_frag = wm_op_load(a + token * 16u);
+    const wm_op b_frag = wm_op_load(b + token * 16u);
+    wm_f8v c_frag;
+    for (unsigned int i = 0; i < 8u; ++i)
+        c_frag[i] = initial_c[wm_acc_row(i) * 16u + token];
+    wm_f8v result = wm_mma(a_frag, b_frag, c_frag);
+    if (steps == 2u)
+        result = wm_mma(a_frag, b_frag, result);
+    // Fixture values are exactly representable as FP16. Checking this roundtrip
+    // catches a wrong gfx12 fragment reconstruction without a tolerance mask.
+    const wm_op next_operand = wm_op_from_f8(result);
+    for (unsigned int i = 0; i < 8u; ++i)
+        output[wm_acc_row(i) * 16u + token] = static_cast<float>(wm_op_acc_elem(next_operand, i));
+}

tools/wine_nvcuda_bridge.c

@@ -389,11 +389,21 @@ static void native_cleanup(void)
         remove_directory(native_served);
 }
 
-/* The first GPU's gfx target from the KFD topology ("gfx1101"), "" when unknown. */
+/* The gfx target ("gfx1101") of the GPU DLSS runs on, "" when unknown: D4R_GPU_ARCH if set, else the KFD
+   topology's GPU with the most SIMDs. With an integrated and a discrete GPU (Ryzen 7000/9000 desktops) the
+   first KFD GPU node can be the integrated one, whose kernels would not match the discrete GPU. */
 static void gpu_architecture(char* out, size_t size)
 {
     out[0] = '\0';
-    for (int node = 0; node < 16 && out[0] == '\0'; ++node)
+    const char* forced = getenv("D4R_GPU_ARCH");
+    if (forced != NULL && strncmp(forced, "gfx", 3) == 0)
+    {
+        snprintf(out, size, "%s", forced);
+        return;
+    }
+    unsigned long best_simds = 0;
+    int gpus = 0;
+    for (int node = 0; node < 16; ++node)
     {
         char path[128];
         snprintf(path, sizeof(path), "/sys/class/kfd/kfd/topology/nodes/%d/properties", node);
@@ -408,9 +418,17 @@ static void gpu_architecture(char* out, size_t size)
             sscanf(line, "gfx_target_version %lu", &target);
         }
         fclose(properties);
-        if (simds != 0 && target != 0)
+        if (simds == 0 || target == 0)
+            continue;
+        ++gpus;
+        if (simds > best_simds)
+        {
+            best_simds = simds;
             snprintf(out, size, "gfx%lu%lu%lx", target / 10000, (target / 100) % 100, target % 100);
+        }
     }
+    if (gpus > 1)
+        tracef("native kernels: %d GPUs; using the one with the most SIMDs (%s); D4R_GPU_ARCH overrides", gpus, out);
 }
 
 static void prepare_native_kernels(const char* cache_home)
@@ -421,9 +439,22 @@ static void prepare_native_kernels(const char* cache_home)
     /* a release directory holds one manifest and set of code objects per GPU target */
     char architecture[32], source[1024], path[1200];
     gpu_architecture(architecture, sizeof(architecture));
-    snprintf(source, sizeof(source), "%s/%s", configured, architecture);
-    snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
-    FILE* manifest = architecture[0] != '\0' ? fopen(path, "r") : NULL;
+    FILE* manifest = NULL;
+    /* RDNA4 with native FP8 WMMA (D4R_ZLUDA_WMMA_FP8_NATIVE=1, d4r.ini NativeFp8): the <target>-fp8 variant,
+       whose kernels match ZLUDA's FP8 lowering; no other GPU has such a directory */
+    const char* fp8 = getenv("D4R_ZLUDA_WMMA_FP8_NATIVE");
+    if (fp8 != NULL && strcmp(fp8, "1") == 0 && strncmp(architecture, "gfx12", 5) == 0)
+    {
+        snprintf(source, sizeof(source), "%s/%s-fp8", configured, architecture);
+        snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
+        manifest = fopen(path, "r");
+    }
+    if (manifest == NULL)
+    {
+        snprintf(source, sizeof(source), "%s/%s", configured, architecture);
+        snprintf(path, sizeof(path), "%s/d4r-kernels.txt", source);
+        manifest = architecture[0] != '\0' ? fopen(path, "r") : NULL;
+    }
     if (manifest == NULL)
     {
         snprintf(source, sizeof(source), "%s", configured);
@@ -679,7 +710,7 @@ static void verify_native_kernels(const void* image)
     pthread_mutex_unlock(&native_lock);
 }
 
-/* DLSS's output kernels (hiluma_engine_output_*): NGX looks up every variant, then launches the one
+/* DLSS's output kernels (hiluma_engine_output_*, rrlite_downsample_kernel_*): NGX looks up every variant, then launches the one
    matching the feature's flags. The shim's direct output relies on stores that only d4r's native
    output kernel makes, so it asks whether the last output kernel launched was native (a file in
    D4R_ZLUDA_NATIVE_DIR, which is the verified per-process directory in a portable install). */
@@ -694,7 +725,9 @@ static int last_output_native = -1;
 
 static void note_function_lookup(CUfunction function, const char* name)
 {
-    if (name == NULL || strncmp(name, "hiluma_engine_output", 20) != 0)
+    /* DLSS 4 writes its result with hiluma_engine_output_*, DLSS 4.5 (rrlite) with its final
+       rrlite_downsample_kernel_* (the post kernel works at a larger internal resolution) */
+    if (name == NULL || (strncmp(name, "hiluma_engine_output", 20) != 0 && strncmp(name, "rrlite_downsample_kernel", 24) != 0))
         return;
     const char* directory = getenv("D4R_ZLUDA_NATIVE_DIR");
     int native = 0;