wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add d4r inference engines and Windows parity tooling

Browse Source 
Add native K/M/RR inference engines, HIP kernels, replay and validation
harnesses, and Windows DLSS setup and runtime support. Include GPU
profiling tools, build and packaging scripts, platform integration
patches, and implementation documentation.
This commit is contained in:
ayo committed 2026-10-16 08:54:47 -04:00
1 parent 25493b3198
commit 7c5d3c9157
430 files changed
+76191
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

patches/optiscaler/0001-windows-external-d4r-backend.patch
+328
View File
@@ -0,0 +1,328 @@
From 33bbac250ac7e2ca4eff67e8817ada9dad578208 Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 16:10:08 +0300
Subject: [PATCH] Support explicitly selected Windows d4r DLSS backend on
physical AMD adapters
---
OptiScaler/Config.cpp | 3 ++
OptiScaler/Config.h | 1 +
OptiScaler/OptiScaler.vcxproj | 4 +--
OptiScaler/State.h | 1 +
OptiScaler/Util.cpp | 12 +++++--
OptiScaler/dllmain.cpp | 33 +++++++++++++++++--
OptiScaler/hooks/LibraryLoad_Hooks.cpp | 17 ++++++++++
OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp | 17 ++++++++++
OptiScaler/misc/IdentifyGpu.cpp | 20 +++++++++++
OptiScaler/proxies/NVNGX_Proxy.h | 5 ++-
OptiScaler/upscalers/FeatureProvider_Dx12.cpp | 10 ++++++
11 files changed, 115 insertions(+), 8 deletions(-)
diff --git a/OptiScaler/Config.cpp b/OptiScaler/Config.cpp
index d8c62d3..f444387 100644
--- a/OptiScaler/Config.cpp
+++ b/OptiScaler/Config.cpp
@@ -323,6 +323,7 @@ bool Config::Reload(std::filesystem::path iniPath)
{
// Don't enable again if set false because of no nvngx found
DLSSEnabled.set_from_config(readBool("DLSS", "Enabled"));
+ DLSSAllowExternalBackend.set_from_config(readBool("DLSS", "AllowExternalBackend"));
UseGenericAppIdWithDlss.set_from_config(readBool("DLSS", "UseGenericAppIdWithDlss"));
RenderPresetOverride.set_from_config(readBool("DLSS", "RenderPresetOverride"));
@@ -1168,6 +1169,8 @@ bool Config::SaveIni()
// DLSS
{
ini.SetValue("DLSS", "Enabled", GetBoolValue(Instance()->DLSSEnabled.value_for_config()).c_str());
+ ini.SetValue("DLSS", "AllowExternalBackend",
+ GetBoolValue(Instance()->DLSSAllowExternalBackend.value_for_config()).c_str());
ini.SetValue("DLSS", "RenderPresetOverride",
GetBoolValue(Instance()->RenderPresetOverride.value_for_config()).c_str());
ini.SetValue("DLSS", "RenderPresetForAll",
diff --git a/OptiScaler/Config.h b/OptiScaler/Config.h
index 4ab62a4..0caf8c8 100644
--- a/OptiScaler/Config.h
+++ b/OptiScaler/Config.h
@@ -275,6 +275,7 @@ class Config
// DLSS
CustomOptional<bool> DLSSEnabled { true };
+ CustomOptional<bool> DLSSAllowExternalBackend { false };
CustomOptional<bool> RenderPresetOverride { false };
CustomOptional<uint32_t> RenderPresetForAll { 0 };
CustomOptional<uint32_t> RenderPresetDLAA { 0 };
diff --git a/OptiScaler/OptiScaler.vcxproj b/OptiScaler/OptiScaler.vcxproj
index 24ca80b..dfbfc97 100644
--- a/OptiScaler/OptiScaler.vcxproj
+++ b/OptiScaler/OptiScaler.vcxproj
@@ -236,7 +236,7 @@
<SubSystem>Windows</SubSystem>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
- <GenerateDebugInformation>false</GenerateDebugInformation>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
<EnableUAC>false</EnableUAC>
<AdditionalDependencies>winhttp.lib;WindowsApp.lib;Dbghelp.lib;dxgi.lib;d3d11.lib;d3d12.lib;vulkan-1.lib;dxguid.lib;freetype.lib;ffx_fsr2_api_x64.lib;ffx_fsr2_api_dx11_x64.lib;ffx_fsr2_api_dx12_x64.lib;ffx_fsr2_api_vk_x64.lib;ffx_fsr2_212_api_dx12_x64.lib;ffx_fsr2_212_api_vk_x64.lib;ffx_fsr2_212_api_x64.lib;d3dcompiler.lib;ffx_backend_dx11_x64.lib;ffx_fsr3_x64.lib;ffx_frameinterpolation_x64.lib;ffx_fsr3upscaler_x64.lib;ffx_opticalflow_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ModuleDefinitionFile>Source.def</ModuleDefinitionFile>
@@ -873,4 +873,4 @@ copy NUL "$(SolutionDir)x64\Release\a\!! EXTRACT ALL FILES TO GAME FOLDER !!" /Y
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
-</Project>
\ No newline at end of file
+</Project>
diff --git a/OptiScaler/State.h b/OptiScaler/State.h
index 42164de..84f8263 100644
--- a/OptiScaler/State.h
+++ b/OptiScaler/State.h
@@ -265,6 +265,7 @@ class State
int dlssgDetectedInterpolationCount = 0;
// DLSS
+ bool dlssExternalBackend = false;
bool dlssPresetsOverriddenExternally = false;
bool dlssPresetsOverridenByOpti = false;
uint32_t dlssRenderPresetExternal = 0;
diff --git a/OptiScaler/Util.cpp b/OptiScaler/Util.cpp
index f7ac6fa..7367a91 100644
--- a/OptiScaler/Util.cpp
+++ b/OptiScaler/Util.cpp
@@ -486,8 +486,16 @@ bool Util::GetFileVersion(std::wstring dllPath, version_t* fileVersionOut, versi
bool Util::IsSubpath(const std::filesystem::path& path, const std::filesystem::path& base)
{
- auto rel = std::filesystem::relative(path, base);
- auto first = *rel.begin();
+ // Called by LdrLoadDll hooks: relative()/weakly_canonical() perform file
+ // queries, can re-enter the hooks, and throw when an optional directory or
+ // a bare DLL search name cannot be resolved. This is a lexical routing
+ // check, not a filesystem access control decision.
+ auto normalizedPath = std::filesystem::path(ToLower(path.lexically_normal().wstring()));
+ auto normalizedBase = std::filesystem::path(ToLower(base.lexically_normal().wstring()));
+ auto rel = normalizedPath.lexically_relative(normalizedBase);
+ if (rel.empty())
+ return false;
+ const auto first = *rel.begin();
return first != "." && first != "..";
}
diff --git a/OptiScaler/dllmain.cpp b/OptiScaler/dllmain.cpp
index cba14b1..b4b786e 100644
--- a/OptiScaler/dllmain.cpp
+++ b/OptiScaler/dllmain.cpp
@@ -948,7 +948,7 @@ static void CheckWorkingMode()
// NVAPI
// Doesn't seem to like GetModuleHandle for some reason, so call our load to make sure
- if (GetDllNameWModule(&nvapiNamesW) != nullptr)
+ if (!State::Instance().dlssExternalBackend && GetDllNameWModule(&nvapiNamesW) != nullptr)
{
// This hooks nvapi as well when possible
auto nvapi64 = LibraryLoadHooks::LoadNvApi();
@@ -1958,7 +1958,33 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv
return nvidiaDetected;
};
- bool possibleNvidia = isNvidiaViaNvapi();
+ // Opt in only to a configured, versioned Windows d4r backend. The
+ // marker performs no HIP/NGX initialization under DLL_PROCESS_ATTACH.
+ // GPU support is still checked by its public NGX requirements API.
+ if (Config::Instance()->DLSSEnabled.value_or_default() &&
+ Config::Instance()->DLSSAllowExternalBackend.value_or_default() &&
+ Config::Instance()->NvngxPath.has_value())
+ {
+ auto backendPath = std::filesystem::path(Config::Instance()->NvngxPath.value()) / L"_nvngx.dll";
+ if (backendPath.is_absolute())
+ {
+ auto module = NtdllProxy::LoadLibraryExW_Ldr(backendPath.c_str(), nullptr,
+ LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+ if (module)
+ {
+ using BackendVersion = unsigned (*)();
+ auto version = reinterpret_cast<BackendVersion>(
+ KernelBaseProxy::GetProcAddress_()(module, "d4r_WindowsBackendVersion"));
+ State::Instance().dlssExternalBackend = version && version() == 1;
+ NtdllProxy::FreeLibrary_Ldr(module);
+ }
+ }
+ if (State::Instance().dlssExternalBackend)
+ spdlog::info("Validated external Windows d4r DLSS backend; retaining physical GPU identity");
+ else
+ spdlog::error("AllowExternalBackend requires an absolute NvngxPath containing Windows d4r ABI 1");
+ }
+ bool possibleNvidia = !State::Instance().dlssExternalBackend && isNvidiaViaNvapi();
spdlog::info("");
spdlog::info("Check for DLSS files");
@@ -1992,7 +2018,8 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserv
State::Instance().NVNGX_DLSSG_Path = Util::FindFilePath(exePath, "nvngx_dlssg.dll");
// Not 100% accurate for Nvidia cards without DLSS
- if (Config::Instance()->DLSSEnabled.value_or_default() && possibleNvidia)
+ if (Config::Instance()->DLSSEnabled.value_or_default() &&
+ (possibleNvidia || State::Instance().dlssExternalBackend))
{
if (State::Instance().NVNGX_DLSS_Path.has_value())
{
diff --git a/OptiScaler/hooks/LibraryLoad_Hooks.cpp b/OptiScaler/hooks/LibraryLoad_Hooks.cpp
index 90bdb89..108ea5b 100644
--- a/OptiScaler/hooks/LibraryLoad_Hooks.cpp
+++ b/OptiScaler/hooks/LibraryLoad_Hooks.cpp
@@ -46,6 +46,18 @@ HMODULE LibraryLoadHooks::LoadLibraryCheckW(std::wstring libName, LPCWSTR lpLibF
{
auto libNameA = wstring_to_string(libName);
+ // The external D3D12 shim itself uses a locally supplied CUDA NGX core.
+ // Keep that exact absolute path distinct from game-facing NGX interception.
+ if (State::Instance().dlssExternalBackend)
+ {
+ wchar_t corePath[32768] {};
+ const auto length = GetEnvironmentVariableW(L"D4R_NGX_CORE", corePath, 32768);
+ if (length && length < 32768 &&
+ Util::ToLower(std::filesystem::path(libName).lexically_normal().wstring()) ==
+ Util::ToLower(std::filesystem::path(corePath).lexically_normal().wstring()))
+ return nullptr;
+ }
+
#ifdef LOG_LIB_OPERATIONS
LOG_TRACE("{}", libNameA);
#endif
@@ -132,6 +144,11 @@ HMODULE LibraryLoadHooks::LoadLibraryCheckW(std::wstring libName, LPCWSTR lpLibF
// NvApi64.dll
if (CheckDllNameW(&libName, &nvapiNamesW))
{
+ // An explicit external CUDA backend owns its NVAPI topology provider.
+ // Returning OptiScaler's fake-NVAPI would replace that provider and can
+ // recurse into DXGI while the NGX loader holds the loader lock.
+ if (State::Instance().dlssExternalBackend)
+ return nullptr; // Continue through the original loader.
LOG_INFO("{} call!", libNameA);
return LibraryLoadHooks::LoadNvApi();
diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
index 68e1b42..89c9ba8 100644
--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
@@ -687,6 +687,7 @@ static NVSDK_NGX_Result TryCreateOptiFeature(ID3D12GraphicsCommandList* InCmdLis
}
// Assign handle
+ const bool allocatedHandle = *OutHandle == nullptr;
if (*OutHandle == nullptr)
*OutHandle = new NVSDK_NGX_Handle { handleId };
else
@@ -705,6 +706,16 @@ static NVSDK_NGX_Result TryCreateOptiFeature(ID3D12GraphicsCommandList* InCmdLis
}
else
{
+ if (state.dlssExternalBackend && upscalerBackend == Upscaler::DLSS)
+ {
+ LOG_ERROR("External DLSS initialization failed; refusing FSR fallback");
+ if (shouldRestoreSigs)
+ D3D12Hooks::RestoreRoot(InCmdList);
+ D3D12Hooks::SetRootSignatureTracking(true);
+ Dx12Contexts.erase(handleId);
+ if (allocatedHandle) { delete *OutHandle; *OutHandle = nullptr; }
+ return NVSDK_NGX_Result_Fail;
+ }
LOG_ERROR("Feature '{}' initialization failed falling back to FSR 2.1.2", UpscalerDisplayName(upscalerBackend));
state.newBackend = Upscaler::FSR21;
state.changeBackend[handleId] = true;
@@ -1027,6 +1038,12 @@ static NVSDK_NGX_Result TryEvaluateOptiFeature(ID3D12GraphicsCommandList* InCmdL
// Fallback to FSR 2.1.2 if feature failed to initialize and user didn't explicitly request it
if (!feature->IsInited() && cfg.Dx12Upscaler.value_or_default() != Upscaler::FSR21)
{
+ if (state.dlssExternalBackend && cfg.Dx12Upscaler.value_or_default() == Upscaler::DLSS)
+ {
+ LOG_ERROR("External DLSS feature is not initialized; refusing FSR fallback");
+ D3D12Hooks::SetRootSignatureTracking(true);
+ return NVSDK_NGX_Result_Fail;
+ }
LOG_WARN("Feature '{}' failed to initialize. Falling back to FSR 2.1.2", feature->Name());
ImGui::InsertNotification({ ImGuiToastType::Warning, 10000, "Falling back to FSR 2.1.2" });
diff --git a/OptiScaler/misc/IdentifyGpu.cpp b/OptiScaler/misc/IdentifyGpu.cpp
index 3877f9f..6c05904 100644
--- a/OptiScaler/misc/IdentifyGpu.cpp
+++ b/OptiScaler/misc/IdentifyGpu.cpp
@@ -8,6 +8,7 @@
#include <proxies/D3d12_Proxy.h>
#include "nvapi/NvApiTypes.h"
#include <hooks/Amdxc64_Hooks.h>
+#include <proxies/NVNGX_Proxy.h>
using Microsoft::WRL::ComPtr;
@@ -187,6 +188,25 @@ std::vector<GpuInformation> IdentifyGpu::checkGpuInfo()
for (auto& gpuInfo : localCachedInfo)
{
+ if (State::Instance().dlssExternalBackend && !gpuInfo.softwareAdapter)
+ {
+ // Query the configured D3D12 backend outside DLL_PROCESS_ATTACH.
+ // Preserve vendor/device/architecture data obtained from DXGI.
+ auto requirements = NVNGXProxy::D3D12_GetFeatureRequirements();
+ ComPtr<IDXGIAdapter1> physicalAdapter;
+ if (requirements && SUCCEEDED(factory->EnumAdapterByLuid(gpuInfo.luid, IID_PPV_ARGS(&physicalAdapter))))
+ {
+ NVSDK_NGX_FeatureDiscoveryInfo discovery {};
+ discovery.SDKVersion = NVSDK_NGX_Version_API;
+ discovery.FeatureID = NVSDK_NGX_Feature_SuperSampling;
+ NVSDK_NGX_FeatureRequirement supported {};
+ const auto result = requirements(physicalAdapter.Get(), &discovery, &supported);
+ gpuInfo.dlssCapable = result == NVSDK_NGX_Result_Success &&
+ supported.FeatureSupported == NVSDK_NGX_FeatureSupportResult_Supported;
+ LOG_INFO("External D3D12 DLSS backend: {} support={} result={:X}",
+ gpuInfo.name, gpuInfo.dlssCapable, (unsigned) result);
+ }
+ }
if (gpuInfo.vendorId == VendorId::Nvidia)
{
queryNvapi(gpuInfo);
diff --git a/OptiScaler/proxies/NVNGX_Proxy.h b/OptiScaler/proxies/NVNGX_Proxy.h
index f6c08e5..7892009 100644
--- a/OptiScaler/proxies/NVNGX_Proxy.h
+++ b/OptiScaler/proxies/NVNGX_Proxy.h
@@ -500,7 +500,10 @@ class NVNGXProxy
{
LOG_INFO("Getting nvngx method addresses");
- HookNgxApi(_module.dll);
+ // The external bridge reports physical adapter support itself.
+ // Do not overwrite its unsupported result with NVIDIA spoofing.
+ if (!State::Instance().dlssExternalBackend)
+ HookNgxApi(_module.dll);
_module.D3D11_Init =
(PFN_D3D11_Init) KernelBaseProxy::GetProcAddress_()(_module.dll, "NVSDK_NGX_D3D11_Init");
diff --git a/OptiScaler/upscalers/FeatureProvider_Dx12.cpp b/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
index d738498..1191b29 100644
--- a/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
+++ b/OptiScaler/upscalers/FeatureProvider_Dx12.cpp
@@ -42,6 +42,11 @@ bool FeatureProvider_Dx12::GetFeature(Upscaler upscaler, UINT handleId, NVSDK_NG
break;
case Upscaler::DLSS:
+ if (state.dlssExternalBackend && (!primaryGpu.dlssCapable || !state.NVNGX_DLSS_Path.has_value()))
+ {
+ LOG_ERROR("Configured external D3D12 DLSS backend is unavailable; refusing FSR fallback");
+ return false;
+ }
if (primaryGpu.dlssCapable && state.NVNGX_DLSS_Path.has_value())
{
*feature = std::make_unique<DLSSFeatureDx12>(handleId, parameters);
@@ -77,6 +82,11 @@ bool FeatureProvider_Dx12::GetFeature(Upscaler upscaler, UINT handleId, NVSDK_NG
if (!loaded)
{
+ if (state.dlssExternalBackend && upscaler == Upscaler::DLSS)
+ {
+ LOG_ERROR("External D3D12 DLSS module failed to load; refusing FSR fallback");
+ return false;
+ }
// Fail after the constructor
ImGui::InsertNotification({ ImGuiToastType::Warning, 10000, "Falling back to FSR 2.1.2" });
*feature = std::make_unique<FSR2FeatureDx12_212>(handleId, parameters);
--
2.55.0.windows.5
patches/optiscaler/0002-early-d3d12-command-backend.patch
+45
View File
@@ -0,0 +1,45 @@
From 7ed1f1d66b430ba35a197fbd9880cd6a76af8f06 Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 16:45:01 +0300
Subject: [PATCH] Install the external d4r command backend before NGX
initialization
---
OptiScaler/hooks/D3D12_Hooks.cpp | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/OptiScaler/hooks/D3D12_Hooks.cpp b/OptiScaler/hooks/D3D12_Hooks.cpp
index 29c0bec..3b757be 100644
--- a/OptiScaler/hooks/D3D12_Hooks.cpp
+++ b/OptiScaler/hooks/D3D12_Hooks.cpp
@@ -14,6 +14,7 @@
#include <proxies/IGDExt_Proxy.h>
#include <proxies/Streamline_Proxy.h>
#include <proxies/KernelBase_Proxy.h>
+#include <proxies/NVNGX_Proxy.h>
#include <detours/detours.h>
@@ -2230,6 +2231,19 @@ static void HookToDevice(ID3D12Device* InDevice)
if (Util::CheckForRealObject(__FUNCTION__, InDevice, (IUnknown**) &realDevice))
pVTable = *(PVOID**) realDevice;
+ if (State::Instance().dlssExternalBackend)
+ {
+ // Capture command signatures/resources created before the game's NGX
+ // Init. Install before adding OptiScaler's own method trampolines.
+ using InstallCommandBackend = unsigned (*)(ID3D12Device*);
+ auto install = reinterpret_cast<InstallCommandBackend>(KernelBaseProxy::GetProcAddress_()(
+ NVNGXProxy::NVNGXModule(), "d4r_D3D12_InstallCommandBackend"));
+ auto result = install ? install(realDevice ? realDevice : InDevice) : 0;
+ LOG_INFO("External d4r command backend installed at device discovery: result={:X}", result);
+ if (result != 1)
+ LOG_ERROR("External d4r early command tracking failed; DLSS initialization must be checked");
+ }
+
// hudless
o_D3D12DeviceRelease = (PFN_Release) pVTable[2];
o_CreateSampler = (PFN_CreateSampler) pVTable[22];
--
2.55.0.windows.5
patches/optiscaler/0003-legacy-project-initialization.patch
+37
View File
@@ -0,0 +1,37 @@
From 09488c47855aadbf85f68ab8c72c0f3ac5f915e9 Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 17:01:32 +0300
Subject: [PATCH] Preserve legacy ProjectID initialization in the external
backend
---
OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
index 89c9ba8..1f723f1 100644
--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
@@ -313,6 +313,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_ProjectID(const char* InProj
if (result == NVSDK_NGX_Result_Success)
NVNGXProxy::SetDx12Inited(true);
+ else if (State::Instance().dlssExternalBackend)
+ return result;
}
}
@@ -343,6 +345,10 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_with_ProjectID(
{
LOG_FUNC();
+ if (State::Instance().dlssExternalBackend)
+ return NVSDK_NGX_D3D12_Init_ProjectID(InProjectId, InEngineType, InEngineVersion,
+ InApplicationDataPath, InDevice, InSDKVersion, InFeatureInfo);
+
LOG_INFO("InProjectId: {0}", InProjectId);
LOG_INFO("InEngineType: {0}", (int) InEngineType);
LOG_INFO("InEngineVersion: {0}", InEngineVersion);
--
2.55.0.windows.5
patches/optiscaler/0004-serialize-device-hook-installation.patch
+34
View File
@@ -0,0 +1,34 @@
diff --git a/OptiScaler/hooks/D3D12_Hooks.cpp b/OptiScaler/hooks/D3D12_Hooks.cpp
index 3b757be..05fc194 100644
--- a/OptiScaler/hooks/D3D12_Hooks.cpp
+++ b/OptiScaler/hooks/D3D12_Hooks.cpp
@@ -2217,11 +2217,20 @@ static HRESULT hkD3D12GetInterface(REFCLSID rclsid, REFIID riid, void** ppvDebug
return result;
}
+// Device discovery and NGX initialization can overlap. Installing d4r's
+// command hooks can also create a device and re-enter this function. Never
+// overwrite live Detours trampolines from a second installation attempt.
+static std::recursive_mutex deviceHookMutex;
+static bool deviceHookInstalling = false;
static void HookToDevice(ID3D12Device* InDevice)
{
- if (o_CreateSampler != nullptr || InDevice == nullptr)
+ std::lock_guard<std::recursive_mutex> lock(deviceHookMutex);
+ if (o_CreateSampler != nullptr || InDevice == nullptr || deviceHookInstalling)
return;
+ deviceHookInstalling = true;
+ struct InstallationGuard { ~InstallationGuard() { deviceHookInstalling = false; } } installationGuard;
+
LOG_DEBUG("Dx12");
// Get the vtable pointer
@@ -2334,6 +2343,7 @@ static void HookToDevice(ID3D12Device* InDevice)
static void UnhookDevice()
{
+ std::lock_guard<std::recursive_mutex> lock(deviceHookMutex);
LOG_FUNC();
DetourTransactionBegin();
patches/optiscaler/0005-forward-numeric-init-errors.patch
+22
View File
@@ -0,0 +1,22 @@
diff --git a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
index 1f723f1..ef5ec77 100644
--- a/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
+++ b/OptiScaler/inputs/NVNGX_DLSS_Dx12.cpp
@@ -184,6 +184,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init_Ext(unsigned long long InApp
if (result == NVSDK_NGX_Result_Success)
NVNGXProxy::SetDx12Inited(true);
+ else if (State::Instance().dlssExternalBackend)
+ return result;
}
else
{
@@ -254,6 +256,8 @@ NVSDK_NGX_API NVSDK_NGX_Result NVSDK_NGX_D3D12_Init(unsigned long long InApplica
if (result == NVSDK_NGX_Result_Success)
NVNGXProxy::SetDx12Inited(true);
+ else if (State::Instance().dlssExternalBackend)
+ return result;
}
}
patches/optiscaler/0006-relative-d4r-library-paths.patch
+30
View File
@@ -0,0 +1,30 @@
Subject: [PATCH] Resolve relative d4r library paths against the OptiScaler folder
Lets a drag-in d4r install ship OptiScaler.ini with NvngxPath=d4r instead of
an absolute path that only an installer could know. Absolute paths behave
exactly as before.
---
diff --git a/OptiScaler/Config.cpp b/OptiScaler/Config.cpp
--- a/OptiScaler/Config.cpp
+++ b/OptiScaler/Config.cpp
@@ -787,9 +787,17 @@
{
MainDllPath.set_from_config(readWString("Libraries", "OptiDllPath"));
- NvngxPath.set_from_config(readWString("Libraries", "NvngxPath"));
- NVNGX_DLSS_Library.set_from_config(readWString("Libraries", "NvngxDlssPath"));
- DLSSFeaturePath.set_from_config(readWString("Libraries", "NvngxFeaturePath"));
+ // A relative library folder (d4r drag-in installs use "d4r") is resolved
+ // against this DLL's folder, as LogFileName is. Absolute paths are unchanged.
+ auto libraryPath = [](std::optional<std::wstring> value) -> std::optional<std::wstring>
+ {
+ if (!value.has_value() || value->empty() || std::filesystem::path(*value).is_absolute())
+ return value;
+ return (Util::DllPath().parent_path() / *value).lexically_normal().wstring();
+ };
+ NvngxPath.set_from_config(libraryPath(readWString("Libraries", "NvngxPath")));
+ NVNGX_DLSS_Library.set_from_config(libraryPath(readWString("Libraries", "NvngxDlssPath")));
+ DLSSFeaturePath.set_from_config(libraryPath(readWString("Libraries", "NvngxFeaturePath")));
NvapiDllPath.set_from_config(readWString("Libraries", "NvapiPath"));
FfxDx12Path.set_from_config(readWString("Libraries", "FfxDx12Path"));
patches/optiscaler/README.md
+62
View File
@@ -0,0 +1,62 @@
# Explicit Windows d4r backend
Base: OptiScaler `45a2001303ddff632e279f77aef85ceede5832cb` (2026-09-30).
`0004` serializes D3D12 device hook installation/removal and guards recursive
device discovery while the d4r command backend installs. The previous frontend
can start two installations, fail a Detours transaction with `0x10dd`, clear
live original-function pointers and call null. This was reproduced in the
numeric-ID standalone harness; no game was required.
`0005` propagates numeric Init/Init_Ext failures for the explicitly selected
external backend, matching the existing ProjectID behavior. FeatureNotFound
must fail initialization instead of being hidden until repeated CreateFeature
attempts. `Libraries.NvngxDlssPath` takes a directory, not the DLL filename;
the Windows installer and standalone runner now use that directory.
The modifications to OptiScaler are GPL-3.0, matching its upstream license.
No NVIDIA DLL is supplied or patched.
`0001` introduces `[DLSS] AllowExternalBackend=true`, default false. Set
`[Libraries] NvngxPath` to the absolute directory containing the Windows d4r
shim as `_nvngx.dll`. Discovery requires its GPU-free ABI-1 marker. Physical
adapter support is queried through the bridge's D3D12 requirements API after
DLL startup; vendor, LUID and HIP architecture are retained.
The configured `D4R_NGX_CORE` absolute path bypasses game-facing NGX DLL
redirection. The external backend loads its own NVAPI provider. An explicitly
selected external DLSS backend reports initialization/load errors instead of
silently switching to FSR. The patch fixes the upstream loader's filesystem
exception by making IsSubpath a lexical check, and retains a PDB for diagnosis.
`0002` installs the command backend at D3D12 device discovery, before NGX Init
and before OptiScaler's own method hooks. Command signatures created during
game startup therefore retain their public argument descriptions for replay.
The standalone runner's `-EarlyIndirectProbe` exercises this startup order.
`0003` preserves the Project ID and engine identity through OptiScaler's
older `Init_with_ProjectID` entry point used by its XeSS/FSR inputs. It also
propagates an explicit external backend initialization failure. Reproduce
with the D3D12 runner's `-NgxAbi project-legacy`.
Build with Microsoft Build Tools 2022, v143 and Windows SDK 10.0.26100:
```powershell
.\scripts\windows\build-optiscaler-windows.ps1
```
The script clones the pinned source if absent, checks/applies the patch,
initializes pinned submodules, invokes MSBuild and writes the DLL, PDB, license
and build metadata into `dist/optiscaler-windows-d4r`. Existing source is not
reset. The upstream packaging post-build event is disabled for this build.
The official bootstrapper is https://aka.ms/vs/17/release/vs_buildtools.exe;
install the `Microsoft.VisualStudio.Workload.VCTools`,
`Microsoft.VisualStudio.Component.VC.Tools.x86.x64` and
`Microsoft.VisualStudio.Component.Windows11SDK.26100` components. The audited
local build uses MSVC 14.44.35207 / Build Tools 17.14.
For standalone verification add
`-OptiScalerDll <absolute-dist-path>/OptiScaler.dll -CaptureExceptions`
to the documented K/M runner. Its private INI selects DLSS, disables frame
generation and keeps RestoreComputeSignature, RestoreGraphicSignature and
ExtendedStateRestore false. Use `-RequireNativeNetwork`; finite output alone
cannot distinguish DLSS from an unwanted FSR replacement.
patches/rocm/0001-external-memory-view-ownership.patch
+25
View File
@@ -0,0 +1,25 @@
From: d4r Windows port
Subject: [PATCH] Backport AMD external-memory view ownership fix to ROCm 7.2
Upstream fix: ROCm/clr 529f6b1641de436dafbb095d5438b0fbf773765d
(ROCm/rocm-systems #3528). Already present in current upstream.
The constructor owns reference 1. MemObjMap does not retain the view;
hipFree removes the entry and releases reference 1. The extra retain
leaks the view and its parent imported allocation.
Hardware regression: d4r_d3d12_hip_interop_probe --interop-mode map
--iterations 64. Windows SDK 7.2.60201 leaks one HANDLE / 256 KiB each
cycle; tested TheRock 10.2.0a20260929 (HIP 7.17.26386) stays bounded.
This patch is for rebuilding old runtime sources, never binary patching.
diff --git a/hipamd/src/hip_memory.cpp b/hipamd/src/hip_memory.cpp
--- a/hipamd/src/hip_memory.cpp
+++ b/hipamd/src/hip_memory.cpp
@@ -179,7 +179,6 @@ hipError_t hipExternalMemoryGetMappedBuffer(void** devPtr, hipExternalMemory_t extMem,
// Map the device memory to the user pointer
*devPtr = reinterpret_cast<void*>(devMem->virtualAddress());
amd::MemObjMap::AddMemObj(*devPtr, view);
- view->retain();
HIP_RETURN(hipSuccess);
}
patches/vkd3d-proton/0001-split-command-list-for-external-wait.patch
+304
View File
@@ -0,0 +1,304 @@
diff --git a/include/vkd3d_device_vkd3d_ext.idl b/include/vkd3d_device_vkd3d_ext.idl
index 680f2c96..1fddc28e 100644
--- a/include/vkd3d_device_vkd3d_ext.idl
+++ b/include/vkd3d_device_vkd3d_ext.idl
@@ -104,6 +104,21 @@ interface ID3D12DXVKInteropDevice3 : ID3D12DXVKInteropDevice2
UINT32 *vk_memory_type);
}
+[
+ uuid(5a7c8b3e-2f61-4d0e-9c1a-7e3b52d4a901),
+ object,
+ local,
+ pointer_default(unique)
+]
+interface ID3D12DXVKInteropDeviceD4R : ID3D12DXVKInteropDevice3
+{
+ /* d4r: ends the command list's current Vulkan command buffer. Everything
+ * recorded afterwards is submitted separately and waits (as a queue
+ * submission wait, not a GPU-side wait) until the timeline semaphore
+ * vk_semaphore reaches value. */
+ HRESULT SplitCommandListForExternalWait(ID3D12CommandList *list, UINT64 vk_semaphore, UINT64 value);
+}
+
[
uuid(f3112584-41f9-348d-a59b-00b7e1d285d6),
object,
diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
index 611e373b..23af78a5 100644
--- a/libs/vkd3d/command.c
+++ b/libs/vkd3d/command.c
@@ -2557,6 +2557,38 @@ static void d3d12_command_list_consider_new_sequence(struct d3d12_command_list *
}
}
+HRESULT d3d12_command_list_split_for_external_wait(struct d3d12_command_list *list,
+ VkSemaphore semaphore, uint64_t value)
+{
+ struct d3d12_command_list_iteration *iteration;
+ unsigned int count;
+
+ if (!list->is_recording)
+ return E_INVALIDARG;
+ if (!d3d12_command_list_allows_new_sequence(list))
+ {
+ WARN("Cannot split command list for external wait with a running query.\n");
+ return E_FAIL;
+ }
+
+ /* Flush everything recorded so far into the current command buffer. */
+ d3d12_command_list_decay_tracked_state(list);
+ d3d12_command_list_invalidate_all_state(list);
+
+ count = list->cmd.iteration_count;
+ d3d12_command_list_begin_new_sequence(list, false);
+ if (list->cmd.iteration_count == count)
+ {
+ WARN("Out of command list sequences for external wait.\n");
+ return E_FAIL;
+ }
+
+ iteration = &list->cmd.iterations[list->cmd.iteration_count - 1];
+ iteration->external_wait_semaphore = semaphore;
+ iteration->external_wait_value = value;
+ return S_OK;
+}
+
static void d3d12_command_list_copy_queue_fallback(struct d3d12_command_list *list)
{
if (!list->cmd.iterations[list->cmd.iteration_count - 1].fallback)
@@ -22769,6 +22801,7 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
struct vkd3d_initial_transition *transitions;
size_t num_transitions, num_command_buffers;
VkCommandBufferSubmitInfo *buffers, *buffer;
+ VkSemaphoreSubmitInfo *external_waits = NULL;
struct d3d12_command_allocator **allocators;
struct d3d12_command_queue_submission sub;
unsigned int indirect_barrier_hoist_index;
@@ -22954,6 +22987,7 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
vkd3d_free(allocators);
vkd3d_free(buffers);
vkd3d_free(cmd_cost);
+ vkd3d_free(external_waits);
#ifdef VKD3D_ENABLE_BREADCRUMBS
vkd3d_free(breadcrumb_indices);
#endif
@@ -23036,6 +23070,21 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
buffer->commandBuffer = cmd_list->cmd.iterations[iter].vk_command_buffer;
/* Pilfer this for purposes of signalling aux data. No need to allocate more stuff just to signal this. */
buffer->deviceMask = cmd_list->cmd.iterations[iter].fallback ? VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_FALLBACK_QUEUE : 0;
+
+ if (cmd_list->cmd.iterations[iter].external_wait_semaphore)
+ {
+ if (!external_waits)
+ external_waits = vkd3d_calloc(num_command_buffers, sizeof(*external_waits));
+ if (external_waits)
+ {
+ VkSemaphoreSubmitInfo *wait = &external_waits[cmd_submit_count - 1];
+ wait->sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
+ wait->semaphore = cmd_list->cmd.iterations[iter].external_wait_semaphore;
+ wait->value = cmd_list->cmd.iterations[iter].external_wait_value;
+ wait->stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
+ buffer->deviceMask |= VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_EXTERNAL_WAIT;
+ }
+ }
}
need_suspend_fixup = cmd_list->cmd.suspend_resume.suspend.vk_fixup_cmd_buffer && (i + 1 == command_list_count ||
@@ -23155,6 +23204,7 @@ static void STDMETHODCALLTYPE d3d12_command_queue_ExecuteCommandLists(ID3D12Comm
sub.type = VKD3D_SUBMISSION_EXECUTE;
sub.execute.cmd = buffers;
sub.execute.cmd_cost = cmd_cost;
+ sub.execute.external_waits = external_waits;
sub.execute.cmd_count = cmd_submit_count;
sub.execute.command_allocators = allocators;
sub.execute.num_command_allocators = command_list_count;
@@ -24377,6 +24427,8 @@ static void d3d12_command_queue_execute(struct d3d12_command_queue *command_queu
VkSemaphoreSubmitInfo *binary_semaphore_info;
bool stagger_submissions, is_first, is_last;
VkSemaphoreSubmitInfo wait_semaphore_info;
+ VkSemaphoreSubmitInfo combined_wait_infos[2];
+ bool external_wait;
uint32_t cmd_index, cmd_count, total_cost;
struct vkd3d_fence_wait_info fence_info;
VkSubmitInfo2 submit_desc[2], *submit;
@@ -24516,7 +24568,15 @@ static void d3d12_command_queue_execute(struct d3d12_command_queue *command_queu
total_cost = 0;
for (i = cmd_index + cmd_count; i < exec->cmd_count && total_cost < VKD3D_COMMAND_COST_MERGE_THRESHOLD; i++)
+ {
+ /* d4r: never merge across an external wait. */
+ if (exec->cmd[i].deviceMask & VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_EXTERNAL_WAIT)
+ {
+ total_cost = VKD3D_COMMAND_COST_MERGE_THRESHOLD;
+ break;
+ }
total_cost += exec->cmd_cost[i];
+ }
if (total_cost < VKD3D_COMMAND_COST_MERGE_THRESHOLD)
cmd_count = exec->cmd_count - cmd_index;
@@ -24525,8 +24585,14 @@ static void d3d12_command_queue_execute(struct d3d12_command_queue *command_queu
{
/* Submit everything at once, until we hit a fallback queue submission. Pick that up next iteration. */
for (cmd_count = 0; cmd_index + cmd_count < exec->cmd_count; cmd_count++)
+ {
if (exec->cmd[cmd_index + cmd_count].deviceMask & VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_FALLBACK_QUEUE)
break;
+ /* d4r: an external wait starts a new submit. */
+ if (cmd_count && (exec->cmd[cmd_index + cmd_count].deviceMask &
+ VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_EXTERNAL_WAIT))
+ break;
+ }
}
submit = &submit_desc[num_submits++];
@@ -24536,6 +24602,9 @@ static void d3d12_command_queue_execute(struct d3d12_command_queue *command_queu
submit->signalSemaphoreInfoCount = 1;
submit->pSignalSemaphoreInfos = signal_semaphore_infos;
+ external_wait = cmd_count && exec->external_waits &&
+ (exec->cmd[cmd_index].deviceMask & VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_EXTERNAL_WAIT);
+
/* Clear out any shenanigans we added due to fallback submit. */
for (i = 0; i < cmd_count; i++)
exec->cmd[cmd_index + i].deviceMask = 0;
@@ -24555,6 +24624,23 @@ static void d3d12_command_queue_execute(struct d3d12_command_queue *command_queu
submit->pWaitSemaphoreInfos = &wait_semaphore_info;
}
+ if (external_wait)
+ {
+ /* d4r: a queue-level wait-before-signal; the host signals the value later. */
+ if (submit->waitSemaphoreInfoCount)
+ {
+ combined_wait_infos[0] = submit->pWaitSemaphoreInfos[0];
+ combined_wait_infos[1] = exec->external_waits[cmd_index];
+ submit->waitSemaphoreInfoCount = 2;
+ submit->pWaitSemaphoreInfos = combined_wait_infos;
+ }
+ else
+ {
+ submit->waitSemaphoreInfoCount = 1;
+ submit->pWaitSemaphoreInfos = &exec->external_waits[cmd_index];
+ }
+ }
+
cmd_index += cmd_count;
is_last = cmd_index == exec->cmd_count;
@@ -25347,6 +25433,7 @@ static void *d3d12_command_queue_submission_worker_main(void *userdata)
* On error, the counters are freed early, so there is no risk of leak. */
vkd3d_free(submission.execute.cmd);
vkd3d_free(submission.execute.cmd_cost);
+ vkd3d_free(submission.execute.external_waits);
vkd3d_free(submission.execute.transitions);
#ifdef VKD3D_ENABLE_BREADCRUMBS
for (i = 0; i < submission.execute.breadcrumb_indices_count; i++)
diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c
index f6db6954..06955941 100644
--- a/libs/vkd3d/device.c
+++ b/libs/vkd3d/device.c
@@ -4657,7 +4657,8 @@ HRESULT STDMETHODCALLTYPE d3d12_device_QueryInterface(d3d12_device_iface *iface,
if (IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice)
|| IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice1)
|| IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice2)
- || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice3))
+ || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice3)
+ || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R))
{
d3d12_dxvk_interop_device_AddRef(&device->ID3D12DXVKInteropDevice_iface);
*object = &device->ID3D12DXVKInteropDevice_iface;
@@ -11379,7 +11380,7 @@ static void d3d12_device_replace_vtable(struct d3d12_device *device)
}
extern CONST_VTBL struct ID3D12DeviceExt5Vtbl d3d12_device_vkd3d_ext_vtbl;
-extern CONST_VTBL struct ID3D12DXVKInteropDevice3Vtbl d3d12_dxvk_interop_device_vtbl;
+extern CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl;
extern CONST_VTBL struct ID3DLowLatencyDeviceVtbl d3d_low_latency_device_vtbl;
extern CONST_VTBL struct IAmdExtAntiLagApiVtbl d3d_amd_ext_anti_lag_vtbl;
diff --git a/libs/vkd3d/device_vkd3d_ext.c b/libs/vkd3d/device_vkd3d_ext.c
index 94818b9a..bd5dd7a5 100644
--- a/libs/vkd3d/device_vkd3d_ext.c
+++ b/libs/vkd3d/device_vkd3d_ext.c
@@ -1145,7 +1145,19 @@ static HRESULT STDMETHODCALLTYPE d3d12_dxvk_interop_device_GetVulkanHeapInfo(d3d
return S_OK;
}
-CONST_VTBL struct ID3D12DXVKInteropDevice3Vtbl d3d12_dxvk_interop_device_vtbl =
+static HRESULT STDMETHODCALLTYPE d3d12_dxvk_interop_device_SplitCommandListForExternalWait(
+ d3d12_dxvk_interop_device_iface *iface, ID3D12CommandList *list, UINT64 vk_semaphore, UINT64 value)
+{
+ struct d3d12_command_list *cmd_list = d3d12_command_list_from_iface(list);
+
+ TRACE("iface %p, list %p, vk_semaphore 0x%"PRIx64", value %"PRIu64".\n", iface, list, vk_semaphore, value);
+
+ if (!cmd_list || !vk_semaphore)
+ return E_INVALIDARG;
+ return d3d12_command_list_split_for_external_wait(cmd_list, (VkSemaphore)vk_semaphore, value);
+}
+
+CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl =
{
/* IUnknown methods */
d3d12_dxvk_interop_device_QueryInterface,
@@ -1177,6 +1189,9 @@ CONST_VTBL struct ID3D12DXVKInteropDevice3Vtbl d3d12_dxvk_interop_device_vtbl =
/* ID3D12DXVKInteropDevice3 methods */
d3d12_dxvk_interop_device_GetVulkanHeapInfo,
+
+ /* ID3D12DXVKInteropDeviceD4R methods */
+ d3d12_dxvk_interop_device_SplitCommandListForExternalWait,
};
static inline struct d3d12_device *d3d12_device_from_ID3DLowLatencyDevice(d3d_low_latency_device_iface *iface)
diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
index a2a78000..64338585 100644
--- a/libs/vkd3d/vkd3d_private.h
+++ b/libs/vkd3d/vkd3d_private.h
@@ -3207,6 +3207,9 @@ struct d3d12_command_list_iteration
uint32_t estimated_cost;
bool fallback; /* Only applies to vk_command_buffer. */
struct d3d12_command_list_iteration_indirect_meta indirect_meta;
+ /* d4r: vk_command_buffer is submitted separately, after this timeline value is reached. */
+ VkSemaphore external_wait_semaphore;
+ uint64_t external_wait_value;
};
enum
@@ -3216,9 +3219,12 @@ enum
* without having to add more allocations. */
VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_DEFAULT = 0,
VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_FALLBACK_QUEUE = 1u << 0,
+ /* d4r: starts a new submit that waits for submission_execute::external_waits[index]. */
+ VKD3D_COMMAND_BUFFER_SUBMIT_INFO_DEVICE_MASK_EXTERNAL_WAIT = 1u << 1,
};
-#define VKD3D_MAX_COMMAND_LIST_SEQUENCES 2
+/* d4r: 4 so an external-wait split fits next to DGC and fallback splits. */
+#define VKD3D_MAX_COMMAND_LIST_SEQUENCES 4
#define VKD3D_COMMAND_COST_LOW (1u)
#define VKD3D_COMMAND_COST_HIGH (16u)
@@ -3737,6 +3743,8 @@ struct d3d12_command_queue_submission_execute
{
VkCommandBufferSubmitInfo *cmd;
uint32_t *cmd_cost;
+ /* d4r: indexed like cmd, only allocated when a command buffer waits externally. */
+ VkSemaphoreSubmitInfo *external_waits;
struct d3d12_command_allocator **command_allocators;
UINT cmd_count;
UINT num_command_allocators;
@@ -5463,7 +5471,10 @@ struct vkd3d_timestamp_profiler;
typedef ID3D12DeviceExt5 d3d12_device_vkd3d_ext_iface;
/* ID3D12DXVKInteropDevice */
-typedef ID3D12DXVKInteropDevice3 d3d12_dxvk_interop_device_iface;
+HRESULT d3d12_command_list_split_for_external_wait(struct d3d12_command_list *list,
+ VkSemaphore semaphore, uint64_t value);
+
+typedef ID3D12DXVKInteropDeviceD4R d3d12_dxvk_interop_device_iface;
/* ID3DLowLatencyDevice */
typedef ID3DLowLatencyDevice d3d_low_latency_device_iface;
patches/vkd3d-proton/0002-retain-external-resources.patch
+148
View File
@@ -0,0 +1,148 @@
diff --git a/include/vkd3d_device_vkd3d_ext.idl b/include/vkd3d_device_vkd3d_ext.idl
--- a/include/vkd3d_device_vkd3d_ext.idl
+++ b/include/vkd3d_device_vkd3d_ext.idl
@@ -120,6 +120,19 @@
}
[
+ uuid(4f8c9462-8197-4dc1-a584-cf44d9706db2),
+ object,
+ local,
+ pointer_default(unique)
+]
+interface ID3D12DXVKInteropDeviceD4R2 : ID3D12DXVKInteropDeviceD4R
+{
+ /* Retain external resources until the allocator can safely reset or is
+ * destroyed. Pending submissions already retain the allocator. */
+ HRESULT RetainExternalResources(ID3D12CommandList *list, IUnknown *resources);
+}
+
+[
uuid(f3112584-41f9-348d-a59b-00b7e1d285d6),
object,
local,
diff --git a/libs/vkd3d/command.c b/libs/vkd3d/command.c
--- a/libs/vkd3d/command.c
+++ b/libs/vkd3d/command.c
@@ -2791,6 +2791,24 @@
}
}
+HRESULT d3d12_command_list_retain_external_resources(struct d3d12_command_list *list, IUnknown *resources)
+{
+ struct d3d12_command_allocator *allocator = list->allocator;
+ size_t i;
+
+ if (!list->is_recording || !allocator)
+ return E_INVALIDARG;
+ for (i = 0; i < allocator->external_resources_count; i++)
+ if (allocator->external_resources[i] == resources)
+ return S_OK;
+ if (!vkd3d_array_reserve((void **)&allocator->external_resources, &allocator->external_resources_size,
+ allocator->external_resources_count + 1, sizeof(*allocator->external_resources)))
+ return E_OUTOFMEMORY;
+ IUnknown_AddRef(resources);
+ allocator->external_resources[allocator->external_resources_count++] = resources;
+ return S_OK;
+}
+
static bool d3d12_command_allocator_add_view(struct d3d12_command_allocator *allocator,
struct vkd3d_view *view)
{
@@ -2858,6 +2876,10 @@
struct d3d12_device *device = allocator->device;
const struct vkd3d_vk_device_procs *vk_procs = &device->vk_procs;
unsigned int i;
+
+ for (i = 0; i < allocator->external_resources_count; ++i)
+ IUnknown_Release(allocator->external_resources[i]);
+ allocator->external_resources_count = 0;
for (i = 0; i < allocator->buffer_view_count; ++i)
{
@@ -2993,6 +3015,7 @@
d3d12_command_list_allocator_destroyed(allocator->current_command_list);
d3d12_command_allocator_free_resources(allocator);
+ vkd3d_free(allocator->external_resources);
vkd3d_free(allocator->buffer_views);
vkd3d_free(allocator->views);
vkd3d_free(allocator->pipelines);
diff --git a/libs/vkd3d/device.c b/libs/vkd3d/device.c
--- a/libs/vkd3d/device.c
+++ b/libs/vkd3d/device.c
@@ -4658,7 +4658,8 @@
|| IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice1)
|| IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice2)
|| IsEqualGUID(riid, &IID_ID3D12DXVKInteropDevice3)
- || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R))
+ || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R)
+ || IsEqualGUID(riid, &IID_ID3D12DXVKInteropDeviceD4R2))
{
d3d12_dxvk_interop_device_AddRef(&device->ID3D12DXVKInteropDevice_iface);
*object = &device->ID3D12DXVKInteropDevice_iface;
@@ -11380,7 +11381,7 @@
}
extern CONST_VTBL struct ID3D12DeviceExt5Vtbl d3d12_device_vkd3d_ext_vtbl;
-extern CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl;
+extern CONST_VTBL struct ID3D12DXVKInteropDeviceD4R2Vtbl d3d12_dxvk_interop_device_vtbl;
extern CONST_VTBL struct ID3DLowLatencyDeviceVtbl d3d_low_latency_device_vtbl;
extern CONST_VTBL struct IAmdExtAntiLagApiVtbl d3d_amd_ext_anti_lag_vtbl;
diff --git a/libs/vkd3d/device_vkd3d_ext.c b/libs/vkd3d/device_vkd3d_ext.c
--- a/libs/vkd3d/device_vkd3d_ext.c
+++ b/libs/vkd3d/device_vkd3d_ext.c
@@ -1157,7 +1157,16 @@
return d3d12_command_list_split_for_external_wait(cmd_list, (VkSemaphore)vk_semaphore, value);
}
-CONST_VTBL struct ID3D12DXVKInteropDeviceD4RVtbl d3d12_dxvk_interop_device_vtbl =
+static HRESULT STDMETHODCALLTYPE d3d12_dxvk_interop_device_RetainExternalResources(
+ d3d12_dxvk_interop_device_iface *iface, ID3D12CommandList *list, IUnknown *resources)
+{
+ struct d3d12_command_list *cmd_list = d3d12_command_list_from_iface(list);
+ if (!cmd_list || !resources)
+ return E_INVALIDARG;
+ return d3d12_command_list_retain_external_resources(cmd_list, resources);
+}
+
+CONST_VTBL struct ID3D12DXVKInteropDeviceD4R2Vtbl d3d12_dxvk_interop_device_vtbl =
{
/* IUnknown methods */
d3d12_dxvk_interop_device_QueryInterface,
@@ -1192,6 +1201,9 @@
/* ID3D12DXVKInteropDeviceD4R methods */
d3d12_dxvk_interop_device_SplitCommandListForExternalWait,
+
+ /* ID3D12DXVKInteropDeviceD4R2 methods */
+ d3d12_dxvk_interop_device_RetainExternalResources,
};
static inline struct d3d12_device *d3d12_device_from_ID3DLowLatencyDevice(d3d_low_latency_device_iface *iface)
diff --git a/libs/vkd3d/vkd3d_private.h b/libs/vkd3d/vkd3d_private.h
--- a/libs/vkd3d/vkd3d_private.h
+++ b/libs/vkd3d/vkd3d_private.h
@@ -2758,6 +2758,10 @@
LONG refcount;
LONG internal_refcount;
+ IUnknown **external_resources;
+ size_t external_resources_size;
+ size_t external_resources_count;
+
D3D12_COMMAND_LIST_TYPE type;
struct vkd3d_view **views;
@@ -5474,7 +5478,9 @@
HRESULT d3d12_command_list_split_for_external_wait(struct d3d12_command_list *list,
VkSemaphore semaphore, uint64_t value);
-typedef ID3D12DXVKInteropDeviceD4R d3d12_dxvk_interop_device_iface;
+HRESULT d3d12_command_list_retain_external_resources(struct d3d12_command_list *list, IUnknown *resources);
+
+typedef ID3D12DXVKInteropDeviceD4R2 d3d12_dxvk_interop_device_iface;
/* ID3DLowLatencyDevice */
typedef ID3DLowLatencyDevice d3d_low_latency_device_iface;
patches/zluda-llvm/0001-backport-gfx1154.patch
+128
View File
@@ -0,0 +1,128 @@
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index e619b186d..dd3e9d92b 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -856,7 +856,7 @@ enum : unsigned {
EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC = 0x054,
EF_AMDGPU_MACH_AMDGCN_GFX1152 = 0x055,
EF_AMDGPU_MACH_AMDGCN_RESERVED_0X56 = 0x056,
- EF_AMDGPU_MACH_AMDGCN_RESERVED_0X57 = 0x057,
+ EF_AMDGPU_MACH_AMDGCN_GFX1154 = 0x057,
EF_AMDGPU_MACH_AMDGCN_GFX1153 = 0x058,
EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC = 0x059,
EF_AMDGPU_MACH_AMDGCN_GFX1251 = 0x05a,
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index aad985926..c0aaa465b 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -109,6 +109,7 @@ enum GPUKind : uint32_t {
GK_GFX1151,
GK_GFX1152,
GK_GFX1153,
+ GK_GFX1154,
GK_GFX1200,
GK_GFX1201,
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index f9fda2346..70ddaae22 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -591,6 +591,8 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const {
return "gfx1152";
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153:
return "gfx1153";
+ case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154:
+ return "gfx1154";
// AMDGCN GFX12.
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200:
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index d9cce1eb7..7936876f4 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -632,6 +632,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1151, EF_AMDGPU_MACH);
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1152, EF_AMDGPU_MACH);
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1153, EF_AMDGPU_MACH);
+ BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1154, EF_AMDGPU_MACH);
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1200, EF_AMDGPU_MACH);
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1201, EF_AMDGPU_MACH);
BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1250, EF_AMDGPU_MACH);
diff --git a/llvm/lib/Target/AMDGPU/GCNProcessors.td b/llvm/lib/Target/AMDGPU/GCNProcessors.td
index c6fb31f9c..5d21894a3 100644
--- a/llvm/lib/Target/AMDGPU/GCNProcessors.td
+++ b/llvm/lib/Target/AMDGPU/GCNProcessors.td
@@ -304,7 +304,12 @@ def : ProcessorModel<"gfx1153", GFX11SpeedModel,
FeatureISAVersion11_5_3.Features
>;
-// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153]
+// Backported from LLVM 7a0829e41228: actual gfx1154 features and identity.
+def : ProcessorModel<"gfx1154", GFX11SpeedModel,
+ FeatureISAVersion11_5_Common.Features
+>;
+
+// [gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1152, gfx1153, gfx1154]
def : ProcessorModel<"gfx11-generic", GFX11SpeedModel,
FeatureISAVersion11_Generic.Features
>;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 5a0857322..c506fc308 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -114,6 +114,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) {
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151: AK = GK_GFX1151; break;
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152: AK = GK_GFX1152; break;
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153: AK = GK_GFX1153; break;
+ case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154: AK = GK_GFX1154; break;
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break;
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break;
case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250: AK = GK_GFX1250; break;
@@ -201,6 +202,7 @@ unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {
case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151;
case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152;
case GK_GFX1153: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153;
+ case GK_GFX1154: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154;
case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;
case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;
case GK_GFX1250: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250;
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 34b09b14b..39acbb942 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -172,6 +172,7 @@ constexpr GPUInfo AMDGCNGPUs[] = {
{{"gfx1151"}, {"gfx1151"}, GK_GFX1151, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
{{"gfx1152"}, {"gfx1152"}, GK_GFX1152, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
{{"gfx1153"}, {"gfx1153"}, GK_GFX1153, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+ {{"gfx1154"}, {"gfx1154"}, GK_GFX1154, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
{{"gfx1200"}, {"gfx1200"}, GK_GFX1200, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
{{"gfx1201"}, {"gfx1201"}, GK_GFX1201, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
{{"gfx1250"}, {"gfx1250"}, GK_GFX1250, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32},
@@ -328,6 +329,7 @@ AMDGPU::IsaVersion AMDGPU::getIsaVersion(StringRef GPU) {
case GK_GFX1151: return {11, 5, 1};
case GK_GFX1152: return {11, 5, 2};
case GK_GFX1153: return {11, 5, 3};
+ case GK_GFX1154: return {11, 5, 4};
case GK_GFX1200: return {12, 0, 0};
case GK_GFX1201: return {12, 0, 1};
case GK_GFX1250: return {12, 5, 0};
@@ -475,6 +477,7 @@ static void fillAMDGCNFeatureMap(StringRef GPU, const Triple &T,
Features["atomic-fmin-fmax-global-f32"] = true;
break;
case GK_GFX1153:
+ case GK_GFX1154:
case GK_GFX1152:
case GK_GFX1151:
case GK_GFX1150:
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index ab9331690..0ade2d412 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1655,6 +1655,7 @@ const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1151, "gfx1151"), \
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1152, "gfx1152"), \
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1153, "gfx1153"), \
+ ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1154, "gfx1154"), \
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"), \
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"), \
ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1250, "gfx1250"), \
patches/zluda-llvm/README.md
+24
View File
@@ -0,0 +1,24 @@
# LLVM source patches for the Windows compiler
Base: ZLUDA's LLVM submodule `ff4dc1f7c9e1c64d4d69e40f4ed30c2280a96dfd`
(LLVM 22). `build-zluda-llvm.ps1` applies these patches idempotently before
building. ZLUDA's parent source patches remain in `patches/zluda`.
`0001-backport-gfx1154.patch` adapts official LLVM
[`7a0829e41228`](https://github.com/llvm/llvm-project/commit/7a0829e41228513299c5108685b0bc127463c6a1)
to the older parser/ELF structures. It adds actual gfx1154 processor features,
ISA 11.5.4, parser identity and ELF machine 0x57. It does not map gfx1154 onto
another CPU, patch binary headers or alter existing processors' features.
LLVM's gfx1154 uses `FeatureISAVersion11_5_Common` and the GFX11 speed model.
LLVM code remains Apache-2.0 WITH LLVM-exception (see the dependency license).
The prior offline compiler aborts on a public PTX pattern for gfx1154 with
unrecognized-processor diagnostics and exit `0xC0000409`. After rebuilding
LLVM, rebuild/relink ZLUDA and its offline emitter; changing native Clang
objects alone cannot fix the translated PTX path. Run
`scripts/windows/test-zluda-target-compilation.ps1` to compile the public
Driver API pattern and FP16 MMA fixtures (strict/reference and WMMA modes)
for every target without GPU work. All 33 compilations pass with correct ELF
targets and wave32. Strict gfx12 F16 reference retains its prior FP64 helper;
WMMA mode here is a compilation check, not a default arithmetic change.
This checks code generation and ELF identity, not hardware execution/numerics.
patches/zluda/0002-dlss-runtime-cumulative.patch
+2768
View File
File diff suppressed because it is too large. Load diff
patches/zluda/0003-perf-implicit-bound-gfx11-wmma.patch
+772
View File
@@ -0,0 +1,772 @@
diff --git a/ptx/src/pass/mod.rs b/ptx/src/pass/mod.rs
index 15dd9ba..cff04dd 100644
--- a/ptx/src/pass/mod.rs
+++ b/ptx/src/pass/mod.rs
@@ -61,6 +61,42 @@ pub struct Attributes {
pub clock_rate: u32,
}
+/// Work-group size bound applied to kernels without `.maxntid`/`.reqntid`
+/// (D4R_ZLUDA_IMPLICIT_MAX_BLOCK, 1..=1024; unset keeps LLVM's 1024 default).
+///
+/// Without a bound, LLVM must assume 1024-thread work-groups, which on gfx11
+/// in CU mode caps each wave at 96 VGPRs and makes register-heavy kernels
+/// (e.g. DLSS's tensor-core convolutions) spill kilobytes to scratch. CUDA's
+/// own compiler gives unbounded kernels up to 255 registers, which likewise
+/// limits them to 256-thread blocks; a launch exceeding the bound fails
+/// (as it would on NVIDIA) instead of misbehaving.
+/// Optional AMDGPU machine-scheduler strategy for kernels
+/// (D4R_ZLUDA_SCHED_STRATEGY: max-ilp, max-memory-clause, iterative-ilp,
+/// iterative-minreg or iterative-maxocc); unset keeps LLVM's default.
+pub fn sched_strategy() -> Option<&'static str> {
+ static VALUE: std::sync::OnceLock<Option<String>> = std::sync::OnceLock::new();
+ VALUE
+ .get_or_init(|| {
+ std::env::var("D4R_ZLUDA_SCHED_STRATEGY").ok().filter(|value| {
+ matches!(
+ value.as_str(),
+ "max-ilp" | "max-memory-clause" | "iterative-ilp" | "iterative-minreg" | "iterative-maxocc"
+ )
+ })
+ })
+ .as_deref()
+}
+
+pub fn implicit_max_block() -> Option<u32> {
+ static VALUE: std::sync::OnceLock<Option<u32>> = std::sync::OnceLock::new();
+ *VALUE.get_or_init(|| {
+ std::env::var("D4R_ZLUDA_IMPLICIT_MAX_BLOCK")
+ .ok()
+ .and_then(|value| value.trim().parse::<u32>().ok())
+ .filter(|value| (1..=1024).contains(value))
+ })
+}
+
pub fn to_llvm_module<'input>(
ast: ast::Module<'input>,
attributes: Attributes,
diff --git a/ptx/src/lib.rs b/ptx/src/lib.rs
index 8daa637..79ed812 100644
--- a/ptx/src/lib.rs
+++ b/ptx/src/lib.rs
@@ -4,4 +4,6 @@ mod test;
pub use pass::to_llvm_module;
pub use pass::Attributes;
+pub use pass::implicit_max_block;
+pub use pass::sched_strategy;
pub use pass::TranslateError;
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 5ab8985..e891c34 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -147,6 +147,23 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
}
self.emit_target_features(fn_);
self.emit_tuning(fn_, &method.tuning);
+ if method.is_kernel()
+ && !method.tuning.iter().any(|tuning| {
+ matches!(
+ tuning,
+ ptx_parser::TuningDirective::MaxNtid(..) | ptx_parser::TuningDirective::ReqNtid(..)
+ )
+ })
+ {
+ if let Some(size) = crate::pass::implicit_max_block() {
+ self.emit_fn_attribute_string(fn_, "amdgpu-flat-work-group-size", &format!("1,{size}"));
+ }
+ }
+ if method.is_kernel() {
+ if let Some(strategy) = crate::pass::sched_strategy() {
+ self.emit_fn_attribute_string(fn_, "amdgpu-sched-strategy", strategy);
+ }
+ }
if let Some(kernel_attrs) = &method.kernel_attributes {
self.emit_fn_attribute_string(
fn_,
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index a6adeef..0fbc7f7 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -260,6 +260,9 @@ fn get_best_ptx_and_compile(
let attributes = ExtraCacheAttributes {
clock_rate: hip_properties.clockRate as u32,
is_debug: cfg!(debug_assertions),
+ implicit_max_block: ptx::implicit_max_block(),
+ sched_strategy: ptx::sched_strategy(),
+ wmma: llvm_zluda::compile::wmma_rewrite_requested(),
};
let mut cache_with_key = match (text, global_state.cache_path.as_ref()) {
(Some(text), Some(p)) => (|| {
@@ -296,6 +299,9 @@ pub(crate) fn load_hip_module(
struct ExtraCacheAttributes {
is_debug: bool,
clock_rate: u32,
+ implicit_max_block: Option<u32>,
+ sched_strategy: Option<&'static str>,
+ wmma: bool,
}
fn get_hip_properties<'a>() -> Result<hipDeviceProp_tR0600, CUerror> {
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index b80262b..1284cbc 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -87,6 +87,18 @@ fn create_oclc_constants(ctx: &Context, gcn_arch: &str) -> Result<Module, String
Ok(module)
}
+/// D4R_ZLUDA_WMMA=1 lowers f16 m16n8k16 MMAs to gfx11 WMMA (see
+/// LLVMZludaRewriteMmaF16Wmma); anything else keeps the dot-product fallback.
+pub fn wmma_rewrite_requested() -> bool {
+ static VALUE: OnceLock<bool> = OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WMMA").map_or(false, |value| value.trim() == "1"))
+}
+
+fn wmma_rewrite_enabled(gcn_arch: &str) -> bool {
+ // The helpers assume RDNA3's wave32 WMMA register layout.
+ wmma_rewrite_requested() && gcn_arch.starts_with("gfx11")
+}
+
pub fn compile(
ctx: &Context,
gcn_arch: &str,
@@ -106,6 +118,9 @@ pub fn compile(
let oclc_constants = create_oclc_constants(ctx, gcn_arch)?;
+ if wmma_rewrite_enabled(gcn_arch) {
+ unsafe { crate::ffi::LLVMZludaRewriteMmaF16Wmma(main.get()) };
+ }
linked.link(main)?;
linked.link(attributes)?;
linked.link(oclc_constants)?;
diff --git a/llvm_zluda/src/ffi.rs b/llvm_zluda/src/ffi.rs
index 7ca4289..6bf0d30 100644
--- a/llvm_zluda/src/ffi.rs
+++ b/llvm_zluda/src/ffi.rs
@@ -42,6 +42,9 @@ pub const LLVMZludaFastMathAll: ::std::ffi::c_uint = LLVMZludaFastMathAllowReass
pub type LLVMZludaFastMathFlags = std::ffi::c_uint;
extern "C" {
+ /// See LLVMZludaRewriteMmaF16Wmma in lib.cpp.
+ pub fn LLVMZludaRewriteMmaF16Wmma(M: LLVMModuleRef) -> u32;
+
pub fn LLVMZludaBuildAlloca(
B: LLVMBuilderRef,
Ty: LLVMTypeRef,
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 14b4f40..30d5490 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -9,6 +9,11 @@
#include <llvm/Support/raw_ostream.h>
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/raw_ostream.h"
+#include <llvm/ADT/DenseMap.h>
+#include <llvm/ADT/DenseSet.h>
+#include <llvm/ADT/SmallVector.h>
+#include <llvm/IR/Module.h>
+#include <llvm/IR/InstIterator.h>
#pragma GCC diagnostic pop
#include <mutex>
@@ -255,6 +260,367 @@ int LLVMZludaLinkWithLLD(uint32_t inputs, const char **input_paths, const char *
return 0;
}
+// gfx11 WMMA lowering of mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16,
+// run on ZLUDA's emitted module before linking, while PTX registers are still
+// allocas. Each MMA becomes
+// d = d4r_wmma_t_to_nv(d4r_wmma_t_mma_f16(a_from_b(B), b_from_a(A), C'))
+// (see zluda_ptx_impl.cpp for the layouts). Register pairs used as MMA
+// accumulators get a shadow alloca holding the pair in WMMA layout: an MMA
+// whose C is a plain load of such a pair reads the shadow, an MMA whose D is
+// stored to it writes the shadow, and any other store to either register
+// refreshes the shadow with d4r_wmma_t_from_nv. The invariant "shadow ==
+// WMMA layout of the pair" thus holds at every program point, and once
+// mem2reg promotes the allocas the layout conversions between chained MMAs
+// become dead code. Returns the number of MMAs rewritten.
+namespace
+{
+constexpr const char *kMmaF16 = "__zluda_ptx_impl_mma_sync_aligned_m16n8k16_row_col_f16_f16_f16_f16";
+
+struct RegisterPair
+{
+ AllocaInst *Low = nullptr;
+ AllocaInst *High = nullptr;
+ bool operator==(const RegisterPair &Other) const { return Low == Other.Low && High == Other.High; }
+};
+
+// Only allocas used purely as the address of loads and stores are tracked.
+static bool isPlainRegister(AllocaInst *Alloca)
+{
+ for (User *U : Alloca->users())
+ {
+ if (isa<LoadInst>(U))
+ continue;
+ auto *Store = dyn_cast<StoreInst>(U);
+ if (Store == nullptr || Store->getPointerOperand() != Alloca)
+ return false;
+ }
+ return true;
+}
+
+// Matches insertelement(insertelement(undef, load Low, 0), load High, 1) where
+// the loads sit in the call's block with no store to either register between
+// them and the call.
+static bool matchLoadedPair(Value *Vector, CallInst *Call, RegisterPair &Pair)
+{
+ auto *Second = dyn_cast<InsertElementInst>(Vector);
+ if (Second == nullptr)
+ return false;
+ auto *First = dyn_cast<InsertElementInst>(Second->getOperand(0));
+ auto *SecondIndex = dyn_cast<ConstantInt>(Second->getOperand(2));
+ if (First == nullptr || SecondIndex == nullptr || SecondIndex->getZExtValue() != 1)
+ return false;
+ auto *FirstIndex = dyn_cast<ConstantInt>(First->getOperand(2));
+ if (FirstIndex == nullptr || FirstIndex->getZExtValue() != 0 || !isa<UndefValue>(First->getOperand(0)))
+ return false;
+ auto *LowLoad = dyn_cast<LoadInst>(First->getOperand(1));
+ auto *HighLoad = dyn_cast<LoadInst>(Second->getOperand(1));
+ if (LowLoad == nullptr || HighLoad == nullptr || LowLoad->getParent() != Call->getParent() ||
+ HighLoad->getParent() != Call->getParent())
+ return false;
+ Pair.Low = dyn_cast<AllocaInst>(LowLoad->getPointerOperand());
+ Pair.High = dyn_cast<AllocaInst>(HighLoad->getPointerOperand());
+ if (Pair.Low == nullptr || Pair.High == nullptr || Pair.Low == Pair.High ||
+ !Pair.Low->getAllocatedType()->isIntegerTy(32) || !Pair.High->getAllocatedType()->isIntegerTy(32) ||
+ !isPlainRegister(Pair.Low) || !isPlainRegister(Pair.High))
+ return false;
+ Instruction *Earliest = LowLoad->comesBefore(HighLoad) ? LowLoad : HighLoad;
+ for (Instruction *I = Earliest->getNextNode(); I != Call; I = I->getNextNode())
+ if (auto *Store = dyn_cast<StoreInst>(I))
+ if (Store->getPointerOperand() == Pair.Low || Store->getPointerOperand() == Pair.High)
+ return false;
+ return true;
+}
+
+// Finds the stores of both lanes of an MMA result into a register pair.
+static bool matchStoredPair(CallInst *Call, RegisterPair &Pair, StoreInst *Stores[2])
+{
+ Stores[0] = Stores[1] = nullptr;
+ for (User *U : Call->users())
+ {
+ auto *Extract = dyn_cast<ExtractElementInst>(U);
+ auto *Index = Extract != nullptr ? dyn_cast<ConstantInt>(Extract->getIndexOperand()) : nullptr;
+ if (Index == nullptr || Index->getZExtValue() > 1 || !Extract->hasOneUse())
+ return false;
+ auto *Store = dyn_cast<StoreInst>(Extract->user_back());
+ if (Store == nullptr || Store->getValueOperand() != Extract || Stores[Index->getZExtValue()] != nullptr)
+ return false;
+ Stores[Index->getZExtValue()] = Store;
+ }
+ if (Stores[0] == nullptr || Stores[1] == nullptr)
+ return false;
+ Pair.Low = dyn_cast<AllocaInst>(Stores[0]->getPointerOperand());
+ Pair.High = dyn_cast<AllocaInst>(Stores[1]->getPointerOperand());
+ return Pair.Low != nullptr && Pair.High != nullptr && Pair.Low != Pair.High;
+}
+
+// Matches an N-element vector built as a chain of insertelements (indices
+// 0..N-1 into undef) of loads from plain i32 registers; returns the registers.
+static bool matchLoadedVector(Value *Vector, unsigned Elements, SmallVectorImpl<AllocaInst *> &Registers)
+{
+ Registers.assign(Elements, nullptr);
+ for (unsigned Remaining = Elements; Remaining > 0; --Remaining)
+ {
+ auto *Insert = dyn_cast<InsertElementInst>(Vector);
+ auto *Index = Insert != nullptr ? dyn_cast<ConstantInt>(Insert->getOperand(2)) : nullptr;
+ if (Index == nullptr || Index->getZExtValue() != Remaining - 1)
+ return false;
+ auto *Load = dyn_cast<LoadInst>(Insert->getOperand(1));
+ auto *Register = Load != nullptr ? dyn_cast<AllocaInst>(Load->getPointerOperand()) : nullptr;
+ if (Register == nullptr || !Register->getAllocatedType()->isIntegerTy(32) || !isPlainRegister(Register))
+ return false;
+ Registers[Remaining - 1] = Register;
+ Vector = Insert->getOperand(0);
+ }
+ return isa<UndefValue>(Vector);
+}
+
+// The registers are plain allocas (only loaded and stored directly), so only
+// explicit stores can change them.
+static bool storesBetween(Instruction *From, Instruction *To, ArrayRef<AllocaInst *> Registers)
+{
+ for (Instruction *I = From->getNextNode(); I != nullptr && I != To; I = I->getNextNode())
+ if (auto *Store = dyn_cast<StoreInst>(I))
+ if (is_contained(Registers, Store->getPointerOperand()))
+ return true;
+ return false;
+}
+
+struct Helpers
+{
+ FunctionCallee AFromB, BFromA, Mma, FromNv, ToNv, AFromBPair, MmaPair;
+};
+
+static Value *loadPair(IRBuilder<> &Builder, const RegisterPair &Pair)
+{
+ Type *I32 = Builder.getInt32Ty();
+ Value *Vector = UndefValue::get(FixedVectorType::get(I32, 2));
+ Vector = Builder.CreateInsertElement(Vector, Builder.CreateLoad(I32, Pair.Low), Builder.getInt8(0));
+ return Builder.CreateInsertElement(Vector, Builder.CreateLoad(I32, Pair.High), Builder.getInt8(1));
+}
+
+static unsigned rewriteFunction(Function &F, Function *Mma, const Helpers &H)
+{
+ SmallVector<CallInst *, 64> Calls;
+ for (Instruction &I : instructions(F))
+ if (auto *Call = dyn_cast<CallInst>(&I))
+ if (Call->getCalledFunction() == Mma && Call->arg_size() == 3)
+ Calls.push_back(Call);
+ if (Calls.empty())
+ return 0;
+
+ struct Site
+ {
+ CallInst *Call;
+ bool HasC;
+ RegisterPair C;
+ bool HasD;
+ RegisterPair D;
+ StoreInst *DStores[2];
+ };
+ SmallVector<Site, 64> Sites;
+ for (CallInst *Call : Calls)
+ {
+ Site S{Call, false, {}, false, {}, {nullptr, nullptr}};
+ S.HasC = matchLoadedPair(Call->getArgOperand(2), Call, S.C);
+ S.HasD = matchStoredPair(Call, S.D, S.DStores);
+ Sites.push_back(S);
+ }
+
+ // Accumulator pairs whose registers belong to no other pair are replaced
+ // outright by a shadow in WMMA layout.
+ SmallVector<RegisterPair, 32> Shadowed;
+ DenseMap<AllocaInst *, int> Owner; // register -> shadow index, -1 if ambiguous
+ auto consider = [&](const RegisterPair &Pair) {
+ for (const RegisterPair &Known : Shadowed)
+ if (Known == Pair)
+ return;
+ int Index = static_cast<int>(Shadowed.size());
+ for (AllocaInst *Member : {Pair.Low, Pair.High})
+ {
+ auto Found = Owner.find(Member);
+ if (Found != Owner.end())
+ Found->second = -1;
+ else
+ Owner[Member] = Index;
+ }
+ Shadowed.push_back(Pair);
+ };
+ for (const Site &S : Sites)
+ if (S.HasC)
+ consider(S.C);
+ auto shadowOf = [&](const RegisterPair &Pair) -> int {
+ auto Low = Owner.find(Pair.Low);
+ auto High = Owner.find(Pair.High);
+ if (Low == Owner.end() || High == Owner.end() || Low->second < 0 || Low->second != High->second)
+ return -1;
+ return Shadowed[Low->second] == Pair ? Low->second : -1;
+ };
+
+ Type *Pair32 = FixedVectorType::get(Type::getInt32Ty(F.getContext()), 2);
+ SmallVector<AllocaInst *, 32> Shadows;
+ {
+ IRBuilder<> Builder(&*F.getEntryBlock().getFirstInsertionPt());
+ for (size_t Index = 0; Index < Shadowed.size(); ++Index)
+ Shadows.push_back(
+ shadowOf(Shadowed[Index]) == static_cast<int>(Index)
+ ? Builder.CreateAlloca(Pair32, Shadowed[Index].Low->getAddressSpace(), nullptr, "wmma.shadow")
+ : nullptr);
+ }
+ auto shadowFor = [&](const RegisterPair &Pair) -> AllocaInst * {
+ int Index = shadowOf(Pair);
+ return Index >= 0 ? Shadows[Index] : nullptr;
+ };
+
+ // Snapshot every access to shadowed registers before rewriting anything.
+ SmallVector<LoadInst *, 64> Loads;
+ SmallVector<StoreInst *, 64> Stores;
+ for (auto &[Register, Index] : Owner)
+ {
+ if (Index < 0 || Shadows[Index] == nullptr)
+ continue;
+ for (User *U : Register->users())
+ {
+ if (auto *Load = dyn_cast<LoadInst>(U))
+ Loads.push_back(Load);
+ else
+ Stores.push_back(cast<StoreInst>(U));
+ }
+ }
+ DenseSet<StoreInst *> ResultStores; // MMA result stores replaced by shadow stores
+
+ auto accumulator = [&](const Site &S, IRBuilder<> &Builder) -> Value * {
+ if (S.HasC)
+ if (AllocaInst *Shadow = shadowFor(S.C))
+ return Builder.CreateLoad(Pair32, Shadow);
+ return Builder.CreateCall(H.FromNv, {S.Call->getArgOperand(2)});
+ };
+ auto finish = [&](const Site &S, Value *D, IRBuilder<> &Builder) {
+ if (S.HasD)
+ if (AllocaInst *Shadow = shadowFor(S.D))
+ {
+ // The result goes to the shadow where its register stores were.
+ StoreInst *Last = S.DStores[0]->comesBefore(S.DStores[1]) ? S.DStores[1] : S.DStores[0];
+ IRBuilder<> At(Last);
+ At.CreateStore(D, Shadow);
+ ResultStores.insert(S.DStores[0]);
+ ResultStores.insert(S.DStores[1]);
+ }
+ return Builder.CreateCall(H.ToNv, {D});
+ };
+ // Two MMAs that share A (same registers, no store in between to anything
+ // the second one reads) run as one 16x16x16 WMMA placed at the first.
+ auto pairable = [&](const Site &First, const Site &Second) {
+ SmallVector<AllocaInst *, 4> AFirst, ASecond, BSecond;
+ if (First.Call->getParent() != Second.Call->getParent() || !Second.HasC ||
+ shadowFor(Second.C) == nullptr || !First.Call->comesBefore(Second.Call) ||
+ !matchLoadedVector(First.Call->getArgOperand(0), 4, AFirst) ||
+ !matchLoadedVector(Second.Call->getArgOperand(0), 4, ASecond) || AFirst != ASecond ||
+ !matchLoadedVector(Second.Call->getArgOperand(1), 2, BSecond))
+ return false;
+ // The second B is reloaded at the first MMA, so it must not live in a
+ // shadow (those registers are no longer kept up to date).
+ for (AllocaInst *Register : BSecond)
+ {
+ auto Found = Owner.find(Register);
+ if (Found != Owner.end() && Found->second >= 0 && Shadows[Found->second] != nullptr)
+ return false;
+ }
+ SmallVector<AllocaInst *, 8> Read(AFirst.begin(), AFirst.end());
+ Read.append(BSecond.begin(), BSecond.end());
+ Read.push_back(Second.C.Low);
+ Read.push_back(Second.C.High);
+ return !storesBetween(First.Call, Second.Call, Read);
+ };
+ for (size_t Index = 0; Index < Sites.size(); ++Index)
+ {
+ const Site &S = Sites[Index];
+ CallInst *Call = S.Call;
+ IRBuilder<> Builder(Call);
+ Value *BTransposed = Builder.CreateCall(H.BFromA, {Call->getArgOperand(0)});
+ if (Index + 1 < Sites.size() && pairable(S, Sites[Index + 1]))
+ {
+ const Site &Next = Sites[Index + 1];
+ SmallVector<AllocaInst *, 2> BNext;
+ matchLoadedVector(Next.Call->getArgOperand(1), 2, BNext);
+ Value *B1 = loadPair(Builder, RegisterPair{BNext[0], BNext[1]});
+ Value *ATransposed = Builder.CreateCall(H.AFromBPair, {Call->getArgOperand(1), B1});
+ Value *C0 = accumulator(S, Builder);
+ Value *C1 = accumulator(Next, Builder);
+ Value *D = Builder.CreateCall(H.MmaPair, {ATransposed, BTransposed, C0, C1});
+ Value *D0 = Builder.CreateShuffleVector(D, ArrayRef<int>{0, 1});
+ Value *D1 = Builder.CreateShuffleVector(D, ArrayRef<int>{2, 3});
+ Call->replaceAllUsesWith(finish(S, D0, Builder));
+ Next.Call->replaceAllUsesWith(finish(Next, D1, Builder));
+ Call->eraseFromParent();
+ Next.Call->eraseFromParent();
+ ++Index;
+ continue;
+ }
+ Value *ATransposed = Builder.CreateCall(H.AFromB, {Call->getArgOperand(1)});
+ Value *D = Builder.CreateCall(H.Mma, {ATransposed, BTransposed, accumulator(S, Builder)});
+ Call->replaceAllUsesWith(finish(S, D, Builder));
+ Call->eraseFromParent();
+ }
+
+ // Every other access now goes through the shadow.
+ for (LoadInst *Load : Loads)
+ {
+ auto *Register = cast<AllocaInst>(Load->getPointerOperand());
+ const RegisterPair &Pair = Shadowed[Owner[Register]];
+ IRBuilder<> Builder(Load);
+ Value *Nv = Builder.CreateCall(H.ToNv, {Builder.CreateLoad(Pair32, shadowFor(Pair))});
+ Load->replaceAllUsesWith(Builder.CreateExtractElement(Nv, Builder.getInt8(Register == Pair.High)));
+ Load->eraseFromParent();
+ }
+ for (StoreInst *Store : Stores)
+ {
+ if (!ResultStores.contains(Store))
+ {
+ auto *Register = cast<AllocaInst>(Store->getPointerOperand());
+ const RegisterPair &Pair = Shadowed[Owner[Register]];
+ AllocaInst *Shadow = shadowFor(Pair);
+ IRBuilder<> Builder(Store);
+ Value *Nv = Builder.CreateCall(H.ToNv, {Builder.CreateLoad(Pair32, Shadow)});
+ Nv = Builder.CreateInsertElement(Nv, Store->getValueOperand(), Builder.getInt8(Register == Pair.High));
+ Builder.CreateStore(Builder.CreateCall(H.FromNv, {Nv}), Shadow);
+ }
+ auto *Operand = dyn_cast<Instruction>(Store->getValueOperand());
+ Store->eraseFromParent();
+ if (Operand != nullptr && Operand->use_empty() && !Operand->mayHaveSideEffects())
+ Operand->eraseFromParent();
+ }
+ return Calls.size();
+}
+} // namespace
+
+unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef)
+{
+ Module *M = unwrap(ModuleRef);
+ Function *Mma = M->getFunction(kMmaF16);
+ if (Mma == nullptr || Mma->use_empty())
+ return 0;
+ LLVMContext &Context = M->getContext();
+ Type *I32 = Type::getInt32Ty(Context);
+ Type *V2 = FixedVectorType::get(I32, 2);
+ Type *V4 = FixedVectorType::get(I32, 4);
+ Type *V8 = FixedVectorType::get(I32, 8);
+ AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
+ AttrBuilder(Context, Mma->getAttributes().getFnAttrs()));
+ Helpers H;
+ H.AFromB = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b", Attributes, V8, V2);
+ H.BFromA = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_b_from_a", Attributes, V8, V4);
+ H.Mma = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16", Attributes, V2, V8, V8, V2);
+ H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
+ H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
+ H.AFromBPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b_pair", Attributes, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16_pair", Attributes, V4, V8, V8, V2, V2);
+ unsigned Rewritten = 0;
+ for (Function &F : *M)
+ if (!F.isDeclaration())
+ Rewritten += rewriteFunction(F, Mma, H);
+ return Rewritten;
+}
+
bool LLVMZludaParseCommandLineOptions(int argc, const char *const *argv,
char **ErrorMessage)
{
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index 69d1faa..c11e06a 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1235,6 +1235,184 @@ extern "C"
}
}
+// gfx11 WMMA lowering of mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16.
+//
+// llvm_zluda's MMA rewrite (LLVMZludaRewriteMmaF16Wmma) replaces each such MMA
+// with these helpers. It computes D^T = B^T A^T + C^T with
+// v_wmma_f32_16x16x16_f16, so the 16x8 accumulator occupies WMMA result
+// rows 0..7, i.e. VGPRs 0..3 of the f32 result: lane l holds D[m][n] for
+// m = l % 16 and n = 2v + l / 16, v = 0..3 (the "transposed WMMA layout").
+// Those four values are kept as two f16x2 words (v0 | v1 << 16,
+// v2 | v3 << 16), the same register footprint as NVIDIA's own fragment.
+// Converting between the two layouts is pure 16-bit data movement, so the
+// rewrite keeps accumulators in WMMA layout across chained MMAs and converts
+// only where other code reads or writes them. Each MMA still rounds its result
+// to f16, as the .f16 accumulator requires; only the f32 summation order
+// within one MMA differs from the dot-product fallback.
+typedef _Float16 wmma_half16 __attribute__((ext_vector_type(16)));
+typedef float wmma_float8 __attribute__((ext_vector_type(8)));
+typedef uint32_t wmma_uint8 __attribute__((ext_vector_type(8)));
+__device__ wmma_float8 __zluda_wmma_f32_16x16x16_f16(wmma_half16 a, wmma_half16 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16");
+
+__device__ static uint32_t wmma_lane()
+{
+ return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
+}
+
+__device__ static uint32_t wmma_from_lane(uint32_t lane, uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_ds_bpermute(int(lane << 2), int(value)));
+}
+
+// Gathers WMMA operand dword q for every lane from two NVIDIA fragment
+// registers: rows r < 8 take `low` from lane 4r+q, rows r >= 8 take `high`
+// from lane 4(r-8)+q (r = lane % 16). Swapping `high` between quad halves
+// (DPP quad_perm [2,3,0,1]) parks the value rows r >= 8 need in lane
+// 4(r-8)+(q^2), so one bpermute serves both halves.
+__device__ static uint32_t wmma_gather_rows(uint32_t low, uint32_t high_swapped, uint32_t q)
+{
+ uint32_t lane = wmma_lane();
+ uint32_t row = lane % 16;
+ uint32_t source = lane % 4 == q ? low : high_swapped;
+ uint32_t from = row < 8 ? row * 4 + q : (row - 8) * 4 + (q ^ 2);
+ return wmma_from_lane(from, source);
+}
+
+__device__ static uint32_t wmma_swap_quad_halves(uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_mov_dpp(int(value), 0b01'00'11'10, 0xf, 0xf, true));
+}
+
+extern "C"
+{
+ // WMMA A operand (lane l holds row l % 16 over k) = B^T, from an NVIDIA B
+ // fragment (lane 4n+q: B[2q..2q+1][n] in .x, B[2q+8..2q+9][n] in .y).
+ // Rows 8..15 (padding columns of the 16x8 B) are zero.
+ wmma_uint8 FUNC(d4r_wmma_t_a_from_b)(uint2::Native_vec_ b)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t n = wmma_lane() % 16;
+ uint32_t source = (n % 8) * 4;
+ wmma_uint8 result;
+ for (uint32_t q = 0; q < 4; q++)
+ {
+ uint32_t low = wmma_from_lane(source + q, b.x);
+ uint32_t high = wmma_from_lane(source + q, b.y);
+ result[q] = n < 8 ? low : 0u;
+ result[q + 4] = n < 8 ? high : 0u;
+ }
+ return result;
+ }
+
+ // WMMA B operand (lane l holds column l % 16 over k) = A^T, i.e. row
+ // m = l % 16 of the NVIDIA A fragment (lane 4g+t: rows g/g+8, k 2t..2t+1 in
+ // .x/.y and k 2t+8..2t+9 in .z/.w).
+ wmma_uint8 FUNC(d4r_wmma_t_b_from_a)(uint4::Native_vec_ a)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t ay = wmma_swap_quad_halves(a.y);
+ uint32_t aw = wmma_swap_quad_halves(a.w);
+ wmma_uint8 result;
+ for (uint32_t p = 0; p < 4; p++)
+ {
+ result[p] = wmma_gather_rows(a.x, ay, p);
+ result[p + 4] = wmma_gather_rows(a.z, aw, p);
+ }
+ return result;
+ }
+
+ // Paired form: the WMMA A operand rows 0..7 are B0^T and rows 8..15 are
+ // B1^T, so one WMMA computes two m16n8k16 MMAs that share A.
+ wmma_uint8 FUNC(d4r_wmma_t_a_from_b_pair)(uint2::Native_vec_ b0, uint2::Native_vec_ b1)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t b1x = wmma_swap_quad_halves(b1.x);
+ uint32_t b1y = wmma_swap_quad_halves(b1.y);
+ wmma_uint8 result;
+ for (uint32_t q = 0; q < 4; q++)
+ {
+ result[q] = wmma_gather_rows(b0.x, b1x, q);
+ result[q + 4] = wmma_gather_rows(b0.y, b1y, q);
+ }
+ return result;
+ }
+
+ // Two accumulators (for B0 and B1) in transposed WMMA layout: .xy is the
+ // first, .zw the second; they are WMMA result VGPRs 0..3 and 4..7.
+ uint4::Native_vec_ FUNC(d4r_wmma_t_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ f16x2 c01 = std::bit_cast<f16x2>(c0_t.x);
+ f16x2 c23 = std::bit_cast<f16x2>(c0_t.y);
+ f16x2 c45 = std::bit_cast<f16x2>(c1_t.x);
+ f16x2 c67 = std::bit_cast<f16x2>(c1_t.y);
+ wmma_float8 c = {float(c01.x), float(c01.y), float(c23.x), float(c23.y),
+ float(c45.x), float(c45.y), float(c67.x), float(c67.y)};
+ wmma_float8 d = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a_t), std::bit_cast<wmma_half16>(b_t), c);
+ f16x2 d01{static_cast<f16>(d[0]), static_cast<f16>(d[1])};
+ f16x2 d23{static_cast<f16>(d[2]), static_cast<f16>(d[3])};
+ f16x2 d45{static_cast<f16>(d[4]), static_cast<f16>(d[5])};
+ f16x2 d67{static_cast<f16>(d[6]), static_cast<f16>(d[7])};
+ return {std::bit_cast<uint32_t>(d01), std::bit_cast<uint32_t>(d23),
+ std::bit_cast<uint32_t>(d45), std::bit_cast<uint32_t>(d67)};
+ }
+
+ // D = round_f16(A B + C) with C and D in the transposed WMMA layout.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ f16x2 c01 = std::bit_cast<f16x2>(c_t.x);
+ f16x2 c23 = std::bit_cast<f16x2>(c_t.y);
+ wmma_float8 c = {float(c01.x), float(c01.y), float(c23.x), float(c23.y), 0.0f, 0.0f, 0.0f, 0.0f};
+ wmma_float8 d = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a_t), std::bit_cast<wmma_half16>(b_t), c);
+ f16x2 d01{static_cast<f16>(d[0]), static_cast<f16>(d[1])};
+ f16x2 d23{static_cast<f16>(d[2]), static_cast<f16>(d[3])};
+ return {std::bit_cast<uint32_t>(d01), std::bit_cast<uint32_t>(d23)};
+ }
+
+ // NVIDIA accumulator fragment (lane 4g+t: D[g][2t..2t+1] in .x,
+ // D[g+8][2t..2t+1] in .y) -> transposed WMMA layout.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_from_nv)(uint2::Native_vec_ c)
+ {
+ uint32_t lane = wmma_lane();
+ uint32_t m = lane % 16;
+ uint32_t shift = (lane / 16) * 16;
+ uint32_t source = (m % 8) * 4;
+ uint32_t halves[4];
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ uint32_t low_rows = wmma_from_lane(source + v, c.x);
+ uint32_t high_rows = wmma_from_lane(source + v, c.y);
+ halves[v] = ((m >= 8 ? high_rows : low_rows) >> shift) & 0xffffu;
+ }
+ return {halves[0] | (halves[1] << 16), halves[2] | (halves[3] << 16)};
+ }
+
+ // Transposed WMMA layout -> NVIDIA accumulator fragment.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_to_nv)(uint2::Native_vec_ d)
+ {
+ uint32_t lane = wmma_lane();
+ uint32_t g = lane / 4;
+ uint32_t t = lane % 4;
+ uint32_t sources[4] = {g, g + 16, g + 8, g + 24}; // D[g][2t], D[g][2t+1], D[g+8][2t], D[g+8][2t+1]
+ uint32_t halves[4];
+ for (uint32_t i = 0; i < 4; i++)
+ {
+ uint32_t v01 = wmma_from_lane(sources[i], d.x);
+ uint32_t v23 = wmma_from_lane(sources[i], d.y);
+ halves[i] = ((t >= 2 ? v23 : v01) >> ((t % 2) * 16)) & 0xffffu;
+ }
+ return {halves[0] | (halves[1] << 16), halves[2] | (halves[3] << 16)};
+ }
+}
+
template <typename T, typename A, typename B, int BOffset, typename OcklFunc>
__device__ T dp2a_impl(T a, T b, T c, OcklFunc ockl_func)
{
diff --git a/zluda_common/examples/d4r_fatbin_ptx.rs b/zluda_common/examples/d4r_fatbin_ptx.rs
new file mode 100644
index 0000000..50f7a09
--- /dev/null
+++ b/zluda_common/examples/d4r_fatbin_ptx.rs
@@ -0,0 +1,36 @@
+// Extracts (and decompresses) every PTX file from captured CUDA fatbin images.
+// usage: d4r_fatbin_ptx <outdir> <image>...
+use cuda_types::dark_api::FatbinFileHeader;
+use std::{env, fs, path::PathBuf};
+use zluda_common::{CodeLibraryRef, CodeModuleRef};
+
+fn main() {
+ let mut args = env::args().skip(1);
+ let outdir = PathBuf::from(args.next().expect("output directory"));
+ fs::create_dir_all(&outdir).expect("create output directory");
+ for path in args {
+ let mut bytes = fs::read(&path).expect("read image");
+ bytes.extend_from_slice(&[0; 16]);
+ let stem = PathBuf::from(&path).file_stem().unwrap().to_string_lossy().into_owned();
+ let library = unsafe { CodeLibraryRef::try_load(bytes.as_ptr().cast()) }.expect("load image");
+ let mut count = 0;
+ unsafe {
+ library.iterate_modules(|_, module| {
+ let text = match module {
+ Ok(CodeModuleRef::Text(ptx)) => Some(ptx.to_string()),
+ Ok(CodeModuleRef::File(file)) if file.header.kind == FatbinFileHeader::HEADER_KIND_PTX => file
+ .get_or_decompress_content(true)
+ .ok()
+ .map(|content| String::from_utf8_lossy(&content).trim_end_matches('\0').to_string()),
+ _ => None,
+ };
+ if let Some(text) = text {
+ let name = outdir.join(format!("{stem}-{count:02}.ptx"));
+ fs::write(&name, text).expect("write PTX");
+ count += 1;
+ }
+ })
+ };
+ println!("{path}: {count} PTX modules");
+ }
+}
patches/zluda/0004-native-overrides-and-gpu-handoffs.patch
+1672
View File
File diff suppressed because it is too large. Load diff
patches/zluda/0005-null-texture-objects-read-zero.patch
+40
View File
@@ -0,0 +1,40 @@
From: d4r
Subject: [PATCH] null texture objects read as zero
CUDA texture object 0 reads as zeros on NVIDIA GPUs, and DLSS samples absent optional inputs
(its exposure scales, with the flags Ghost of Tsushima uses) that way. A ZLUDA texture object is a
descriptor pointer, so handle 0 made the GPU read address 0 and fault. Handle 0 now points at an
all-zero descriptor, which AMD hardware treats as a null resource. Rebuild ptx/lib/*.bc after
applying (scripts/build_zluda_ptx_helpers.sh).
---
--- a/ptx/lib/zluda_ptx_impl.cpp 2026-09-27 16:47:02.815623944 -0400
+++ b/ptx/lib/zluda_ptx_impl.cpp 2026-09-27 16:47:02.815623944 -0400
@@ -1725,9 +1725,18 @@
c;
}
+ // d4r: CUDA texture object 0 reads as zeros on NVIDIA GPUs, and DLSS samples absent optional inputs
+ // (exposure scales) that way. A ZLUDA texture object is a descriptor pointer, so 0 is redirected to
+ // an all-zero descriptor, which AMD hardware treats as a null resource (reads return 0).
+ __device__ static unsigned int d4r_null_texture[32];
+ static uint64_t texture_or_null(uint64_t texobj) __device__
+ {
+ return texobj != 0 ? texobj : uint64_t(uintptr_t(d4r_null_texture));
+ }
+
static std::pair<GLOBAL_SPACE void *, GLOBAL_SPACE void *> get_image_and_sampler(uint64_t texobj) __device__
{
- unsigned int GLOBAL_SPACE *image = (unsigned int GLOBAL_SPACE *)texobj;
+ unsigned int GLOBAL_SPACE *image = (unsigned int GLOBAL_SPACE *)texture_or_null(texobj);
unsigned int GLOBAL_SPACE *sampler = image + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
return {image, sampler};
}
@@ -1868,7 +1877,7 @@
#define tex_2d_gather4(COMPONENT) \
v4f32 FUNC(texobj_gather4##COMPONENT##_2d_v4_f32_f32)(uint64_t texobj, v2f32 coord) \
{ \
- auto image = reinterpret_cast<unsigned int CONSTANT_SPACE *>(static_cast<uintptr_t>(texobj)); \
+ auto image = reinterpret_cast<unsigned int CONSTANT_SPACE *>(static_cast<uintptr_t>(texture_or_null(texobj))); \
auto sampler = image + HIP_SAMPLER_OBJECT_OFFSET_DWORD; \
float2 coordinates{coord.x, coord.y}; \
auto result = __ockl_image_gather4##COMPONENT##_2D(image, sampler, get_native_vector(coordinates)); \
patches/zluda/0006-k8-wmma-prep-slots-wave64.patch
+444
View File
@@ -0,0 +1,444 @@
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 5d8c0a5..b2870e5 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -70,8 +70,10 @@ fn create_oclc_constants(ctx: &Context, gcn_arch: &str) -> Result<Module, String
let module = Module::new(ctx, c"oclc_constants");
// used by ockl
- add_constant(ctx, &module, c"__oclc_wavefrontsize64", 0);
- add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", 5);
+ // d4r: D4R_ZLUDA_WAVE64=1 (see ptx::pass::wave64_requested)
+ let wave64 = std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1");
+ add_constant(ctx, &module, c"__oclc_wavefrontsize64", if wave64 { 1 } else { 0 });
+ add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", if wave64 { 6 } else { 5 });
add_constant(ctx, &module, c"__oclc_ABI_version", 500);
add_constant(
ctx,
@@ -174,7 +176,13 @@ pub fn compile(
let triple = c"amdgcn-amd-amdhsa";
let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
- let features = c"-wavefrontsize64,+cumode";
+ // functions without their own target features (the linked helper libraries) take these, so under
+ // D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
+ let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
+ c"-wavefrontsize32,+wavefrontsize64,+cumode"
+ } else {
+ c"-wavefrontsize64,+cumode"
+ };
let mut target = unsafe { std::mem::zeroed() };
let mut err = std::ptr::null_mut();
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 88848b1..0d1f9cf 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -869,10 +869,111 @@ unsigned LLVMZludaSplitWmmaM16n16k16(LLVMModuleRef ModuleRef)
return static_cast<unsigned>(Calls.size());
}
+// d4r: an m16n8k8 f16 MMA is an m16n8k16 one whose second K half is zero (its A fragment is the k16
+// A's registers 0-1, its B the k16 B's register 0), and adding zero products is exact. Each k8 call
+// becomes a k16 call so the WMMA rewrite below lowers it too. Zero halves are loads of one zeroed
+// register, so the operands stay "loads of plain registers" and two k8 MMAs that share A still fuse
+// into one WMMA. Returns the number of calls converted.
+static unsigned widenMmaK8(Module *M)
+{
+ Function *K8 = M->getFunction("__zluda_ptx_impl_mma_sync_aligned_m16n8k8_row_col_f16_f16_f16_f16");
+ if (K8 == nullptr || K8->use_empty())
+ return 0;
+ LLVMContext &Context = M->getContext();
+ Type *I32 = Type::getInt32Ty(Context);
+ Type *V2 = FixedVectorType::get(I32, 2);
+ Type *V4 = FixedVectorType::get(I32, 4);
+ AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
+ AttrBuilder(Context, K8->getAttributes().getFnAttrs()));
+ FunctionCallee K16 = M->getOrInsertFunction(kMmaF16, Attributes, V2, V4, V2, V2);
+ SmallVector<CallInst *, 64> Calls;
+ for (User *U : K8->users())
+ if (auto *Call = dyn_cast<CallInst>(U); Call != nullptr && Call->getCalledFunction() == K8)
+ Calls.push_back(Call);
+ DenseMap<Function *, AllocaInst *> Zeros;
+ for (CallInst *Call : Calls)
+ {
+ Function *F = Call->getFunction();
+ AllocaInst *&Zero = Zeros[F];
+ if (Zero == nullptr)
+ {
+ // an alloca like ZLUDA's register allocas (same address space), stored once at entry
+ AllocaInst *Like = nullptr;
+ for (Instruction &I : F->getEntryBlock())
+ if ((Like = dyn_cast<AllocaInst>(&I)) != nullptr)
+ break;
+ IRBuilder<> Entry(&*F->getEntryBlock().getFirstInsertionPt());
+ Zero = Entry.CreateAlloca(I32, Like != nullptr ? Like->getAddressSpace() : 5, nullptr, "d4r.k8.zero");
+ Entry.CreateStore(Entry.getInt32(0), Zero);
+ }
+ IRBuilder<> Builder(Call);
+ SmallVector<AllocaInst *, 2> ARegisters, BRegisters;
+ auto build = [&](ArrayRef<Value *> Elements) {
+ Value *Vector = UndefValue::get(FixedVectorType::get(I32, Elements.size()));
+ for (unsigned Index = 0; Index < Elements.size(); ++Index)
+ Vector = Builder.CreateInsertElement(Vector, Elements[Index], Builder.getInt8(Index));
+ return Vector;
+ };
+ auto zero = [&]() { return Builder.CreateLoad(I32, Zero); };
+ // the registers may be reloaded at the call only if no store reaches them after their original loads
+ auto unchangedSinceLoads = [&](Value *Vector) {
+ for (Value *V = Vector; auto *Insert = dyn_cast<InsertElementInst>(V); V = Insert->getOperand(0))
+ {
+ auto *Load = cast<LoadInst>(Insert->getOperand(1));
+ if (Load->getParent() != Call->getParent() ||
+ storesBetween(Load, Call, {cast<AllocaInst>(Load->getPointerOperand())}))
+ return false;
+ }
+ return true;
+ };
+ // B is one register: a <1 x i32> vector or a plain i32
+ Value *BArg = Call->getArgOperand(1);
+ auto bRegister = [&]() {
+ if (BArg->getType()->isVectorTy())
+ return matchLoadedVector(BArg, 1, BRegisters) && unchangedSinceLoads(BArg);
+ auto *Load = dyn_cast<LoadInst>(BArg);
+ auto *Register = Load != nullptr ? dyn_cast<AllocaInst>(Load->getPointerOperand()) : nullptr;
+ if (Register == nullptr || !Register->getAllocatedType()->isIntegerTy(32) || !isPlainRegister(Register) ||
+ Load->getParent() != Call->getParent() || storesBetween(Load, Call, {Register}))
+ return false;
+ BRegisters.assign(1, Register);
+ return true;
+ };
+ auto element = [&](Value *V, unsigned Index) -> Value * {
+ return V->getType()->isVectorTy() ? Builder.CreateExtractElement(V, Builder.getInt8(Index)) : V;
+ };
+ Value *A, *B;
+ if (matchLoadedVector(Call->getArgOperand(0), 2, ARegisters) && bRegister() &&
+ unchangedSinceLoads(Call->getArgOperand(0)))
+ {
+ // reload the same registers right at the call (nothing lies between these loads and it)
+ Value *A0 = Builder.CreateLoad(I32, ARegisters[0]), *A1 = Builder.CreateLoad(I32, ARegisters[1]);
+ Value *B0 = Builder.CreateLoad(I32, BRegisters[0]);
+ A = build({A0, A1, zero(), zero()});
+ B = build({B0, zero()});
+ }
+ else
+ {
+ Value *A0 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(0));
+ Value *A1 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(1));
+ Value *B0 = element(BArg, 0);
+ A = build({A0, A1, Builder.getInt32(0), Builder.getInt32(0)});
+ B = build({B0, Builder.getInt32(0)});
+ }
+ CallInst *Wide = Builder.CreateCall(K16, {A, B, Call->getArgOperand(2)});
+ Call->replaceAllUsesWith(Wide);
+ Call->eraseFromParent();
+ }
+ return static_cast<unsigned>(Calls.size());
+}
+
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
Module *M = unwrap(ModuleRef);
+ // D4R_ZLUDA_WMMA_K8=0 keeps m16n8k8 on the dot-product fallback
+ if (const char *K8Env = getenv("D4R_ZLUDA_WMMA_K8"); K8Env == nullptr || K8Env[0] != '0')
+ widenMmaK8(M);
Function *Mma = M->getFunction(kMmaF16);
Function *MmaE4m3 = (Flags & 1) != 0 ? M->getFunction(kMmaE4m3) : nullptr;
if (Mma != nullptr && Mma->use_empty())
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index cdefdf1..74cf36f 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -127,9 +127,32 @@ extern "C"
{
extern "C" __attribute__((constant)) uint32_t __oclc_ISA_version __device__;
+ // d4r: kernels may be compiled as wave64 (D4R_ZLUDA_WAVE64, offline texture-kernel builds, linked with
+ // the zluda_ptx_impl*_w64.bc helpers built with -DD4R_WAVE64), where each wave holds two CUDA warps:
+ // lanes 0-31 and 32-63. Warp-level helpers then act on the lane's own 32-lane half.
+#ifdef D4R_WAVE64
+ __device__ static inline uint32_t d4r_wave_lane()
+ {
+ return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+ }
+ __device__ static inline uint32_t d4r_warp_base()
+ {
+ return d4r_wave_lane() & 32u;
+ }
+#else
+ __device__ static inline uint32_t d4r_warp_base()
+ {
+ return 0u;
+ }
+#endif
+
uint32_t FUNC(activemask)()
{
+#ifdef D4R_WAVE64
+ return (uint32_t)(__builtin_amdgcn_read_exec() >> d4r_warp_base());
+#else
return __builtin_amdgcn_read_exec_lo();
+#endif
}
size_t __ockl_get_local_id(uint32_t) __device__;
@@ -158,7 +181,11 @@ extern "C"
uint32_t FUNC(sreg_laneid)()
{
+#ifdef D4R_WAVE64
+ return d4r_wave_lane() & 31u;
+#else
return __lane_id();
+#endif
}
uint32_t FUNC(sreg_lanemask_le)()
@@ -322,7 +349,7 @@ extern "C"
int32_t section_mask = (opts >> 8) & 0b11111; \
int32_t warp_end = opts & 0b11111; \
delta &= 0b11111; \
- int32_t self = (int32_t)__lane_id(); \
+ int32_t self = (int32_t)__zluda_ptx_impl_sreg_laneid(); \
int32_t subsection = section_mask & self; \
int32_t subsection_end = subsection | (~section_mask & warp_end); \
int32_t idx = calculate_index; \
@@ -331,7 +358,7 @@ extern "C"
{ \
idx = self; \
} \
- int32_t output = __builtin_amdgcn_ds_bpermute(idx << 2, (int32_t)input); \
+ int32_t output = __builtin_amdgcn_ds_bpermute((int32_t)(d4r_warp_base() | (uint32_t)idx) << 2, (int32_t)input); \
return {(uint32_t)output, uint32_t(!out_of_bounds)}; \
} \
\
@@ -850,7 +877,11 @@ extern "C"
__device__ static inline uint32_t ballot(bool value, bool negate)
{
__builtin_amdgcn_wave_barrier();
+#ifdef D4R_WAVE64
+ return (uint32_t)(__builtin_amdgcn_ballot_w64(negate ? !value : value) >> d4r_warp_base());
+#else
return __builtin_amdgcn_ballot_w32(negate ? !value : value);
+#endif
}
bool FUNC(vote_sync_any_pred)(bool value, uint32_t membermask __attribute__((unused)))
@@ -866,7 +897,7 @@ extern "C"
// IMPORTANT: exec mask must be a subset of membermask, the behavior is undefined otherwise
bool FUNC(vote_sync_all_pred)(bool value, uint32_t membermask __attribute__((unused)))
{
- return ballot(value, false) == __builtin_amdgcn_read_exec_lo();
+ return ballot(value, false) == __zluda_ptx_impl_activemask();
}
// also known as "none"
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 8294bb4..6af2495 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -397,7 +397,12 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
fn emit_target_features(&mut self, fn_: LLVMValueRef) {
let value = format!(
- "+wavefrontsize32,-wavefrontsize64,+cumode{}",
+ "{},+cumode{}",
+ if crate::pass::wave64_requested() {
+ "-wavefrontsize32,+wavefrontsize64"
+ } else {
+ "+wavefrontsize32,-wavefrontsize64"
+ },
if cfg!(debug_assertions) {
",+precise-memory"
} else {
diff --git a/ptx/src/pass/mod.rs b/ptx/src/pass/mod.rs
index 63aa98c..e3baa72 100644
--- a/ptx/src/pass/mod.rs
+++ b/ptx/src/pass/mod.rs
@@ -37,6 +37,18 @@ mod test;
static ZLUDA_PTX_IMPL: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl.bc");
static ZLUDA_PTX_IMPL_CONSTRAINED: &'static [u8] =
include_bytes!("../../lib/zluda_ptx_impl_constrained.bc");
+static ZLUDA_PTX_IMPL_W64: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl_w64.bc");
+static ZLUDA_PTX_IMPL_CONSTRAINED_W64: &'static [u8] =
+ include_bytes!("../../lib/zluda_ptx_impl_constrained_w64.bc");
+
+/// d4r: D4R_ZLUDA_WAVE64=1 compiles kernels as wave64: each wave runs two CUDA warps (lanes 0-31 and
+/// 32-63; the warp-level helpers act on the lane's own half) and RDNA3 issues most FP32 VALU work for all
+/// 64 lanes at once. Meant for offline builds of texture kernels whose warp operations are limited to
+/// lane ids, shuffles and votes; not for MMA/matrix kernels (their helpers assume one warp per wave).
+pub fn wave64_requested() -> bool {
+ static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1"))
+}
const ZLUDA_PTX_PREFIX: &'static str = "__zluda_ptx_impl_";
quick_error! {
@@ -194,10 +206,11 @@ pub struct Module {
impl Module {
pub fn linked_bitcode(&self) -> &'static [u8] {
- if self.constrained_fp {
- ZLUDA_PTX_IMPL_CONSTRAINED
- } else {
- ZLUDA_PTX_IMPL
+ match (self.constrained_fp, wave64_requested()) {
+ (true, false) => ZLUDA_PTX_IMPL_CONSTRAINED,
+ (false, false) => ZLUDA_PTX_IMPL,
+ (true, true) => ZLUDA_PTX_IMPL_CONSTRAINED_W64,
+ (false, true) => ZLUDA_PTX_IMPL_W64,
}
}
}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index bacb93d..033d9a0 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -18,9 +18,14 @@ pub(crate) struct Function {
/// d4r native override: persistent kernels launch this many workgroups in x (0 = as launched)
pub(crate) grid_x: u32,
/// d4r native override: byte offset of the u64 parameter that keys the prep kernel's output
- /// (0 = run the prep before every launch) and the key of the last prep launched
- pub(crate) prep_key: u32,
+ /// (None = run the prep before every launch) and the key of the last prep launched
+ pub(crate) prep_key: Option<u32>,
pub(crate) prep_last: std::sync::atomic::AtomicU64,
+ /// d4r native override: with N > 0 prep slots the kernel keeps one prepared image per key for
+ /// up to N keys, so the prep runs only for keys not seen yet (and for every key beyond N, which
+ /// the kernel prepares into a shared overflow slot). 0 = only the last key is kept.
+ pub(crate) prep_slots: u32,
+ pub(crate) prep_seen: std::sync::Mutex<Vec<u64>>,
}
impl<'a, E: zluda_common::CudaErrorType> zluda_common::FromCuda<'a, CUfunction, E>
@@ -140,7 +145,7 @@ pub(crate) fn launch_kernel(
translated_extra_storage.as_mut_ptr()
};
// the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
- let prep_key = if f.prep.is_some() && f.prep_key != 0 {
+ let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
let mut block: *const u8 = std::ptr::null();
if !extra.is_null() {
let mut i = 0usize;
@@ -160,12 +165,27 @@ pub(crate) fn launch_kernel(
if block.is_null() {
0
} else {
- unsafe { std::ptr::read_unaligned(block.add(f.prep_key as usize) as *const u64) }
+ unsafe { std::ptr::read_unaligned(block.add(key_offset as usize) as *const u64) }
}
} else {
0
};
- let run_prep = prep_key == 0 || f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key;
+ let run_prep = if prep_key == 0 {
+ true
+ } else if f.prep_slots == 0 {
+ f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key
+ } else {
+ // prepared once per key while slots last; keys beyond them are prepared on every launch
+ let mut seen = f.prep_seen.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+ if seen.contains(&prep_key) {
+ false
+ } else {
+ if seen.len() < f.prep_slots as usize {
+ seen.push(prep_key);
+ }
+ true
+ }
+ };
if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
unsafe {
hipModuleLaunchKernel(
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 5db5ace..c5a39c9 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -37,12 +37,12 @@ impl ModuleMutable {
Ok(match self.functions.entry(name) {
hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
hash_map::Entry::Vacant(entry) => {
- let (func_handle, prep, block_z, grid_x, prep_key) = match native_override(entry.key()) {
+ let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
Some(native) => native,
None => {
let mut func_handle = unsafe { std::mem::zeroed() };
unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
- (func_handle, None, 0, 0, 0)
+ (func_handle, None, 0, 0, None, 0)
}
};
let func = Box::new(Function {
@@ -54,6 +54,8 @@ impl ModuleMutable {
grid_x,
prep_key,
prep_last: std::sync::atomic::AtomicU64::new(0),
+ prep_slots,
+ prep_seen: std::sync::Mutex::new(Vec::new()),
});
&*entry.insert(func)
}
@@ -67,7 +69,9 @@ impl ModuleMutable {
/// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
/// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
/// (persistent kernels, which derive the original grid from their parameters) the grid.
-fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, u32)> {
+fn native_override(
+ name: &CStr,
+) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
let image = fs::read(&path).ok()?;
@@ -103,34 +107,38 @@ fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t,
} else {
None
};
- let read_u32 = |symbol: &CStr| -> u32 {
+ let read_u32_opt = |symbol: &CStr| -> Option<u32> {
let mut ptr = unsafe { mem::zeroed() };
let mut size = 0usize;
let mut value = 0u32;
- if unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
+ (unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
&& size == 4
- && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok()
- {
- value
- } else {
- 0
- }
+ && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok())
+ .then_some(value)
};
+ let read_u32 = |symbol: &CStr| -> u32 { read_u32_opt(symbol).unwrap_or(0) };
let block_z = read_u32(c"d4r_block_z");
let grid_x = read_u32(c"d4r_grid_x");
// byte offset of a u64 in the parameter block that fully determines the prep kernel's output
- // (the weights pointer): the prep is skipped while it repeats (0 = prep before every launch)
- let prep_key = read_u32(c"d4r_prep_key_offset");
+ // (the weights pointer): the prep is skipped while it repeats. d4r_prep_key_at holds offset + 1
+ // (so a key at offset 0 is possible); the older d4r_prep_key_offset uses 0 for "no key".
+ let prep_key = match read_u32_opt(c"d4r_prep_key_at") {
+ Some(at) if at > 0 => Some(at - 1),
+ _ => Some(read_u32(c"d4r_prep_key_offset")).filter(|&offset| offset != 0),
+ };
+ // d4r_prep_key_slots: the kernel keeps one prepared image per key (see Function::prep_slots)
+ let prep_slots = read_u32(c"d4r_prep_key_slots");
eprintln!(
- "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {})",
+ "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {:?}, prep_slots: {})",
name,
path.display(),
prep.map(|p| p.1),
block_z,
grid_x,
- prep_key
+ prep_key,
+ prep_slots
);
- Some((func, prep, block_z, grid_x, prep_key))
+ Some((func, prep, block_z, grid_x, prep_key, prep_slots))
}
impl ZludaObject for Module {
patches/zluda/0007-gfx12-wmma-native-fp8.patch
+551
View File
@@ -0,0 +1,551 @@
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -113,9 +113,57 @@
*VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_PARAM_PROVENANCE").map_or(true, |value| value.trim() != "0"))
}
-fn wmma_rewrite_enabled(gcn_arch: &str) -> bool {
- // The helpers assume RDNA3's wave32 WMMA register layout.
- wmma_rewrite_requested() && gcn_arch.starts_with("gfx11")
+/// D4R_ZLUDA_WMMA_FP8_NATIVE=1: on gfx12 (RDNA4), e4m3 MMAs use the native FP8 WMMA instead of widening
+/// to f16. Ignored on every other GPU (gfx11 has no FP8 WMMA).
+pub fn wmma_fp8_native_requested() -> bool {
+ static VALUE: OnceLock<bool> = OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WMMA_FP8_NATIVE").map_or(false, |value| value.trim() == "1"))
+}
+
+/// The WMMA register layout family of a GPU: 11 (RDNA3) or 12 (RDNA4); the MMA helpers exist for these.
+/// D4R_ZLUDA_WMMA_LAYOUT=12 on gfx11 is for tests only: it takes the gfx12 helper names, which then must
+/// come from a helper library built with D4R_WMMA12_SHIM (gfx12 data movement, gfx11 WMMAs).
+fn wmma_family(gcn_arch: &str) -> Option<u32> {
+ let family = if gcn_arch.starts_with("gfx11") {
+ 11
+ } else if gcn_arch.starts_with("gfx12") {
+ 12
+ } else {
+ return None;
+ };
+ if family == 11 && std::env::var("D4R_ZLUDA_WMMA_LAYOUT").map_or(false, |value| value.trim() == "12") {
+ return Some(12);
+ }
+ Some(family)
+}
+
+/// Whether e4m3 MMAs compiled for this GPU use native FP8 WMMA (the setting only acts on gfx12).
+pub fn wmma_fp8_native_effective(gcn_arch: &str) -> bool {
+ wmma_rewrite_flags(gcn_arch).map_or(false, |flags| flags & 4 != 0)
+}
+
+/// Flags for LLVMZludaRewriteMmaF16Wmma, or None to keep the dot-product fallback (logged once when the
+/// rewrite was requested but this GPU has no WMMA lowering).
+fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
+ if !wmma_rewrite_requested() {
+ return None;
+ }
+ let Some(family) = wmma_family(gcn_arch) else {
+ static LOGGED: OnceLock<()> = OnceLock::new();
+ LOGGED.get_or_init(|| {
+ eprintln!("[ZLUDA] D4R_ZLUDA_WMMA: no WMMA lowering for {gcn_arch}; MMAs use the dot-product fallback")
+ });
+ return None;
+ };
+ let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+ if family == 12 {
+ flags |= 2;
+ // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
+ if flags & 1 != 0 && wmma_fp8_native_requested() {
+ flags |= 4;
+ }
+ }
+ Some(flags)
}
pub fn compile(
@@ -139,8 +187,7 @@
// wmma.mma m16n16k16 always becomes m16n8k16 MMAs (there is no helper for it).
unsafe { crate::ffi::LLVMZludaSplitWmmaM16n16k16(main.get()) };
- if wmma_rewrite_enabled(gcn_arch) {
- let flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+ if let Some(flags) = wmma_rewrite_flags(gcn_arch) {
unsafe { crate::ffi::LLVMZludaRewriteMmaF16Wmma(main.get(), flags) };
}
linked.link(main)?;
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -968,6 +968,9 @@
}
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
+// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
+// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
+// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
Module *M = unwrap(ModuleRef);
@@ -990,33 +993,39 @@
Type *V8 = FixedVectorType::get(I32, 8);
AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
AttrBuilder(Context, Reference->getAttributes().getFnAttrs()));
+ // arch-specific helpers (operand gathers, WMMAs), e4m3 MMAs, e4m3 fragment widening; the conversions
+ // between NVIDIA fragments, the transposed layout and f32 shadows are shared
+ const bool Gfx12 = (Flags & 2) != 0, NativeFp8 = Gfx12 && (Flags & 4) != 0;
+ const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
+ const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
+ const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
Helpers H;
- H.AFromB = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b", Attributes, V8, V2);
- H.BFromA = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_b_from_a", Attributes, V8, V4);
- H.Mma = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16", Attributes, V2, V8, V8, V2);
+ H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
+ H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
+ H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
- H.AFromBPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b_pair", Attributes, V8, V2, V2);
- H.MmaPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16_pair", Attributes, V4, V8, V8, V2, V2);
+ H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
{
Type *F = Type::getFloatTy(Context);
Type *V4F = FixedVectorType::get(F, 4);
Type *V8F = FixedVectorType::get(F, 8);
H.SFromT = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_from_t", Attributes, V4F, V2, F);
H.TFromS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_s", Attributes, V2, V4F, F);
- H.MmaS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16", Attributes, V4F, V8, V8, V4F, F, F);
- H.MmaPairS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16_pair", Attributes, V8F, V8, V8, V4F, V4F, F, F);
- H.MmaE4m3S = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3", Attributes, V4F, V8, V8, V8, V8, V4F);
+ H.MmaS = M->getOrInsertFunction((Arch + "s_mma_f16").c_str(), Attributes, V4F, V8, V8, V4F, F, F);
+ H.MmaPairS = M->getOrInsertFunction((Arch + "s_mma_f16_pair").c_str(), Attributes, V8F, V8, V8, V4F, V4F, F, F);
+ H.MmaE4m3S = M->getOrInsertFunction((E4m3 + "s_mma_e4m3").c_str(), Attributes, V4F, V8, V8, V8, V8, V4F);
H.MmaE4m3PairS =
- M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3_pair", Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
+ M->getOrInsertFunction((E4m3 + "s_mma_e4m3_pair").c_str(), Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
}
if (MmaE4m3 != nullptr)
{
- H.E4m3AToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_a_to_f16", Attributes, V8, V4);
- H.E4m3BToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_b_to_f16", Attributes, V4, V2);
- H.MmaE4m3 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3", Attributes, V2, V8, V8, V8, V8, V2);
+ H.E4m3AToF16 = M->getOrInsertFunction((Widen + "e4m3_a_to_f16").c_str(), Attributes, V8, V4);
+ H.E4m3BToF16 = M->getOrInsertFunction((Widen + "e4m3_b_to_f16").c_str(), Attributes, V4, V2);
+ H.MmaE4m3 = M->getOrInsertFunction((E4m3 + "t_mma_e4m3").c_str(), Attributes, V2, V8, V8, V8, V8, V2);
H.MmaE4m3Pair =
- M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3_pair", Attributes, V4, V8, V8, V8, V8, V2, V2);
+ M->getOrInsertFunction((E4m3 + "t_mma_e4m3_pair").c_str(), Attributes, V4, V8, V8, V8, V8, V2, V2);
}
unsigned Rewritten = 0;
for (Function &F : *M)
--- a/ptx/examples/d4r_emit.rs
+++ b/ptx/examples/d4r_emit.rs
@@ -1,9 +1,13 @@
use std::{env, fs, path::PathBuf};
use ptx_parser::parse_module_checked;
+// usage: d4r_emit PTX OUT_DIR [GPU_ARCH]: translates and compiles a PTX module offline (no GPU needed) for
+// GPU_ARCH (default gfx1101), with the same D4R_ZLUDA_* code-generation switches as the runtime, and writes
+// the IR stages, asm.s and module.hsaco to OUT_DIR
fn main() {
let input = env::args().nth(1).expect("PTX path");
let outdir = PathBuf::from(env::args().nth(2).unwrap_or_else(|| "/tmp/d4r-autoexposure-emitted".into()));
+ let arch = env::args().nth(3).unwrap_or_else(|| "gfx1101".into());
fs::create_dir_all(&outdir).expect("create output directory");
let bytes = fs::read(&input).expect("read PTX");
let text = std::str::from_utf8(&bytes).expect("PTX UTF-8").trim_end_matches('\0');
@@ -18,13 +22,14 @@
"opt.ll" => "opt.ll",
"asm" => "asm.s",
"o" => "object.o",
+ "elf" => "linked.elf", // the code object compile() returns (written again as module.hsaco)
other => panic!("unexpected stage: {other}"),
};
fs::write(outdir.join(filename), data).expect("write compiler stage");
};
let elf = llvm_zluda::compile(
&module.context,
- "gfx1101",
+ &arch,
module.llvm_ir,
linked_helpers,
module.attributes_ir,
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1696,6 +1696,349 @@
}
}
+// gfx12 (RDNA4) lowering of the same MMAs (LLVMZludaRewriteMmaF16Wmma with the gfx12 flag: helpers
+// d4r_wmma12_*, and d4r_wmma12f8_* for the e4m3 ones with native FP8 WMMA). The contract is the gfx11 one
+// above: NVIDIA fragments in, the transposed accumulator layout (lane l holds D[l % 16][2v + l / 16] as two
+// f16x2 words, or four floats for the f32 shadows), so the rewrite only switches helper names.
+//
+// A gfx12 WMMA operand holds 8 of the 16 K values per lane: half h = lane / 16 its own 8 (4 dwords, returned
+// in the low half of a wmma_uint8). With A' = B^T and B' = A^T as above, dword j of half h holds the k-pair of
+// NVIDIA thread t = 2h + (j >> 1): the low pair (k 2t, 2t + 1) for even j, the high pair (k 2t + 8, 2t + 9)
+// for odd j. Then every lane of a gather reads the same source register (4 bpermutes per operand; B' also
+// needs a DPP swap per dword for rows 8..15). For e4m3 fragments the same slots hold, after the widening
+// above, e4m3 k 8h .. 8h + 7 in order.
+// The result D' has VGPR i = row n = i + 8h (columns m = lane % 16): unpaired, rows 0..7 are the h = 0 lanes
+// and the h = 1 lanes are padding; paired (A' rows 8..15 = B1^T), D1 is in the h = 1 lanes. Converting from
+// and to the transposed layout takes 2 permlanex16 each way (4 for f32 shadows).
+//
+// D4R_WMMA12_SHIM (tests only): these helpers compiled for gfx11, every gfx12 WMMA done by the gfx11 one on
+// operands rebuilt in the gfx11 helpers' K order (and for e4m3 with the same scaled widening), so the results
+// are bit-identical to the gfx11 helpers and the gfx12 data movement can be checked on RDNA3.
+typedef _Float16 wmma_half8 __attribute__((ext_vector_type(8)));
+typedef uint32_t wmma_uint4 __attribute__((ext_vector_type(4)));
+typedef int wmma_int2 __attribute__((ext_vector_type(2)));
+#ifndef D4R_WMMA12_SHIM
+__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_f16(wmma_half8 a, wmma_half8 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16");
+__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_fp8_fp8(wmma_int2 a, wmma_int2 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32");
+#endif
+
+__device__ static uint32_t wmma12_half()
+{
+ return wmma_lane() / 16;
+}
+
+__device__ static uint32_t wmma12_other_half(uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_permlanex16(int(value), int(value), 0x76543210, 0xfedcba98, false, false));
+}
+
+__device__ static float wmma12_other_half_f(float value)
+{
+ return std::bit_cast<float>(wmma12_other_half(std::bit_cast<uint32_t>(value)));
+}
+
+// dword j of an operand gathered from two candidate registers: rows r < 8 take `low` from lane 4r + t, rows
+// r >= 8 take `high` from lane 4(r - 8) + t (t = 2h + (j >> 1)). The source lane is in the first half-wave,
+// while t differs between the two destination half-waves, so the two candidates must be gathered independently.
+__device__ static uint32_t wmma12_gather2(uint32_t low, uint32_t high, uint32_t j)
+{
+ const uint32_t row = wmma_lane() % 16;
+ const uint32_t t = (j >> 1) + 2 * wmma12_half();
+ const uint32_t lo = wmma_from_lane(row * 4 + t, low);
+ const uint32_t hi = wmma_from_lane((row % 8) * 4 + t, high);
+ return row < 8 ? lo : hi;
+}
+
+__device__ static wmma_uint4 wmma12_low4(wmma_uint8 v)
+{
+ return {v[0], v[1], v[2], v[3]};
+}
+
+// f32 D' = A' B' + C' (f16 operands in the slot order above)
+__device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+{
+#ifndef D4R_WMMA12_SHIM
+ return __zluda_wmma12_f32_16x16x16_f16(std::bit_cast<wmma_half8>(wmma12_low4(a)), std::bit_cast<wmma_half8>(wmma12_low4(b)), c);
+#else
+ // gfx11 operand dword p (K 2p, 2p + 1 in the gfx11 helpers' order) is gfx12 dword jp of half hp
+ const bool high = wmma12_half() != 0;
+ wmma_uint8 a11, b11;
+ for (uint32_t p = 0; p < 8; p++)
+ {
+ const uint32_t hp = (p & 3) >> 1, jp = 2 * (p & 1) + (p >> 2);
+ const uint32_t ao = wmma12_other_half(a[jp]), bo = wmma12_other_half(b[jp]);
+ a11[p] = (hp != 0) == high ? a[jp] : ao;
+ b11[p] = (hp != 0) == high ? b[jp] : bo;
+ }
+ // accumulators: gfx12 VGPR i = row i + 8h, gfx11 VGPR i = row 2i + h
+ wmma_float8 co, c11;
+ for (uint32_t i = 0; i < 8; i++)
+ co[i] = wmma12_other_half_f(c[i]);
+ for (uint32_t i = 0; i < 8; i++)
+ c11[i] = high ? (i < 4 ? co[2 * i + 1] : c[2 * i - 7]) : (i < 4 ? c[2 * i] : co[2 * i - 8]);
+ wmma_float8 d11 = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a11), std::bit_cast<wmma_half16>(b11), c11);
+ wmma_float8 dout, d;
+ for (uint32_t i = 0; i < 8; i++)
+ dout[i] = wmma12_other_half_f(d11[i]);
+ for (uint32_t i = 0; i < 8; i++)
+ d[i] = high ? ((i & 1) ? d11[(i >> 1) + 4] : dout[(i >> 1) + 4]) : ((i & 1) ? dout[i >> 1] : d11[i >> 1]);
+ return d;
+#endif
+}
+
+// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
+// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
+__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+{
+ const wmma_int2 a8 = {int(__builtin_amdgcn_perm(a[1], a[0], 0x05040100u)), int(__builtin_amdgcn_perm(a[3], a[2], 0x05040100u))};
+ const wmma_int2 b8 = {int(__builtin_amdgcn_perm(b[1], b[0], 0x05040100u)), int(__builtin_amdgcn_perm(b[3], b[2], 0x05040100u))};
+#ifndef D4R_WMMA12_SHIM
+ return __zluda_wmma12_f32_16x16x16_fp8_fp8(a8, b8, c);
+#else
+ // the gfx11 helpers' scaled widening (value * 2^-8 per operand, C * 2^-16): exact powers of two
+ auto widen = [](wmma_int2 q) {
+ auto lo = e4m3x4_to_scaled_f16x2x2(uint32_t(q.x)), hi = e4m3x4_to_scaled_f16x2x2(uint32_t(q.y));
+ return wmma_uint8{lo.x, lo.y, hi.x, hi.y, 0u, 0u, 0u, 0u};
+ };
+ return wmma12_f16(widen(a8), widen(b8), c * 0x1p-16f) * 0x1p16f;
+#endif
+}
+
+// transposed f16 accumulator(s) <-> D' / C'
+__device__ static wmma_float8 wmma12_c_from_t(uint2::Native_vec_ c_t)
+{
+ // lane h = 0 needs rows 0..7: its own rows 0, 2, 4, 6 and the other half's 1, 3, 5, 7 (h = 1 lanes: padding)
+ const uint32_t o0 = wmma12_other_half(c_t.x), o1 = wmma12_other_half(c_t.y);
+ f16x2 e01 = std::bit_cast<f16x2>(c_t.x), e23 = std::bit_cast<f16x2>(c_t.y);
+ f16x2 d01 = std::bit_cast<f16x2>(o0), d23 = std::bit_cast<f16x2>(o1);
+ return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
+}
+
+__device__ static wmma_float8 wmma12_c_from_t_pair(uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+{
+ // h = 0: C0 rows 0..7 (own even, other odd); h = 1: C1 rows 0..7 (other even, own odd)
+ const bool high = wmma12_half() != 0;
+ const uint2::Native_vec_ own = high ? c1_t : c0_t, send = high ? c0_t : c1_t;
+ const uint32_t r0 = wmma12_other_half(send.x), r1 = wmma12_other_half(send.y);
+ f16x2 o01 = std::bit_cast<f16x2>(own.x), o23 = std::bit_cast<f16x2>(own.y);
+ f16x2 p01 = std::bit_cast<f16x2>(r0), p23 = std::bit_cast<f16x2>(r1);
+ f16x2 e01 = high ? p01 : o01, e23 = high ? p23 : o23, d01 = high ? o01 : p01, d23 = high ? o23 : p23;
+ return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
+}
+
+__device__ static void wmma12_rows_f16(wmma_float8 d, uint2::Native_vec_ &even, uint2::Native_vec_ &odd)
+{
+ f16x2 e01{static_cast<f16>(d[0]), static_cast<f16>(d[2])}, e23{static_cast<f16>(d[4]), static_cast<f16>(d[6])};
+ f16x2 o01{static_cast<f16>(d[1]), static_cast<f16>(d[3])}, o23{static_cast<f16>(d[5]), static_cast<f16>(d[7])};
+ even = {std::bit_cast<uint32_t>(e01), std::bit_cast<uint32_t>(e23)};
+ odd = {std::bit_cast<uint32_t>(o01), std::bit_cast<uint32_t>(o23)};
+}
+
+__device__ static uint2::Native_vec_ wmma12_t_from_d(wmma_float8 d)
+{
+ // h = 0 keeps rows 0, 2, 4, 6; h = 1 takes rows 1, 3, 5, 7 from the other half
+ uint2::Native_vec_ even, odd;
+ wmma12_rows_f16(d, even, odd);
+ const uint2::Native_vec_ other = {wmma12_other_half(odd.x), wmma12_other_half(odd.y)};
+ return wmma12_half() != 0 ? other : even;
+}
+
+__device__ static uint4::Native_vec_ wmma12_t_from_d_pair(wmma_float8 d)
+{
+ // h = 0 holds D0 rows 0..7, h = 1 D1 rows 0..7; lane h wants rows 2v + h of both
+ const bool high = wmma12_half() != 0;
+ uint2::Native_vec_ even, odd;
+ wmma12_rows_f16(d, even, odd);
+ const uint2::Native_vec_ send = high ? even : odd;
+ const uint2::Native_vec_ recv = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
+ return high ? uint4::Native_vec_{recv.x, recv.y, odd.x, odd.y} : uint4::Native_vec_{even.x, even.y, recv.x, recv.y};
+}
+
+// the same for f32 shadows (element v = row 2v + h)
+__device__ static wmma_float8 wmma12_c_from_s(wmma_float4 c)
+{
+ wmma_float4 o;
+ for (uint32_t v = 0; v < 4; v++)
+ o[v] = wmma12_other_half_f(c[v]);
+ return {c[0], o[0], c[1], o[1], c[2], o[2], c[3], o[3]};
+}
+
+__device__ static wmma_float8 wmma12_c_from_s_pair(wmma_float4 c0, wmma_float4 c1)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float8 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float own = high ? c1[v] : c0[v], recv = wmma12_other_half_f(high ? c0[v] : c1[v]);
+ r[2 * v] = high ? recv : own;
+ r[2 * v + 1] = high ? own : recv;
+ }
+ return r;
+}
+
+__device__ static wmma_float4 wmma12_s_from_d(wmma_float8 d)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float4 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float odd = wmma12_other_half_f(d[2 * v + 1]);
+ r[v] = high ? odd : d[2 * v];
+ }
+ return r;
+}
+
+__device__ static wmma_float8 wmma12_s_from_d_pair(wmma_float8 d)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float8 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float recv = wmma12_other_half_f(high ? d[2 * v] : d[2 * v + 1]);
+ r[v] = high ? recv : d[2 * v];
+ r[4 + v] = high ? d[2 * v + 1] : recv;
+ }
+ return r;
+}
+
+extern "C"
+{
+ wmma_uint8 FUNC(d4r_wmma12_t_a_from_b)(uint2::Native_vec_ b)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t n = wmma_lane() % 16;
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ {
+ uint32_t value = wmma_from_lane((n % 8) * 4 + (j >> 1) + 2 * wmma12_half(), (j & 1) ? b.y : b.x);
+ result[j] = n < 8 ? value : 0u;
+ }
+ return result;
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12_t_b_from_a)(uint4::Native_vec_ a)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ result[j] = wmma12_gather2((j & 1) ? a.z : a.x, (j & 1) ? a.w : a.y, j);
+ return result;
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12_t_a_from_b_pair)(uint2::Native_vec_ b0, uint2::Native_vec_ b1)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ result[j] = wmma12_gather2((j & 1) ? b0.y : b0.x, (j & 1) ? b1.y : b1.x, j);
+ return result;
+ }
+
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ return wmma12_t_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_t(c_t)));
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
+ }
+
+ wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
+ {
+ return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
+ }
+
+ wmma_float8 FUNC(d4r_wmma12_s_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c0, wmma_float4 c1, float up,
+ float down)
+ {
+ return wmma12_s_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_s_pair(c0 * up, c1 * up))) * down;
+ }
+
+ // e4m3 with the f16 widening (d4r_wmma_e4m3_a_to_f16 / b_to_f16 above): two chained f16 WMMAs in the
+ // 2^-16 scaled domain, like the gfx11 helpers
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
+ {
+ wmma_float8 c = wmma12_c_from_t(c_t) * 0x1p-16f;
+ c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
+ return wmma12_t_from_d(c * 0x1p16f);
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
+ uint2::Native_vec_ c1_t)
+ {
+ wmma_float8 c = wmma12_c_from_t_pair(c0_t, c1_t) * 0x1p-16f;
+ c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
+ return wmma12_t_from_d_pair(c * 0x1p16f);
+ }
+
+ wmma_float4 FUNC(d4r_wmma12_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c)
+ {
+ return wmma12_s_from_d(wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s(c))));
+ }
+
+ wmma_float8 FUNC(d4r_wmma12_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
+ {
+ return wmma12_s_from_d_pair(
+ wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s_pair(c0, c1))));
+ }
+
+ // Native FP8 (d4r_wmma12f8_*, D4R_ZLUDA_WMMA_FP8_NATIVE): the "widening" keeps the e4m3 bytes, two per
+ // dword in the low 16 bits and in the same slots the f16 widening fills, so the f16 gathers above move them;
+ // the MMA packs this half's 8 bytes (e4m3 k 8h .. 8h + 7 in order) into the FP8 operand. OCP e4m3 (gfx12)
+ // is NVIDIA's e4m3, so the bytes need no conversion. Shadows keep the 2^-16 scaled domain of the e4m3
+ // functions (exact power-of-two rescaling around the WMMA).
+ __device__ static uint2::Native_vec_ e4m3x4_split(uint32_t packed)
+ {
+ return {__builtin_amdgcn_perm(0u, packed, 0x0c0c0100u), __builtin_amdgcn_perm(0u, packed, 0x0c0c0302u)};
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12f8_e4m3_a_to_f16)(uint4::Native_vec_ a)
+ {
+ auto x = e4m3x4_split(a.x), y = e4m3x4_split(a.y), z = e4m3x4_split(a.z), w = e4m3x4_split(a.w);
+ return {x.x, y.x, x.y, y.y, z.x, w.x, z.y, w.y};
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12f8_e4m3_b_to_f16)(uint2::Native_vec_ b)
+ {
+ auto x = e4m3x4_split(b.x), y = e4m3x4_split(b.y);
+ return {x.x, x.y, y.x, y.y};
+ }
+
+ uint2::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
+ {
+ return wmma12_t_from_d(wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t(c_t))));
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
+ uint2::Native_vec_ c1_t)
+ {
+ return wmma12_t_from_d_pair(
+ wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t_pair(c0_t, c1_t))));
+ }
+
+ wmma_float4 FUNC(d4r_wmma12f8_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c)
+ {
+ wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s(c * 0x1p16f)));
+ return wmma12_s_from_d(d) * 0x1p-16f;
+ }
+
+ wmma_float8 FUNC(d4r_wmma12f8_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
+ {
+ wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high,
+ wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s_pair(c0 * 0x1p16f, c1 * 0x1p16f)));
+ return wmma12_s_from_d_pair(d) * 0x1p-16f;
+ }
+}
+
template <typename T, typename A, typename B, int BOffset, typename OcklFunc>
__device__ T dp2a_impl(T a, T b, T c, OcklFunc ockl_func)
{
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -356,6 +356,7 @@
wmma: llvm_zluda::compile::wmma_rewrite_requested(),
wmma_fp8: llvm_zluda::compile::wmma_fp8_rewrite_requested(),
param_provenance: llvm_zluda::compile::param_provenance_enabled(),
+ wmma_fp8_native: llvm_zluda::compile::wmma_fp8_native_effective(gcn_arch),
};
let mut cache_with_key = match (text, global_state.cache_path.as_ref()) {
(Some(text), Some(p)) => (|| {
@@ -397,6 +398,10 @@
wmma: bool,
wmma_fp8: bool,
param_provenance: bool,
+ // d4r: native FP8 WMMA where it takes effect (gfx12); absent from the key everywhere else, so the setting
+ // (d4r.ini NativeFp8) never forces a recompile on GPUs that ignore it
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ wmma_fp8_native: bool,
}
fn get_hip_properties<'a>() -> Result<hipDeviceProp_tR0600, CUerror> {
@@ -430,6 +435,7 @@
&& !matches!(
key.as_str(),
"D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
+ | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
.map(|(key, value)| format!("{key}={}", value.trim()))
patches/zluda/0008-native-windows-build.patch
+203
View File
@@ -0,0 +1,203 @@
Windows LLVM-MinGW build, portable helper assertions and optional diagnostic UI.
Apply after upstream d4r patches 0002 through 0007, on ZLUDA ee2f25a.
diff --git a/ext/highs-sys/build.rs b/ext/highs-sys/build.rs
--- a/ext/highs-sys/build.rs
+++ b/ext/highs-sys/build.rs
@@ -66,11 +66,12 @@
let linux = target.contains("linux");
let mingw = target.contains("pc-windows-gnu");
if apple {
println!("cargo:rustc-link-lib=dylib=c++");
} else if linux || mingw {
- println!("cargo:rustc-link-lib=dylib=stdc++");
+ let cxx_library = env::var("CXXSTDLIB").unwrap_or_else(|_| "stdc++".to_owned());
+ println!("cargo:rustc-link-lib=dylib={cxx_library}");
}
println!("cargo:rerun-if-changed=../HiGHS/highs/interfaces/highs_c_api.h");
true
}
diff --git a/ext/hip_runtime-sys/build.rs b/ext/hip_runtime-sys/build.rs
--- a/ext/hip_runtime-sys/build.rs
+++ b/ext/hip_runtime-sys/build.rs
@@ -1,8 +1,12 @@
fn main() {
if cfg!(windows) {
- println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
+ println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
+ } else {
+ println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ }
} else {
println!("cargo:rustc-link-lib=dylib=amdhip64");
println!("cargo:rustc-link-search=native=/opt/rocm/lib/");
}
}
diff --git a/llvm_zluda/build.rs b/llvm_zluda/build.rs
--- a/llvm_zluda/build.rs
+++ b/llvm_zluda/build.rs
@@ -19,10 +19,18 @@
fn main() {
let bc_path1 = "src/device-libs/ockl.bc";
println!("cargo:rerun-if-changed={}", bc_path1);
check_lfs_file(bc_path1);
+
+ println!("cargo:rerun-if-env-changed=ZLUDA_LLVM_BUILD_DIR");
+ if let Some(directory) = env::var_os("ZLUDA_LLVM_BUILD_DIR") {
+ let llvm_build_path = PathBuf::from(directory);
+ println!("cargo:build-path={}", llvm_build_path.display());
+ link_prebuilt_llvm(&llvm_build_path);
+ return;
+ }
let mut cmake = Config::new(r"../ext/llvm-project/llvm");
try_use_sccache(&mut cmake);
try_use_ninja(&mut cmake);
cmake
@@ -57,16 +65,31 @@
cmake.build();
}
let cmake_profile = cmake.get_profile();
let llvm_build_path = get_llvm_build_path(&llvm_dir, cmake_profile);
println!("cargo:build-path={}", llvm_build_path.display());
+ link_prebuilt_llvm(&llvm_build_path);
+}
+
+fn link_prebuilt_llvm(llvm_build_path: &PathBuf) {
let (cxxflags, ldflags, libdir, lib_names, system_libs) =
llvm_config(&llvm_build_path.join("bin/llvm-config")).unwrap();
println!("cargo:rustc-link-arg={ldflags}");
println!("cargo:rustc-link-search=native={libdir}");
for lib in system_libs.split_ascii_whitespace() {
- println!("cargo:rustc-link-arg={lib}");
+ if let Some(name) = lib.strip_prefix("-l") {
+ println!("cargo:rustc-link-lib={name}");
+ } else {
+ println!("cargo:rustc-link-arg={lib}");
+ }
+ }
+ // llvm-config omits system libraries on Windows. Propagate these through
+ // this rlib to its final cdylib consumer (link-arg alone cannot do that).
+ if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") {
+ for library in ["psapi", "shell32", "ole32", "uuid", "advapi32", "ws2_32", "ntdll"] {
+ println!("cargo:rustc-link-lib={library}");
+ }
}
link_lld_components();
link_llvm_components(lib_names);
compile_cxx_lib(cxxflags, &llvm_build_path);
}
diff --git a/zluda/build.rs b/zluda/build.rs
--- a/zluda/build.rs
+++ b/zluda/build.rs
@@ -1,11 +1,15 @@
use vergen_gix::{Emitter, Gix};
fn main() {
if cfg!(windows) {
- println!("cargo:rustc-link-arg=delayimp.lib");
- println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
+ println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
+ } else {
+ println!("cargo:rustc-link-arg=delayimp.lib");
+ println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ }
let dll_path = "bin/nvcudart_hybrid64.dll";
println!("cargo:rerun-if-changed={}", dll_path);
check_lfs_file(dll_path);
}
let git = Gix::builder().sha(false).build();
diff --git a/zluda_windows/src/lib.rs b/zluda_windows/src/lib.rs
--- a/zluda_windows/src/lib.rs
+++ b/zluda_windows/src/lib.rs
@@ -11,27 +11,27 @@
use uuid::uuid;
use widestring::{u16str, U16Str, U16String};
use windows::{
core::{w, HRESULT, PCWSTR},
Win32::{
- Foundation::{HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
+ Foundation::{FreeLibrary, HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
System::{
JobObjects::{
AssignProcessToJobObject, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
},
LibraryLoader::{
- GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, LoadLibraryExW,
+ GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, GetProcAddress, LoadLibraryExW,
GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS,
- LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR,
+ LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, LOAD_LIBRARY_SEARCH_SYSTEM32,
},
},
UI::{
Controls::{
- TaskDialogIndirect, TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
+ TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
TASKDIALOG_NOTIFICATIONS, TDCBF_CLOSE_BUTTON, TDF_ENABLE_HYPERLINKS,
TDN_HYPERLINK_CLICKED, TD_ERROR_ICON,
},
WindowsAndMessaging::IDCLOSE,
},
@@ -487,11 +487,23 @@
pszFooter: PCWSTR::null(),
pfCallback: Some(task_dialog_callback),
lpCallbackData: 0,
cxWidth: 0,
};
- TaskDialogIndirect(&config, None, None, None).ok();
+ // A CUDA DLL may be loaded by an executable with no Common Controls
+ // v6 activation manifest. Do not make its diagnostic UI an eager DLL
+ // import: comctl32 v5 has no TaskDialogIndirect export.
+ if let Ok(controls) = LoadLibraryExW(w!("comctl32.dll"), None, LOAD_LIBRARY_SEARCH_SYSTEM32) {
+ if let Some(raw) = GetProcAddress(controls, windows::core::s!("TaskDialogIndirect")) {
+ let show: unsafe extern "system" fn(*const TASKDIALOGCONFIG, *mut i32, *mut i32, *mut i32) -> HRESULT =
+ mem::transmute(raw);
+ let _ = show(&config, ptr::null_mut(), ptr::null_mut(), ptr::null_mut());
+ } else {
+ eprintln!("ZLUDA failed to load {}; set HIP_PATH to the selected HIP SDK", redirect_names[0]);
+ }
+ let _ = FreeLibrary(controls);
+ }
}
result
}
unsafe extern "system" fn task_dialog_callback(
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -405,11 +405,26 @@
uint32_t line,
uint64_t function,
uint64_t char_size)
{
(void)char_size;
+#if defined(_WIN32) || defined(_WIN64)
+ // HIP on Windows only supplies the wide _wassert entry point. Keep the
+ // CUDA assertion's ASCII diagnostics and trap through public OCKL APIs.
+ auto output = __ockl_fprintf_stderr_begin();
+ const char* parts[] = {"CUDA assertion: ", (const char*)message, " at ", (const char*)file,
+ " in ", (const char*)function, "\n"};
+ for (const char* part : parts) {
+ const char* end = part;
+ while (*end) ++end;
+ output = __ockl_fprintf_append_string_n(output, part, end - part, 0);
+ }
+ __ockl_fprintf_append_args(output, 1, line, 0, 0, 0, 0, 0, 0, 1);
+ __builtin_trap();
+#else
[[clang::noinline]] __assert_fail((const char *)message, (const char *)file, line, (const char *)function);
+#endif
}
// * Smallest denormal is 1.4 × 10^-45
// * Smallest normal is ~1.175494351 × 10^(-38)
// * Now, 1.175494351×10^-38 / 1.4 × 10^-45 = 8396388 + 31/140
patches/zluda/0009-resource-descriptor-queries.patch
+124
View File
@@ -0,0 +1,124 @@
Implement the CUDA texture resource descriptor query required by NGX Evaluate.
Regression: tools/windows/cuda_image_probe.cpp in the d4r Windows port.
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -67,12 +67,14 @@
cuda_macros::cuda_function_declarations!(
unimplemented,
implemented
<= [
cuArray3DCreate_v2,
+ cuArray3DGetDescriptor_v2,
cuArrayCreate_v2,
cuArrayDestroy,
+ cuArrayGetDescriptor_v2,
cuCtxCreate_v2,
cuCtxCreate_v4,
cuCtxDestroy_v2,
cuCtxGetApiVersion,
cuCtxGetCurrent,
@@ -203,10 +205,11 @@
cuSurfObjectCreate,
cuSurfObjectDestroy,
cuSurfObjectGetResourceDesc,
cuTexObjectCreate,
cuTexObjectDestroy,
+ cuTexObjectGetResourceDesc,
cuTexRefSetAddress_v2,
cuTexRefSetAddressMode,
cuTexRefSetArray,
cuTexRefSetFilterMode,
cuTexRefSetFlags,
diff --git a/zluda/src/impl/tex.rs b/zluda/src/impl/tex.rs
--- a/zluda/src/impl/tex.rs
+++ b/zluda/src/impl/tex.rs
@@ -1,19 +1,58 @@
use crate::r#impl::hipfix;
use hip_runtime_sys::*;
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+
+fn object_resources() -> &'static Mutex<HashMap<usize, Vec<u8>>> {
+ static RESOURCES: OnceLock<Mutex<HashMap<usize, Vec<u8>>>> = OnceLock::new();
+ RESOURCES.get_or_init(|| Mutex::new(HashMap::new()))
+}
pub(crate) unsafe fn object_create(
p_tex_object: *mut hipTextureObject_t,
p_res_desc: *const HIP_RESOURCE_DESC,
p_tex_desc: *const HIP_TEXTURE_DESC,
p_res_view_desc: *const HIP_RESOURCE_VIEW_DESC,
) -> hipError_t {
- hipTexObjectCreate(p_tex_object, p_res_desc, p_tex_desc, p_res_view_desc)
+ let output = p_tex_object.as_mut().ok_or(hipErrorCode_t::InvalidValue)?;
+ let descriptor = p_res_desc.as_ref().ok_or(hipErrorCode_t::InvalidValue)?;
+ hipTexObjectCreate(output, descriptor, p_tex_desc, p_res_view_desc)?;
+ let bytes = std::slice::from_raw_parts(p_res_desc.cast::<u8>(), std::mem::size_of::<HIP_RESOURCE_DESC>()).to_vec();
+ match object_resources().lock() {
+ Ok(mut resources) => {
+ resources.insert(*output as usize, bytes);
+ Ok(())
+ }
+ Err(_) => {
+ let _ = hipDestroyTextureObject(*output);
+ *output = std::ptr::null_mut();
+ Err(hipErrorCode_t::Unknown)
+ }
+ }
}
pub(crate) unsafe fn object_destroy(tex_object: hipTextureObject_t) -> hipError_t {
- hipDestroyTextureObject(tex_object)
+ let mut resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
+ if !resources.contains_key(&(tex_object as usize)) { return Err(hipErrorCode_t::InvalidValue); }
+ hipDestroyTextureObject(tex_object)?;
+ resources.remove(&(tex_object as usize));
+ Ok(())
+}
+
+pub(crate) unsafe fn object_get_resource_desc(
+ descriptor: &mut HIP_RESOURCE_DESC,
+ tex_object: hipTextureObject_t,
+) -> hipError_t {
+ // NGX probes surface handles as textures before trying the surface API.
+ // Passing that foreign handle to HIP dereferences invalid host metadata.
+ // Return a CUDA error for unknown handles and preserve the complete CUDA
+ // descriptor (HIP's query leaves the flags/reserved bytes unspecified).
+ let resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
+ let bytes = resources.get(&(tex_object as usize)).ok_or(hipErrorCode_t::InvalidValue)?;
+ std::ptr::copy_nonoverlapping(bytes.as_ptr(), (descriptor as *mut HIP_RESOURCE_DESC).cast::<u8>(), bytes.len());
+ Ok(())
}
pub(crate) unsafe fn ref_set_array(
texref: *mut textureReference,
array: hipArray_t,
diff --git a/zluda/src/impl/array.rs b/zluda/src/impl/array.rs
--- a/zluda/src/impl/array.rs
+++ b/zluda/src/impl/array.rs
@@ -70,5 +70,23 @@
handle: *mut hipArray_t,
desc: *const HIP_ARRAY3D_DESCRIPTOR,
) -> hipError_t {
hipArray3DCreate(handle, desc)
}
+
+pub(crate) unsafe fn get_descriptor_v2(
+ output: &mut HIP_ARRAY_DESCRIPTOR,
+ array: hipArray_t,
+) -> hipError_t {
+ hipArrayGetDescriptor(output, array)?;
+ if let Some(format) = original_format(array)? {
+ output.Format = hipArray_Format(format);
+ }
+ Ok(())
+}
+
+pub(crate) unsafe fn _3d_get_descriptor_v2(
+ output: &mut HIP_ARRAY3D_DESCRIPTOR,
+ array: hipArray_t,
+) -> hipError_t {
+ hipArray3DGetDescriptor(output, array)
+}
patches/zluda/0010-native-identity-replay-windows-cache.patch
+440
View File
@@ -0,0 +1,440 @@
Subject: [PATCH] Guard and own native overrides; capture K replay on Windows
Native replacements require the upstream PTX identity manifest. Preserve native
module/image ownership, bypass redundant JIT for completely replaced modules,
include the loaded Windows DLL fingerprint in cache identity, and expose opt-in
private real-weight replay captures with explicit native/translated launch logs.
diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
new file mode 100644
index 0000000..875b2e3
--- /dev/null
+++ b/zluda/src/impl/d4r_replay.rs
@@ -0,0 +1,109 @@
+//! Opt-in, synchronous diagnostic captures. Never enabled in the game fast path.
+//! Captures can contain proprietary network weights; keep them out of releases.
+use hip_runtime_sys::*;
+use std::{collections::HashMap, ffi::{c_void, CStr}, fs, path::PathBuf, sync::{Mutex, OnceLock}};
+
+struct Config { directory: PathBuf, filter: String, limit: u32 }
+static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
+static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
+
+pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
+
+fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
+ eprintln!("[d4r-replay] capture failed: {error}");
+ hipErrorCode_t::r#Unknown
+}
+
+impl Capture {
+ fn save(&self, destination: &std::path::Path) -> hipError_t {
+ fs::create_dir_all(destination).map_err(io_error)?;
+ for (id, &(base, size)) in self.allocations.iter().enumerate() {
+ let mut bytes = vec![0u8; size];
+ unsafe { hipMemcpyDtoH(bytes.as_mut_ptr().cast(), hipDeviceptr_t(base as *mut c_void), size) }?;
+ fs::write(destination.join(format!("alloc-{id}.bin")), bytes).map_err(io_error)?;
+ }
+ Ok(())
+ }
+
+ pub(crate) fn finish(self, stream: hipStream_t) -> hipError_t {
+ unsafe { hipStreamSynchronize(stream) }?;
+ self.save(&self.directory.join("gpu-output"))?;
+ eprintln!("[d4r-replay] complete path={}", self.directory.display());
+ Ok(())
+ }
+}
+
+pub(crate) unsafe fn begin(
+ name: &CStr, stream: hipStream_t, parameters: *mut *mut c_void,
+ extra: *mut *mut c_void, launch: [u32; 7],
+) -> Result<Option<Capture>, hipErrorCode_t> {
+ let Some(config) = CONFIG.get_or_init(|| std::env::var_os("D4R_CUDA_REPLAY_DUMP_DIR").map(|directory| Config {
+ directory: directory.into(),
+ filter: std::env::var("D4R_CUDA_REPLAY_DUMP_FILTER").unwrap_or_default(),
+ limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
+ })) else { return Ok(None) };
+ let name = name.to_str().map_err(io_error)?;
+ // Only the audited, single 176-byte K argument ABI is captured here.
+ if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
+ return Ok(None);
+ }
+ let mut block = std::ptr::null::<u8>();
+ let mut bytes = None;
+ if !extra.is_null() {
+ for i in (0..6).step_by(2) {
+ match *extra.add(i) as usize {
+ 1 => block = (*extra.add(i + 1)).cast(),
+ 2 => bytes = Some(*(*extra.add(i + 1) as *const usize)),
+ 0 => break,
+ _ => return Err(hipErrorCode_t::r#InvalidValue),
+ }
+ }
+ } else if !parameters.is_null() {
+ block = (*parameters).cast();
+ bytes = Some(176);
+ }
+ if block.is_null() || bytes != Some(176) {
+ return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
+ }
+ let sequence = {
+ let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
+ let count = counts.entry(name.to_owned()).or_default();
+ if *count >= config.limit { return Ok(None); }
+ *count += 1;
+ *count
+ };
+ hipStreamSynchronize(stream)?;
+ let arguments = std::slice::from_raw_parts(block, 176);
+ let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
+ fs::create_dir_all(&directory).map_err(io_error)?;
+ fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
+ let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
+ launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
+ let mut allocations = Vec::<(usize, usize)>::new();
+ for offset in (0..176).step_by(8) {
+ let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
+ if pointer == 0 { continue; }
+ let mut base = hipDeviceptr_t(std::ptr::null_mut());
+ let mut size = 0;
+ if hipMemGetAddressRange(&mut base, &mut size, hipDeviceptr_t(pointer as *mut c_void)).is_err() { continue; }
+ let base = base.0 as usize;
+ if pointer < base || pointer - base >= size { continue; }
+ if size > 512 * 1024 * 1024 { return Err(io_error("allocation exceeds diagnostic 512 MiB limit")); }
+ let id = match allocations.iter().position(|&(b, _)| b == base) {
+ Some(id) => id,
+ None => {
+ let id = allocations.len();
+ allocations.push((base, size));
+ manifest.push_str(&format!("alloc {id} 0x{base:x} {size}\n"));
+ id
+ }
+ };
+ manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
+ }
+ if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
+ fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
+ let capture = Capture { directory, allocations };
+ capture.save(&capture.directory)?;
+ eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
+ Ok(Some(capture))
+}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index 033d9a0..7a7470a 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -7,6 +7,8 @@ use hip_runtime_sys::*;
use std::mem;
pub(crate) struct Function {
+ pub(crate) name: std::ffi::CString,
+ pub(crate) native_owner: Option<super::module::NativeModule>,
pub(crate) base: hipFunction_t,
pub(crate) sm_version: u32,
pub(crate) explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
@@ -144,6 +146,8 @@ pub(crate) fn launch_kernel(
}
translated_extra_storage.as_mut_ptr()
};
+ let replay = unsafe { super::d4r_replay::begin(&f.name, stream, kernel_params, extra,
+ [grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y, block_dim_z, shared_mem_bytes]) }?;
// the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
let mut block: *const u8 = std::ptr::null();
@@ -206,7 +210,11 @@ pub(crate) fn launch_kernel(
}
let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
let (grid_dim_x, grid_dim_y, grid_dim_z) = if f.grid_x != 0 { (f.grid_x, 1, 1) } else { (grid_dim_x, grid_dim_y, grid_dim_z) };
- unsafe {
+ if std::env::var_os("D4R_ZLUDA_VERBOSE").is_some() {
+ eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
+ f.name, if f.native_owner.is_some() { "native" } else { "translated" });
+ }
+ let result = unsafe {
hipModuleLaunchKernel(
f.base,
grid_dim_x,
@@ -220,7 +228,10 @@ pub(crate) fn launch_kernel(
kernel_params,
extra_for_hip,
)
- }
+ };
+ result?;
+ if let Some(replay) = replay { replay.finish(stream)?; }
+ Ok(())
}
pub(crate) unsafe fn set_attribute(
diff --git a/zluda/src/impl/library.rs b/zluda/src/impl/library.rs
index 98be33e..1319792 100644
--- a/zluda/src/impl/library.rs
+++ b/zluda/src/impl/library.rs
@@ -21,8 +21,8 @@ impl Library {
let module_lock = self.modules.get(device).ok_or(CUerror::INVALID_DEVICE)?;
*module_lock.get_or_init(|| match self.data {
LibraryData::Lazy(lib) => {
- let (module, sm_version, meta32) = module::load_hip_module(lib)?;
- Ok(Module::new(module, sm_version, meta32).wrap())
+ let (module, sm_version, meta32, ptx_hash, native_image) = module::load_hip_module(lib)?;
+ Ok(Module::new(module, sm_version, meta32, ptx_hash, native_image).wrap())
}
LibraryData::Eager(()) => Err(CUerror::NOT_SUPPORTED),
})
diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
index 5329d32..14e0aee 100644
--- a/zluda/src/impl/mod.rs
+++ b/zluda/src/impl/mod.rs
@@ -5,6 +5,7 @@ pub(super) mod array;
pub(super) mod context;
pub(super) mod device;
pub(super) mod driver;
+pub(super) mod d4r_replay;
pub(super) mod event;
pub(super) mod function;
pub(super) mod graph;
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 9f72f2f..5a79368 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -19,6 +19,8 @@ pub(crate) struct Module {
pub(crate) base: hipModule_t,
pub(crate) sm_version: u32,
pub(crate) bit32: Option<Metadata32Bit>,
+ ptx_hash: Option<u64>,
+ _native_base_image: Option<Vec<u8>>,
mutable: Mutex<ModuleMutable>,
}
@@ -33,12 +35,14 @@ impl ModuleMutable {
sm_version: u32,
name: CString,
explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
+ ptx_hash: Option<u64>,
) -> Result<&Function, CUerror> {
Ok(match self.functions.entry(name) {
hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
hash_map::Entry::Vacant(entry) => {
- let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
- Some(native) => native,
+ let mut native_owner = None;
+ let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key(), ptx_hash) {
+ Some((native, owner)) => { native_owner = Some(owner); native },
None => {
let mut func_handle = unsafe { std::mem::zeroed() };
unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
@@ -46,6 +50,8 @@ impl ModuleMutable {
}
};
let func = Box::new(Function {
+ name: entry.key().clone(),
+ native_owner,
base: func_handle,
sm_version,
explicit_args_size_align,
@@ -69,19 +75,48 @@ impl ModuleMutable {
/// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
/// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
/// (persistent kernels, which derive the original grid from their parameters) the grid.
+pub(crate) struct NativeModule {
+ module: hipModule_t,
+ _image: Vec<u8>,
+}
+
+impl Drop for NativeModule {
+ fn drop(&mut self) {
+ if self.module.0.is_null() { return; }
+ if let Err(error) = unsafe { hipModuleUnload(self.module) } {
+ eprintln!("[d4r-native] unload failed: {error:?}");
+ }
+ }
+}
+
fn native_override(
name: &CStr,
-) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
+ ptx_hash: Option<u64>,
+) -> Option<((hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32), NativeModule)> {
let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
+ let manifest = std::path::Path::new(&dir).join("d4r-kernels.txt");
+ let allowed = fs::read_to_string(&manifest).ok().is_some_and(|text| text.lines().any(|line| {
+ let mut fields = line.split_whitespace();
+ fields.next() == name.to_str().ok()
+ && fields.next().and_then(|s| u64::from_str_radix(s, 16).ok()) == ptx_hash
+ && ptx_hash.is_some()
+ }));
+ if !allowed {
+ eprintln!("[d4r-native] miss kernel={name:?} reason=ptx_identity hash={ptx_hash:016x?} fallback=translated");
+ return None;
+ }
let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
- let image = fs::read(&path).ok()?;
- // HIP may reference the image after loading; the override lives for the whole process.
- let image: &'static [u8] = Box::leak(image.into_boxed_slice());
+ let image = match fs::read(&path) {
+ Ok(image) => image,
+ Err(error) => { eprintln!("[d4r-native] miss kernel={name:?} reason=object_missing error={error} fallback=translated"); return None; }
+ };
let mut native_module = unsafe { mem::zeroed() };
if unsafe { hipModuleLoadData(&mut native_module, image.as_ptr().cast()) }.is_err() {
eprintln!("[d4r-native] failed to load {}", path.display());
return None;
}
+ // Keep the image and module owned by the CUfunction; release at module unload.
+ let owner = NativeModule { module: native_module, _image: image };
let mut func = unsafe { mem::zeroed() };
if unsafe { hipModuleGetFunction(&mut func, native_module, name.as_ptr()) }.is_err() {
eprintln!("[d4r-native] {} has no kernel {:?}", path.display(), name);
@@ -138,7 +173,7 @@ fn native_override(
prep_key,
prep_slots
);
- Some((func, prep, block_z, grid_x, prep_key, prep_slots))
+ Some(((func, prep, block_z, grid_x, prep_key, prep_slots), owner))
}
impl ZludaObject for Module {
@@ -229,7 +264,7 @@ pub(crate) struct Global32Bit {
}
impl Module {
- pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>) -> Self {
+ pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>, ptx_hash: Option<u64>, native_base_image: Option<Vec<u8>>) -> Self {
Self {
base,
sm_version,
@@ -237,6 +272,8 @@ impl Module {
functions: FxHashMap::default(),
}),
bit32,
+ ptx_hash,
+ _native_base_image: native_base_image,
}
}
@@ -248,7 +285,7 @@ impl Module {
.map(Vec::clone)
});
mutable
- .get_function(self.base, self.sm_version, name.to_owned(), explicit_args)
+ .get_function(self.base, self.sm_version, name.to_owned(), explicit_args, self.ptx_hash)
.map(|f| unsafe { (f as *const Function).as_ref().unwrap() })
}
}
@@ -256,7 +293,7 @@ impl Module {
fn get_best_ptx_and_compile(
global_state: &GlobalState,
image: CodeLibraryRef<'_>,
-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
+) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
let mut ptx_modules = Vec::new();
unsafe {
CodeLibraryRef::iterate_modules(image, |_, module| match module {
@@ -280,7 +317,7 @@ fn get_best_ptx_and_compile(
let dump_dir = std::path::PathBuf::from(dump_dir);
let _ = fs::create_dir_all(&dump_dir);
for (index, source) in ptx_modules.iter().enumerate() {
- let stem = format!("module-{index:04}");
+ let stem = format!("module-{:016x}-{index:04}", ptx_identity(source));
let _ = fs::write(dump_dir.join(format!("{stem}.ptx")), source.as_bytes());
let diagnostic = match ptx_parser::parse_module_checked(source.as_ref()) {
Ok(ast) => format!(
@@ -345,6 +382,32 @@ fn get_best_ptx_and_compile(
)
}
};
+ let ptx_hash = text.map(|source| ptx_identity(source));
+ // A fully replaced, single-entry module needs no translated binary. LDS
+ // declarations belong to the replaced kernel; exported globals do not.
+ let only_kernel_and_lds = module.directives.iter().all(|directive| match directive {
+ ptx_parser::Directive::Method(_, method) => method.func_directive.name.is_kernel(),
+ ptx_parser::Directive::Variable(_, variable) => matches!(variable.info.state_space, ptx_parser::StateSpace::Shared),
+ });
+ let entries: Vec<_> = module.directives.iter().filter_map(|directive| match directive {
+ ptx_parser::Directive::Method(_, method) => Some(method), _ => None,
+ }).collect();
+ if module.address_size == 64 && only_kernel_and_lds && entries.len() == 1 {
+ let method = entries[0];
+ {
+ if method.func_directive.name.is_kernel() {
+ if let Ok(name) = CString::new(method.func_directive.name.text()) {
+ if let Some((_, mut owner)) = native_override(&name, ptx_hash) {
+ let base = owner.module;
+ owner.module = hipModule_t(std::ptr::null_mut());
+ let image = mem::take(&mut owner._image);
+ eprintln!("[d4r-native] module_direct kernel={name:?} translated_jit=skipped");
+ return Ok((base, module.sm_version, None, ptx_hash, Some(image)));
+ }
+ }
+ }
+ }
+ }
// TODO: get this information on initialization
let hip_properties = get_hip_properties()?;
let gcn_arch = get_gcn_arch(&hip_properties)?;
@@ -372,7 +435,12 @@ fn get_best_ptx_and_compile(
.or_else(|_| compile_and_cache(gcn_arch, attributes, module, &mut cache_with_key))?;
let mut hip_module = unsafe { mem::zeroed() };
unsafe { hipModuleLoadData(&mut hip_module, elf_module.as_ptr().cast()) }?;
- Ok((hip_module, sm_version, zluda32))
+ Ok((hip_module, sm_version, zluda32, ptx_hash, None))
+}
+
+fn ptx_identity(text: &str) -> u64 {
+ text.trim_end_matches('\0').bytes().fold(0xcbf29ce484222325u64, |hash, byte|
+ (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3))
}
fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
@@ -384,7 +452,7 @@ fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
pub(crate) fn load_hip_module(
library: CodeLibraryRef,
-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
+) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
let global_state = driver::global_state()?;
get_best_ptx_and_compile(global_state, library)
}
@@ -434,7 +502,7 @@ fn zluda_build_version() -> &'static str {
key.starts_with("D4R_ZLUDA_")
&& !matches!(
key.as_str(),
- "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
+ "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
@@ -469,7 +537,10 @@ fn loaded_library_fingerprint() -> Option<String> {
#[cfg(windows)]
fn loaded_library_fingerprint() -> Option<String> {
- None
+ let path = zluda_windows::get_module_path_for_function(zluda_build_version as *const () as usize)?;
+ let metadata = fs::metadata(path).ok()?;
+ let timestamp = metadata.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?;
+ Some(format!("{:x}.{:x}.{:x}", metadata.len(), timestamp.as_secs(), timestamp.subsec_nanos()))
}
fn get_cache_key<'a, 'b>(
@@ -577,16 +648,16 @@ pub(crate) fn load(module: &mut CUmodule, fname: &CStr) -> CUresult {
image.push(0);
let library = unsafe { CodeLibraryRef::try_load(image.as_ptr() as *const std::ffi::c_void) }
.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
- let (hip_module, sm_version, meta32) = load_hip_module(library)?;
- *module = Module::new(hip_module, sm_version, meta32).wrap();
+ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
+ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
Ok(())
}
pub(crate) fn load_data(module: &mut CUmodule, image: &std::ffi::c_void) -> CUresult {
let library =
unsafe { CodeLibraryRef::try_load(image) }.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
- let (hip_module, sm_version, meta32) = load_hip_module(library)?;
- *module = Module::new(hip_module, sm_version, meta32).wrap();
+ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
+ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
Ok(())
}
patches/zluda/0011-audited-k-m-windows-replay.patch
+93
View File
@@ -0,0 +1,93 @@
diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
index 875b2e3..d491ae2 100644
--- a/zluda/src/impl/d4r_replay.rs
+++ b/zluda/src/impl/d4r_replay.rs
@@ -7,6 +7,18 @@ struct Config { directory: PathBuf, filter: String, limit: u32 }
static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
+fn parameter_size(name: &str) -> Option<usize> {
+ match name {
+ "dltss_pwin_enc0_layer" | "dltss_pwin_enc1_layer" | "dltss_pwin_enc2_layer" |
+ "dltss_pwin_enc3_layer" | "dltss_pwin_enc4_layer" | "dltss_pwin_dec5_layer" |
+ "dltss_pwin_dec4_layer" | "dltss_pwin_dec3_layer" | "dltss_pwin_dec2_layer" |
+ "dltss_pwin_dec1_layer" | "dltss_pwin_dec0_layer" => Some(176),
+ "rrlite_enc1_4x4" | "rrlite_enc2_4x4" | "rrlite_dec2_4x4" | "rrlite_dec1_4x4" => Some(56),
+ "rrlite_enc3_tube_4x4" => Some(88),
+ _ => None,
+ }
+}
+
pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
@@ -43,8 +55,10 @@ pub(crate) unsafe fn begin(
limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
})) else { return Ok(None) };
let name = name.to_str().map_err(io_error)?;
- // Only the audited, single 176-byte K argument ABI is captured here.
- if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
+ // Only audited single-by-value K/M argument blocks. Never infer the size
+ // from a prefix: reading a different kernel's parameters can overrun host memory.
+ let Some(parameter_bytes) = parameter_size(name) else { return Ok(None); };
+ if !name.contains(&config.filter) {
return Ok(None);
}
let mut block = std::ptr::null::<u8>();
@@ -60,10 +74,10 @@ pub(crate) unsafe fn begin(
}
} else if !parameters.is_null() {
block = (*parameters).cast();
- bytes = Some(176);
+ bytes = Some(parameter_bytes);
}
- if block.is_null() || bytes != Some(176) {
- return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
+ if block.is_null() || bytes != Some(parameter_bytes) {
+ return Err(io_error(format!("unexpected parameter ABI for {name}: {bytes:?}, expected {parameter_bytes}")));
}
let sequence = {
let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
@@ -73,14 +87,14 @@ pub(crate) unsafe fn begin(
*count
};
hipStreamSynchronize(stream)?;
- let arguments = std::slice::from_raw_parts(block, 176);
+ let arguments = std::slice::from_raw_parts(block, parameter_bytes);
let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
fs::create_dir_all(&directory).map_err(io_error)?;
fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
- let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
+ let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs {parameter_bytes}\n",
launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
let mut allocations = Vec::<(usize, usize)>::new();
- for offset in (0..176).step_by(8) {
+ for offset in (0..parameter_bytes).step_by(8) {
let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
if pointer == 0 { continue; }
let mut base = hipDeviceptr_t(std::ptr::null_mut());
@@ -100,10 +114,23 @@ pub(crate) unsafe fn begin(
};
manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
}
- if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
+ if allocations.is_empty() { return Err(io_error("no allocation pointers resolved")); }
fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
let capture = Capture { directory, allocations };
capture.save(&capture.directory)?;
eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
Ok(Some(capture))
}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn only_audited_argument_sizes() {
+ assert_eq!(super::parameter_size("dltss_pwin_enc0_layer"), Some(176));
+ assert_eq!(super::parameter_size("rrlite_dec2_4x4"), Some(56));
+ assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4"), Some(88));
+ assert_eq!(super::parameter_size("dltss_pwin_enc0_layer_prep"), None);
+ assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4_prep"), None);
+ assert_eq!(super::parameter_size("dltss_pwin_unknown_layer"), None);
+ }
+}
patches/zluda/0012-configurable-windows-jit-cache.patch
+65
View File
@@ -0,0 +1,65 @@
diff --git a/zluda_cache/src/lib.rs b/zluda_cache/src/lib.rs
index acd262e..c3ac1be 100644
--- a/zluda_cache/src/lib.rs
+++ b/zluda_cache/src/lib.rs
@@ -22,18 +22,36 @@ pub struct ModuleCache(SqliteConnection);
impl ModuleCache {
pub fn create_cache_dir_and_get_path() -> Option<String> {
- let mut cache_dir = dirs::cache_dir()?;
- cache_dir.extend(["zluda", "ComputeCache"]);
+ let override_dir = std::env::var_os("ZLUDA_CACHE_DIR");
+ let cache_dir = match override_dir {
+ Some(path) => {
+ let path = std::path::PathBuf::from(path);
+ if !path.is_absolute() {
+ eprintln!("[zluda-cache] ZLUDA_CACHE_DIR must be absolute; cache disabled");
+ return None;
+ }
+ path
+ }
+ None => dirs::cache_dir()?.join("zluda").join("ComputeCache"),
+ };
+ Self::cache_path(&cache_dir)
+ }
+
+ fn cache_path(cache_dir: &std::path::Path) -> Option<String> {
// We ensure that the cache directory exists
- std::fs::create_dir_all(&cache_dir).ok()?;
+ if let Err(error) = std::fs::create_dir_all(cache_dir) {
+ eprintln!("[zluda-cache] cannot create {}: {error}; cache disabled", cache_dir.display());
+ return None;
+ }
// No need to create the file, it will be created by SQLite on first access
// zluda.db might be in use by older versions of ZLUDA
- cache_dir.push("zluda2.db");
- Some(cache_dir.to_string_lossy().into())
+ Some(cache_dir.join("zluda2.db").to_string_lossy().into())
}
pub fn open(file_path: &str) -> Option<Self> {
- let mut conn = SqliteConnection::establish(file_path).ok()?;
+ let mut conn = SqliteConnection::establish(file_path).map_err(|error| {
+ eprintln!("[zluda-cache] cannot open {file_path}: {error}; cache disabled");
+ }).ok()?;
conn.batch_execute("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")
.ok()?;
conn.run_pending_migrations(MIGRATIONS).ok()?;
@@ -86,6 +104,17 @@ mod tests {
use arrayvec::ArrayString;
use diesel::prelude::*;
+ #[test]
+ fn explicit_directory_creates_persistent_database() {
+ let directory = std::env::temp_dir().join(format!("zluda-cache-test-{}-{}", std::process::id(), ModuleCache::time_now()));
+ let path = ModuleCache::cache_path(&directory).unwrap();
+ assert_eq!(std::path::Path::new(&path), directory.join("zluda2.db"));
+ let cache = ModuleCache::open(&path).unwrap();
+ drop(cache);
+ assert!(std::path::Path::new(&path).is_file());
+ std::fs::remove_dir_all(directory).unwrap();
+ }
+
#[derive(Queryable, Selectable)]
#[diesel(table_name = crate::schema::modules)]
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
patches/zluda/0013-gfx12-f16-reference-accumulation.patch
+151
View File
@@ -0,0 +1,151 @@
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 5a54417..90c0034 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
if family == 12 {
flags |= 2;
+ // Explicit diagnostic/correctness mode; retain upstream's fast default.
+ if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") {
+ flags |= 8;
+ }
// native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
if flags & 1 != 0 && wmma_fp8_native_requested() {
flags |= 4;
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 07238db..d262fc6 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M)
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
+// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs.
// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
+ const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16");
Helpers H;
H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
- H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
+ H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2);
H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
- H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
{
Type *F = Type::getFloatTy(Context);
Type *V4F = FixedVectorType::get(F, 4);
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index 4ff25dd..36bd6b1 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8
#endif
}
+// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may
+// reach the other f16 neighbour around cancellation-heavy rounding midpoints.
+// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and
+// round directly to f16 once, matching d4r's independent NumPy/PTX reference.
+// The e4m3 and f32-shadow paths continue using native WMMA.
+__device__ static double wmma12_half_value(uint32_t bits)
+{
+ const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10;
+ const double value = exponent == 0 ? double(magnitude) * 0x1p-24 :
+ double(std::bit_cast<float>(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) :
+ ((magnitude << 13) + 0x38000000u)));
+ return bits & 0x8000u ? -value : value;
+}
+
+// Preserve f16 subnormal result bits even when the target's default f16
+// arithmetic mode flushes them. No intermediate f32 rounding is involved.
+__device__ static uint32_t wmma12_half_round(double value)
+{
+ const uint64_t bits = std::bit_cast<uint64_t>(value);
+ const uint32_t sign = uint32_t(bits >> 48) & 0x8000u;
+ const uint32_t exponent = uint32_t((bits >> 52) & 2047u);
+ const uint64_t fraction = bits & 0xfffffffffffffu;
+ if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u);
+ const int power = int(exponent) - 1023;
+ if (power < -25) return sign;
+ if (power > 15) return sign | 0x7c00u;
+ const unsigned shift = power < -14 ? unsigned(28 - power) : 42u;
+ const uint64_t significand = fraction | 0x10000000000000ull;
+ uint32_t code = uint32_t(significand >> shift);
+ const uint64_t tail = significand & ((uint64_t(1) << shift) - 1);
+ const uint64_t midpoint = uint64_t(1) << (shift - 1);
+ code += tail > midpoint || (tail == midpoint && (code & 1u));
+ if (power >= -14) code += uint32_t(power + 14) << 10;
+ return sign | code;
+}
+
+__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired)
+{
+ const bool high = wmma12_half() != 0;
+ const uint2::Native_vec_ own = paired && high ? c1 : c0;
+ const uint2::Native_vec_ send = paired && !high ? c1 : c0;
+ const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
+ const uint2::Native_vec_ even = paired && high ? other : own;
+ const uint2::Native_vec_ odd = paired && high ? own : other;
+ return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u),
+ (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)};
+}
+
+__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c)
+{
+ // x/y: packed even rows, z/w: packed odd rows, in this half-wave.
+ wmma_uint4 result = {};
+ const uint32_t half = wmma12_half();
+ for (uint32_t i = 0; i < 8; ++i)
+ {
+ const uint32_t row = i + 8 * half;
+ double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu);
+ for (uint32_t khalf = 0; khalf < 2; ++khalf)
+ for (uint32_t j = 0; j < 4; ++j)
+ {
+ const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]);
+ const uint32_t other = wmma12_other_half(b[j]);
+ const uint32_t bv = half == khalf ? b[j] : other;
+ sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu);
+ sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16);
+ }
+ // Do not round through f32: that can reintroduce the midpoint error.
+ result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1));
+ }
+ return result;
+}
+
// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
@@ -1959,6 +2031,22 @@ extern "C"
return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
}
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false));
+ const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)};
+ return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y};
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true));
+ const bool high = wmma12_half() != 0;
+ const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z);
+ const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w);
+ return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1};
+ }
+
wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
{
return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
patches/zluda/0014-windows-kernel-profiling.patch
+148
View File
@@ -0,0 +1,148 @@
From b0161a4166652eee86a8113a38d042b02403db6e Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 18:07:07 +0300
Subject: [PATCH] Add opt-in Windows HIP event and occupancy diagnostics
---
zluda/src/impl/d4r_profile.rs | 60 +++++++++++++++++++++++++++++++++++
zluda/src/impl/function.rs | 9 ++++++
zluda/src/impl/mod.rs | 1 +
zluda/src/impl/module.rs | 2 +-
4 files changed, 71 insertions(+), 1 deletion(-)
create mode 100644 zluda/src/impl/d4r_profile.rs
diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
new file mode 100644
index 0000000..33ba5f6
--- /dev/null
+++ b/zluda/src/impl/d4r_profile.rs
@@ -0,0 +1,60 @@
+//! Optional, deliberately serializing HIP-event diagnostics. Not a production
+//! scheduler or a claim about application FPS / measured hardware occupancy.
+use hip_runtime_sys::*;
+use std::{ffi::CStr, ptr, time::Instant};
+
+pub(super) struct Launch {
+ start: hipEvent_t,
+ end: hipEvent_t,
+ host: Instant,
+}
+
+impl Drop for Launch {
+ fn drop(&mut self) {
+ unsafe {
+ if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
+ if !self.start.is_null() { let _ = hipEventDestroy(self.start); }
+ }
+ }
+}
+
+impl Launch {
+ pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
+ if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
+ let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
+ hipEventCreateWithFlags(&mut result.start, 0)?;
+ hipEventCreateWithFlags(&mut result.end, 0)?;
+ hipEventRecord(result.start, stream)?;
+ result.host = Instant::now();
+ Ok(Some(result))
+ }
+
+ pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
+ phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
+ let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
+ hipEventRecord(self.end, stream)?;
+ hipEventSynchronize(self.end)?;
+ let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
+ let mut gpu_ms = 0.;
+ hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
+ // Module function handles use the Driver API attribute query. The
+ // plural Runtime API expects a registered host function address.
+ let mut lds = 0;
+ let mut regs = 0;
+ let mut private = 0;
+ let mut max_threads = 0;
+ hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
+ hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
+ let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
+ .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
+ let mut blocks_per_multiprocessor = 0;
+ hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
+ function, threads as i32, shared as usize)?;
+ eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
+ grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
+ regs, private, max_threads);
+ Ok(())
+ }
+}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index 7a7470a..c456dfe 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -191,6 +191,7 @@ pub(crate) fn launch_kernel(
}
};
if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
+ let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
unsafe {
hipModuleLaunchKernel(
prep,
@@ -206,6 +207,9 @@ pub(crate) fn launch_kernel(
extra_for_hip,
)
}?;
+ if let Some(profile) = profile {
+ unsafe { profile.finish(prep, &f.name, "native", "prep", [blocks,1,1], [128,1,1], 0, stream) }?;
+ }
f.prep_last.store(prep_key, std::sync::atomic::Ordering::Relaxed);
}
let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
@@ -214,6 +218,7 @@ pub(crate) fn launch_kernel(
eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
f.name, if f.native_owner.is_some() { "native" } else { "translated" });
}
+ let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
let result = unsafe {
hipModuleLaunchKernel(
f.base,
@@ -230,6 +235,10 @@ pub(crate) fn launch_kernel(
)
};
result?;
+ if let Some(profile) = profile {
+ unsafe { profile.finish(f.base, &f.name, if f.native_owner.is_some() { "native" } else { "translated" },
+ "main", [grid_dim_x, grid_dim_y, grid_dim_z], [block_dim_x, block_dim_y, block_dim_z], shared_mem_bytes, stream) }?;
+ }
if let Some(replay) = replay { replay.finish(stream)?; }
Ok(())
}
diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
index 14e0aee..380eb6e 100644
--- a/zluda/src/impl/mod.rs
+++ b/zluda/src/impl/mod.rs
@@ -6,6 +6,7 @@ pub(super) mod context;
pub(super) mod device;
pub(super) mod driver;
pub(super) mod d4r_replay;
+pub(super) mod d4r_profile;
pub(super) mod event;
pub(super) mod function;
pub(super) mod graph;
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 5a79368..232f7d4 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -502,7 +502,7 @@ fn zluda_build_version() -> &'static str {
key.starts_with("D4R_ZLUDA_")
&& !matches!(
key.as_str(),
- "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
+ "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
--
2.55.0.windows.5
patches/zluda/0015-nonserializing-cuda-api-profile.patch
+83
View File
@@ -0,0 +1,83 @@
From a1c506fc956cfc347c56c44311669700c0d500ac Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 22:16:19 +0300
Subject: [PATCH] Profile CUDA host API duration without serializing GPU
launches
---
zluda/src/d4r_api_profile.rs | 34 ++++++++++++++++++++++++++++++++++
zluda/src/lib.rs | 3 +++
2 files changed, 37 insertions(+)
create mode 100644 zluda/src/d4r_api_profile.rs
diff --git a/zluda/src/d4r_api_profile.rs b/zluda/src/d4r_api_profile.rs
new file mode 100644
index 0000000..ef2da1b
--- /dev/null
+++ b/zluda/src/d4r_api_profile.rs
@@ -0,0 +1,34 @@
+//! Host API timings only: no HIP events, stream waits, or altered scheduling.
+use std::{cell::RefCell, collections::BTreeMap, sync::OnceLock, time::Instant};
+
+#[derive(Default)]
+struct Row { calls: u64, total_ms: f64, max_ms: f64 }
+struct Samples { since: Instant, rows: BTreeMap<&'static str, Row> }
+thread_local! {
+ static SAMPLES: RefCell<Samples> = RefCell::new(Samples { since: Instant::now(), rows: BTreeMap::new() });
+}
+pub(crate) struct Call { name: &'static str, start: Instant }
+impl Call {
+ pub(crate) fn begin(name: &'static str) -> Option<Self> {
+ static ENABLED: OnceLock<bool> = OnceLock::new();
+ if !*ENABLED.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_API").is_some()) { return None; }
+ Some(Self { name, start: Instant::now() })
+ }
+}
+impl Drop for Call {
+ fn drop(&mut self) {
+ let ms = self.start.elapsed().as_secs_f64() * 1000.;
+ SAMPLES.with(|samples| {
+ let mut samples = samples.borrow_mut();
+ let row = samples.rows.entry(self.name).or_default();
+ row.calls += 1; row.total_ms += ms; row.max_ms = row.max_ms.max(ms);
+ let period_ms = samples.since.elapsed().as_secs_f64() * 1000.;
+ if period_ms < 2000. { return; }
+ for (name, row) in &samples.rows {
+ eprintln!("D4R_CUDA_API_PROFILE thread={:?} api={name} calls={} total_ms={:.6} max_ms={:.6} period_ms={period_ms:.6} serializing=0",
+ std::thread::current().id(), row.calls, row.total_ms, row.max_ms);
+ }
+ samples.rows.clear(); samples.since = Instant::now();
+ });
+ }
+}
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
index ecdeb3e..31bfa42 100644
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -2,6 +2,7 @@ use cuda_types::cuda::CUerror;
use std::sync::atomic::{AtomicBool, Ordering};
pub(crate) mod r#impl;
+mod d4r_api_profile;
#[cfg_attr(windows, path = "os_win.rs")]
#[cfg_attr(not(windows), path = "os_unix.rs")]
mod os;
@@ -40,6 +41,7 @@ macro_rules! implemented {
if !initialized() {
return Err(CUerror::DEINITIALIZED);
}
+ let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
cuda_macros::cuda_normalize_fn!( crate::r#impl::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
Ok(())
}
@@ -57,6 +59,7 @@ macro_rules! implemented_in_function {
if !initialized() {
return Err(CUerror::DEINITIALIZED);
}
+ let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
cuda_macros::cuda_normalize_fn!( crate::r#impl::function::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
Ok(())
}
--
2.55.0.windows.5
patches/zluda/0016-deferred-kernel-profile.patch
+282
View File
@@ -0,0 +1,282 @@
From 26a43f7047ab933a4afb5ebdd26d5ba9e51e58a3 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 11:50:45 +0300
Subject: [PATCH 16/17] Sample HIP kernel events without per-launch completion
waits
---
zluda/src/impl/context.rs | 4 +-
zluda/src/impl/d4r_profile.rs | 204 ++++++++++++++++++++++++++++------
zluda/src/impl/module.rs | 1 +
3 files changed, 171 insertions(+), 38 deletions(-)
diff --git a/zluda/src/impl/context.rs b/zluda/src/impl/context.rs
index 9ccb372..374d383 100644
--- a/zluda/src/impl/context.rs
+++ b/zluda/src/impl/context.rs
@@ -165,7 +165,7 @@ pub(crate) fn set_cache_config(config: CUfunc_cache) -> CUresult {
}
pub(crate) fn synchronize() -> hipError_t {
- unsafe { hipDeviceSynchronize() }
+ unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
}
pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
@@ -177,7 +177,7 @@ pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
} else {
None
};
- unsafe { hipDeviceSynchronize() }
+ unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
}
struct SetDeviceOnDrop(i32);
diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
index 33ba5f6..1b9d248 100644
--- a/zluda/src/impl/d4r_profile.rs
+++ b/zluda/src/impl/d4r_profile.rs
@@ -1,15 +1,29 @@
-//! Optional, deliberately serializing HIP-event diagnostics. Not a production
-//! scheduler or a claim about application FPS / measured hardware occupancy.
+//! Optional HIP-event diagnostics. Deferred mode never waits for its events,
+//! skips the legacy default stream and bounds outstanding events per thread.
+//! Timestamp insertion still adds overhead; this is not a production scheduler.
use hip_runtime_sys::*;
-use std::{ffi::CStr, ptr, time::Instant};
+use std::{cell::RefCell, ffi::{CStr, CString}, ptr, sync::OnceLock, time::Instant};
-pub(super) struct Launch {
- start: hipEvent_t,
- end: hipEvent_t,
- host: Instant,
+#[derive(Clone, Copy)]
+enum Mode { Off, Serial, Deferred(u64) }
+fn mode() -> Mode {
+ static MODE: OnceLock<Mode> = OnceLock::new();
+ *MODE.get_or_init(|| {
+ if std::env::var_os("D4R_ZLUDA_PROFILE_DEFERRED").is_some() {
+ let every = std::env::var("D4R_ZLUDA_PROFILE_EVERY").ok()
+ .and_then(|v| v.parse::<u64>().ok()).filter(|n| *n > 0).unwrap_or(17);
+ Mode::Deferred(every)
+ } else if std::env::var_os("D4R_ZLUDA_PROFILE").is_some() { Mode::Serial }
+ else { Mode::Off }
+ })
+}
+fn allow_legacy() -> bool {
+ static ALLOW: OnceLock<bool> = OnceLock::new();
+ *ALLOW.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_ALLOW_LEGACY").is_some())
}
-impl Drop for Launch {
+struct Events { start: hipEvent_t, end: hipEvent_t }
+impl Drop for Events {
fn drop(&mut self) {
unsafe {
if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
@@ -17,44 +31,162 @@ impl Drop for Launch {
}
}
}
+impl Events {
+ unsafe fn new() -> Result<Self, hipErrorCode_t> {
+ let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut() };
+ hipEventCreateWithFlags(&mut result.start, 0)?;
+ hipEventCreateWithFlags(&mut result.end, 0)?;
+ Ok(result)
+ }
+}
+
+struct Description {
+ name: CString, backend: String, phase: String, grid: [u32;3], block: [u32;3],
+ shared: u32, lds: i32, regs: i32, private: i32, max_threads: i32, blocks_per_multiprocessor: i32,
+}
+impl Description {
+ unsafe fn new(function: hipFunction_t, name: &CStr, backend: &str, phase: &str,
+ grid: [u32;3], block: [u32;3], shared: u32) -> Result<Self, hipErrorCode_t> {
+ let mut d = Self { name: name.to_owned(), backend: backend.to_owned(), phase: phase.to_owned(),
+ grid, block, shared, lds: 0, regs: 0, private: 0, max_threads: 0, blocks_per_multiprocessor: 0 };
+ // Copy metadata while the function/module is alive. Collection never
+ // dereferences a function handle that the application may have unloaded.
+ hipFuncGetAttribute(&mut d.lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut d.regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
+ hipFuncGetAttribute(&mut d.private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut d.max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
+ let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
+ .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
+ hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut d.blocks_per_multiprocessor,
+ function, threads as i32, shared as usize)?;
+ Ok(d)
+ }
+ fn print(&self, gpu_ms: f32, enqueue_ms: f64, completion_ms: f64, serializing: bool, stream: usize) {
+ if !gpu_ms.is_finite() || gpu_ms < 0. {
+ eprintln!("D4R_KERNEL_PROFILE_INVALID kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} reason=invalid_elapsed_time serializing={} deferred={} stream=0x{stream:x}",
+ self.name, self.backend, self.phase, u32::from(serializing), u32::from(!serializing));
+ return;
+ }
+ eprintln!("D4R_KERNEL_PROFILE kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={} serializing={} deferred={} stream=0x{stream:x} legacy_stream={}",
+ self.name, self.backend, self.phase, self.grid[0], self.grid[1], self.grid[2],
+ self.block[0], self.block[1], self.block[2], self.lds, self.shared, self.regs,
+ self.private, self.max_threads, self.blocks_per_multiprocessor,
+ u32::from(serializing), u32::from(!serializing), u32::from(stream <= 1));
+ }
+}
+
+struct Pending { launch: Launch, description: Description, enqueue_ms: f64, stream: usize }
+const CAPACITY: usize = 64;
+struct Deferred {
+ calls: u64, skipped: u64, pending: Vec<Pending>, free: Vec<(i32, Events)>,
+}
+thread_local! {
+ static DEFERRED: RefCell<Deferred> = RefCell::new(Deferred {
+ calls: 0, skipped: 0, pending: Vec::new(), free: Vec::new(),
+ });
+}
+
+impl Deferred {
+ unsafe fn collect(&mut self, device: i32) -> hipError_t {
+ let mut index = 0;
+ while index < self.pending.len() {
+ let p = &self.pending[index];
+ if p.launch.device != device { index += 1; continue; }
+ match hipEventQuery(p.launch.events.end) {
+ Err(e) if e == hipErrorCode_t::NotReady => { index += 1; continue; }
+ result => result?,
+ }
+ let mut gpu_ms = 0.;
+ hipEventElapsedTime(&mut gpu_ms, p.launch.events.start, p.launch.events.end)?;
+ p.description.print(gpu_ms, p.enqueue_ms, p.launch.host.elapsed().as_secs_f64() * 1000., false, p.stream);
+ let p = self.pending.swap_remove(index);
+ // Re-record only after the old timestamps have been collected.
+ self.free.push((device, p.launch.events));
+ }
+ Ok(())
+ }
+}
+
+/// Called only after the application's existing context synchronization, or
+/// before a selected launch. No event/stream/device wait is added here.
+pub(super) unsafe fn collect_completed() -> hipError_t {
+ if !matches!(mode(), Mode::Deferred(_)) { return Ok(()); }
+ DEFERRED.with(|state| {
+ let mut state = state.borrow_mut();
+ if state.pending.is_empty() { return Ok(()); }
+ let mut device = 0; hipGetDevice(&mut device)?;
+ state.collect(device)
+ })
+}
+
+pub(super) struct Launch {
+ events: Events,
+ host: Instant,
+ device: i32,
+ deferred: bool,
+}
impl Launch {
pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
- if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
- let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
- hipEventCreateWithFlags(&mut result.start, 0)?;
- hipEventCreateWithFlags(&mut result.end, 0)?;
- hipEventRecord(result.start, stream)?;
- result.host = Instant::now();
- Ok(Some(result))
+ let mode = mode();
+ if matches!(mode, Mode::Off) { return Ok(None); }
+ if let Mode::Deferred(every) = mode {
+ return DEFERRED.with(|state| {
+ let mut state = state.borrow_mut();
+ state.calls = state.calls.wrapping_add(1);
+ if (state.calls - 1) % every != 0 { return Ok(None); }
+ // HIP timing events on the legacy default stream can order
+ // other streams. Do not introduce that dependency in this mode.
+ if stream.0 as usize <= 1 && !allow_legacy() {
+ state.skipped += 1;
+ if state.skipped.is_power_of_two() {
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=legacy_default_stream count={} serializing=0", state.skipped);
+ }
+ return Ok(None);
+ }
+ let mut device = 0; hipGetDevice(&mut device)?;
+ state.collect(device)?;
+ let mut capture = hipStreamCaptureStatus::hipStreamCaptureStatusNone;
+ hipStreamIsCapturing(stream, &mut capture)?;
+ if capture != hipStreamCaptureStatus::hipStreamCaptureStatusNone {
+ state.skipped += 1;
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=stream_capture count={} serializing=0", state.skipped);
+ return Ok(None);
+ }
+ let events = if let Some(index) = state.free.iter().position(|(d, _)| *d == device) {
+ state.free.swap_remove(index).1
+ } else if state.pending.len() + state.free.len() < CAPACITY {
+ Events::new()?
+ } else {
+ state.skipped += 1;
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=capacity count={} capacity={CAPACITY} serializing=0", state.skipped);
+ return Ok(None);
+ };
+ hipEventRecord(events.start, stream)?;
+ Ok(Some(Self { events, host: Instant::now(), device, deferred: true }))
+ });
+ }
+ let events = Events::new()?;
+ hipEventRecord(events.start, stream)?;
+ Ok(Some(Self { events, host: Instant::now(), device: -1, deferred: false }))
}
pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
- hipEventRecord(self.end, stream)?;
- hipEventSynchronize(self.end)?;
+ hipEventRecord(self.events.end, stream)?;
+ let description = Description::new(function, name, backend, phase, grid, block, shared)?;
+ if self.deferred {
+ DEFERRED.with(|state| state.borrow_mut().pending.push(Pending {
+ launch: self, description, enqueue_ms, stream: stream.0 as usize,
+ }));
+ return Ok(());
+ }
+ hipEventSynchronize(self.events.end)?;
let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
let mut gpu_ms = 0.;
- hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
- // Module function handles use the Driver API attribute query. The
- // plural Runtime API expects a registered host function address.
- let mut lds = 0;
- let mut regs = 0;
- let mut private = 0;
- let mut max_threads = 0;
- hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
- hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
- hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
- hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
- let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
- .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
- let mut blocks_per_multiprocessor = 0;
- hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
- function, threads as i32, shared as usize)?;
- eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
- grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
- regs, private, max_threads);
+ hipEventElapsedTime(&mut gpu_ms, self.events.start, self.events.end)?;
+ description.print(gpu_ms, enqueue_ms, completion_ms, true, stream.0 as usize);
Ok(())
}
}
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 232f7d4..8d43abf 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -503,6 +503,7 @@ fn zluda_build_version() -> &'static str {
&& !matches!(
key.as_str(),
"D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
+ | "D4R_ZLUDA_PROFILE_API" | "D4R_ZLUDA_PROFILE_DEFERRED" | "D4R_ZLUDA_PROFILE_EVERY" | "D4R_ZLUDA_PROFILE_ALLOW_LEGACY"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
--
2.55.0.windows.5
patches/zluda/0017-asynchronous-2d-array-copies.patch
+50
View File
@@ -0,0 +1,50 @@
From 67127dde599879ada022d6439383d682e09e0b36 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 11:50:45 +0300
Subject: [PATCH 17/17] Implement asynchronous 2D array copies for Windows
input batching
---
zluda/src/impl/memory.rs | 13 +++++++++++++
zluda/src/lib.rs | 1 +
2 files changed, 14 insertions(+)
diff --git a/zluda/src/impl/memory.rs b/zluda/src/impl/memory.rs
index b724942..42facb3 100644
--- a/zluda/src/impl/memory.rs
+++ b/zluda/src/impl/memory.rs
@@ -469,6 +469,19 @@ pub(crate) unsafe fn copy_2d_v2(memcpy: hip_Memcpy2D) -> CUresult {
Ok(())
}
+pub(crate) unsafe fn copy_2d_async_v2(memcpy: hip_Memcpy2D, stream: hipStream_t) -> CUresult {
+ // The format-80 emulation in copy_2d_v2 uses temporary CPU storage.
+ // Reject it explicitly here: an asynchronous copy cannot outlive those
+ // temporaries, and the Windows fast path uses canonical float arrays.
+ let src_packed = memcpy.srcMemoryType == hipMemoryType::hipMemoryTypeArray
+ && array::original_format(memcpy.srcArray)? == Some(80);
+ let dst_packed = memcpy.dstMemoryType == hipMemoryType::hipMemoryTypeArray
+ && array::original_format(memcpy.dstArray)? == Some(80);
+ if src_packed || dst_packed { return Err(CUerror::NOT_SUPPORTED); }
+ hipMemcpyParam2DAsync(&memcpy, stream)?;
+ Ok(())
+}
+
pub(crate) unsafe fn copy_2d_unaligned_v2(memcpy: hip_Memcpy2D) -> CUresult {
hipDrvMemcpy2DUnaligned(&memcpy)?;
Ok(())
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
index 31bfa42..939981e 100644
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -150,6 +150,7 @@ cuda_macros::cuda_function_declarations!(
cuMemAlloc_v2,
cuMemAllocPitch_v2,
cuMemcpy2D_v2,
+ cuMemcpy2DAsync_v2,
cuMemcpy2DUnaligned_v2,
cuMemcpy3D_v2,
cuMemcpyAsync,
--
2.55.0.windows.5
patches/zluda/0018-opt-in-rdna-wgp-placement.patch
+93
View File
@@ -0,0 +1,93 @@
From fb0adc8714f57647320f0601533ec50331e378b0 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 14:40:56 +0300
Subject: [PATCH] Add opt-in RDNA CU/WGP placement for matched native
experiments
---
llvm_zluda/src/compile.rs | 42 ++++++++++++++++++++++++++++++++++-----
ptx/src/pass/llvm/emit.rs | 3 ++-
2 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 90c0034..6a2f452 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -24,6 +24,37 @@ const OCKL_MODULE: &[u8] = include_bytes!("device-libs/ockl.bc");
// https://llvm.org/docs/AMDGPUUsage.html#address-spaces
const CONSTANT_ADDRESS_SPACE: u32 = 4;
+/// Opt-in RDNA scheduling experiment; changes placement, not FP semantics.
+pub fn wgp_requested() -> bool {
+ std::env::var("D4R_ZLUDA_WGP").is_ok_and(|value| value.trim() == "1")
+}
+
+fn target_features(wave64: bool, wgp: bool) -> &'static CStr {
+ match (wave64, wgp) {
+ (false, false) => c"-wavefrontsize64,+cumode",
+ (true, false) => c"-wavefrontsize32,+wavefrontsize64,+cumode",
+ (false, true) => c"-wavefrontsize64,-cumode",
+ (true, true) => c"-wavefrontsize32,+wavefrontsize64,-cumode",
+ }
+}
+
+#[cfg(test)]
+mod shader_mode_tests {
+ use super::target_features;
+
+ #[test]
+ fn wgp_changes_only_cu_placement_in_both_wave_modes() {
+ for wave64 in [false, true] {
+ let cu = target_features(wave64, false).to_str().unwrap();
+ let wgp = target_features(wave64, true).to_str().unwrap();
+ assert_eq!(cu.replace("+cumode", "-cumode"), wgp);
+ assert_eq!(cu.contains("+wavefrontsize64"), wave64);
+ assert_eq!(wgp.contains("+wavefrontsize64"), wave64);
+ }
+ assert_eq!(target_features(false, false), c"-wavefrontsize64,+cumode");
+ }
+}
+
fn load_module(ctx: &Context, bc: &[u8], name: &std::ffi::CStr) -> Result<Module, String> {
let module = Module::try_from_bitcode(ctx, bc, Some(name))
.ok_or(("Failed to parse bitcode").to_string())?;
@@ -229,11 +260,12 @@ pub fn compile(
let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
// functions without their own target features (the linked helper libraries) take these, so under
// D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
- let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
- c"-wavefrontsize32,+wavefrontsize64,+cumode"
- } else {
- c"-wavefrontsize64,+cumode"
- };
+ let wgp = wgp_requested();
+ if wgp && !gcn_arch.starts_with("gfx11") && !gcn_arch.starts_with("gfx12") {
+ return Err(format!("D4R_ZLUDA_WGP requires RDNA3/4, got {gcn_arch}"));
+ }
+ let features = target_features(
+ std::env::var("D4R_ZLUDA_WAVE64").is_ok_and(|v| v.trim() == "1"), wgp);
let mut target = unsafe { std::mem::zeroed() };
let mut err = std::ptr::null_mut();
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 6af2495..8b67c93 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -397,12 +397,13 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
fn emit_target_features(&mut self, fn_: LLVMValueRef) {
let value = format!(
- "{},+cumode{}",
+ "{},{}{}",
if crate::pass::wave64_requested() {
"-wavefrontsize32,+wavefrontsize64"
} else {
"+wavefrontsize32,-wavefrontsize64"
},
+ if llvm_zluda::compile::wgp_requested() { "-cumode" } else { "+cumode" },
if cfg!(debug_assertions) {
",+precise-memory"
} else {
--
2.55.0.windows.5