wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clear windows branch; preserve previous contents in win-old

Browse Source
This commit is contained in:
ayo committed 2026-10-16 05:20:08 -04:00
1 parent f0c881985b
commit 73994d4634
311 files changed
-57018
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

patches/zluda/0002-dlss-runtime-cumulative.patch
-2768
View File
File diff suppressed because it is too large. Load diff
patches/zluda/0003-perf-implicit-bound-gfx11-wmma.patch
-772
View File
@@ -1,772 +0,0 @@
diff --git a/ptx/src/pass/mod.rs b/ptx/src/pass/mod.rs
index 15dd9ba..cff04dd 100644
--- a/ptx/src/pass/mod.rs
+++ b/ptx/src/pass/mod.rs
@@ -61,6 +61,42 @@ pub struct Attributes {
pub clock_rate: u32,
}
+/// Work-group size bound applied to kernels without `.maxntid`/`.reqntid`
+/// (D4R_ZLUDA_IMPLICIT_MAX_BLOCK, 1..=1024; unset keeps LLVM's 1024 default).
+///
+/// Without a bound, LLVM must assume 1024-thread work-groups, which on gfx11
+/// in CU mode caps each wave at 96 VGPRs and makes register-heavy kernels
+/// (e.g. DLSS's tensor-core convolutions) spill kilobytes to scratch. CUDA's
+/// own compiler gives unbounded kernels up to 255 registers, which likewise
+/// limits them to 256-thread blocks; a launch exceeding the bound fails
+/// (as it would on NVIDIA) instead of misbehaving.
+/// Optional AMDGPU machine-scheduler strategy for kernels
+/// (D4R_ZLUDA_SCHED_STRATEGY: max-ilp, max-memory-clause, iterative-ilp,
+/// iterative-minreg or iterative-maxocc); unset keeps LLVM's default.
+pub fn sched_strategy() -> Option<&'static str> {
+ static VALUE: std::sync::OnceLock<Option<String>> = std::sync::OnceLock::new();
+ VALUE
+ .get_or_init(|| {
+ std::env::var("D4R_ZLUDA_SCHED_STRATEGY").ok().filter(|value| {
+ matches!(
+ value.as_str(),
+ "max-ilp" | "max-memory-clause" | "iterative-ilp" | "iterative-minreg" | "iterative-maxocc"
+ )
+ })
+ })
+ .as_deref()
+}
+
+pub fn implicit_max_block() -> Option<u32> {
+ static VALUE: std::sync::OnceLock<Option<u32>> = std::sync::OnceLock::new();
+ *VALUE.get_or_init(|| {
+ std::env::var("D4R_ZLUDA_IMPLICIT_MAX_BLOCK")
+ .ok()
+ .and_then(|value| value.trim().parse::<u32>().ok())
+ .filter(|value| (1..=1024).contains(value))
+ })
+}
+
pub fn to_llvm_module<'input>(
ast: ast::Module<'input>,
attributes: Attributes,
diff --git a/ptx/src/lib.rs b/ptx/src/lib.rs
index 8daa637..79ed812 100644
--- a/ptx/src/lib.rs
+++ b/ptx/src/lib.rs
@@ -4,4 +4,6 @@ mod test;
pub use pass::to_llvm_module;
pub use pass::Attributes;
+pub use pass::implicit_max_block;
+pub use pass::sched_strategy;
pub use pass::TranslateError;
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 5ab8985..e891c34 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -147,6 +147,23 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
}
self.emit_target_features(fn_);
self.emit_tuning(fn_, &method.tuning);
+ if method.is_kernel()
+ && !method.tuning.iter().any(|tuning| {
+ matches!(
+ tuning,
+ ptx_parser::TuningDirective::MaxNtid(..) | ptx_parser::TuningDirective::ReqNtid(..)
+ )
+ })
+ {
+ if let Some(size) = crate::pass::implicit_max_block() {
+ self.emit_fn_attribute_string(fn_, "amdgpu-flat-work-group-size", &format!("1,{size}"));
+ }
+ }
+ if method.is_kernel() {
+ if let Some(strategy) = crate::pass::sched_strategy() {
+ self.emit_fn_attribute_string(fn_, "amdgpu-sched-strategy", strategy);
+ }
+ }
if let Some(kernel_attrs) = &method.kernel_attributes {
self.emit_fn_attribute_string(
fn_,
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index a6adeef..0fbc7f7 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -260,6 +260,9 @@ fn get_best_ptx_and_compile(
let attributes = ExtraCacheAttributes {
clock_rate: hip_properties.clockRate as u32,
is_debug: cfg!(debug_assertions),
+ implicit_max_block: ptx::implicit_max_block(),
+ sched_strategy: ptx::sched_strategy(),
+ wmma: llvm_zluda::compile::wmma_rewrite_requested(),
};
let mut cache_with_key = match (text, global_state.cache_path.as_ref()) {
(Some(text), Some(p)) => (|| {
@@ -296,6 +299,9 @@ pub(crate) fn load_hip_module(
struct ExtraCacheAttributes {
is_debug: bool,
clock_rate: u32,
+ implicit_max_block: Option<u32>,
+ sched_strategy: Option<&'static str>,
+ wmma: bool,
}
fn get_hip_properties<'a>() -> Result<hipDeviceProp_tR0600, CUerror> {
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index b80262b..1284cbc 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -87,6 +87,18 @@ fn create_oclc_constants(ctx: &Context, gcn_arch: &str) -> Result<Module, String
Ok(module)
}
+/// D4R_ZLUDA_WMMA=1 lowers f16 m16n8k16 MMAs to gfx11 WMMA (see
+/// LLVMZludaRewriteMmaF16Wmma); anything else keeps the dot-product fallback.
+pub fn wmma_rewrite_requested() -> bool {
+ static VALUE: OnceLock<bool> = OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WMMA").map_or(false, |value| value.trim() == "1"))
+}
+
+fn wmma_rewrite_enabled(gcn_arch: &str) -> bool {
+ // The helpers assume RDNA3's wave32 WMMA register layout.
+ wmma_rewrite_requested() && gcn_arch.starts_with("gfx11")
+}
+
pub fn compile(
ctx: &Context,
gcn_arch: &str,
@@ -106,6 +118,9 @@ pub fn compile(
let oclc_constants = create_oclc_constants(ctx, gcn_arch)?;
+ if wmma_rewrite_enabled(gcn_arch) {
+ unsafe { crate::ffi::LLVMZludaRewriteMmaF16Wmma(main.get()) };
+ }
linked.link(main)?;
linked.link(attributes)?;
linked.link(oclc_constants)?;
diff --git a/llvm_zluda/src/ffi.rs b/llvm_zluda/src/ffi.rs
index 7ca4289..6bf0d30 100644
--- a/llvm_zluda/src/ffi.rs
+++ b/llvm_zluda/src/ffi.rs
@@ -42,6 +42,9 @@ pub const LLVMZludaFastMathAll: ::std::ffi::c_uint = LLVMZludaFastMathAllowReass
pub type LLVMZludaFastMathFlags = std::ffi::c_uint;
extern "C" {
+ /// See LLVMZludaRewriteMmaF16Wmma in lib.cpp.
+ pub fn LLVMZludaRewriteMmaF16Wmma(M: LLVMModuleRef) -> u32;
+
pub fn LLVMZludaBuildAlloca(
B: LLVMBuilderRef,
Ty: LLVMTypeRef,
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 14b4f40..30d5490 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -9,6 +9,11 @@
#include <llvm/Support/raw_ostream.h>
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/raw_ostream.h"
+#include <llvm/ADT/DenseMap.h>
+#include <llvm/ADT/DenseSet.h>
+#include <llvm/ADT/SmallVector.h>
+#include <llvm/IR/Module.h>
+#include <llvm/IR/InstIterator.h>
#pragma GCC diagnostic pop
#include <mutex>
@@ -255,6 +260,367 @@ int LLVMZludaLinkWithLLD(uint32_t inputs, const char **input_paths, const char *
return 0;
}
+// gfx11 WMMA lowering of mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16,
+// run on ZLUDA's emitted module before linking, while PTX registers are still
+// allocas. Each MMA becomes
+// d = d4r_wmma_t_to_nv(d4r_wmma_t_mma_f16(a_from_b(B), b_from_a(A), C'))
+// (see zluda_ptx_impl.cpp for the layouts). Register pairs used as MMA
+// accumulators get a shadow alloca holding the pair in WMMA layout: an MMA
+// whose C is a plain load of such a pair reads the shadow, an MMA whose D is
+// stored to it writes the shadow, and any other store to either register
+// refreshes the shadow with d4r_wmma_t_from_nv. The invariant "shadow ==
+// WMMA layout of the pair" thus holds at every program point, and once
+// mem2reg promotes the allocas the layout conversions between chained MMAs
+// become dead code. Returns the number of MMAs rewritten.
+namespace
+{
+constexpr const char *kMmaF16 = "__zluda_ptx_impl_mma_sync_aligned_m16n8k16_row_col_f16_f16_f16_f16";
+
+struct RegisterPair
+{
+ AllocaInst *Low = nullptr;
+ AllocaInst *High = nullptr;
+ bool operator==(const RegisterPair &Other) const { return Low == Other.Low && High == Other.High; }
+};
+
+// Only allocas used purely as the address of loads and stores are tracked.
+static bool isPlainRegister(AllocaInst *Alloca)
+{
+ for (User *U : Alloca->users())
+ {
+ if (isa<LoadInst>(U))
+ continue;
+ auto *Store = dyn_cast<StoreInst>(U);
+ if (Store == nullptr || Store->getPointerOperand() != Alloca)
+ return false;
+ }
+ return true;
+}
+
+// Matches insertelement(insertelement(undef, load Low, 0), load High, 1) where
+// the loads sit in the call's block with no store to either register between
+// them and the call.
+static bool matchLoadedPair(Value *Vector, CallInst *Call, RegisterPair &Pair)
+{
+ auto *Second = dyn_cast<InsertElementInst>(Vector);
+ if (Second == nullptr)
+ return false;
+ auto *First = dyn_cast<InsertElementInst>(Second->getOperand(0));
+ auto *SecondIndex = dyn_cast<ConstantInt>(Second->getOperand(2));
+ if (First == nullptr || SecondIndex == nullptr || SecondIndex->getZExtValue() != 1)
+ return false;
+ auto *FirstIndex = dyn_cast<ConstantInt>(First->getOperand(2));
+ if (FirstIndex == nullptr || FirstIndex->getZExtValue() != 0 || !isa<UndefValue>(First->getOperand(0)))
+ return false;
+ auto *LowLoad = dyn_cast<LoadInst>(First->getOperand(1));
+ auto *HighLoad = dyn_cast<LoadInst>(Second->getOperand(1));
+ if (LowLoad == nullptr || HighLoad == nullptr || LowLoad->getParent() != Call->getParent() ||
+ HighLoad->getParent() != Call->getParent())
+ return false;
+ Pair.Low = dyn_cast<AllocaInst>(LowLoad->getPointerOperand());
+ Pair.High = dyn_cast<AllocaInst>(HighLoad->getPointerOperand());
+ if (Pair.Low == nullptr || Pair.High == nullptr || Pair.Low == Pair.High ||
+ !Pair.Low->getAllocatedType()->isIntegerTy(32) || !Pair.High->getAllocatedType()->isIntegerTy(32) ||
+ !isPlainRegister(Pair.Low) || !isPlainRegister(Pair.High))
+ return false;
+ Instruction *Earliest = LowLoad->comesBefore(HighLoad) ? LowLoad : HighLoad;
+ for (Instruction *I = Earliest->getNextNode(); I != Call; I = I->getNextNode())
+ if (auto *Store = dyn_cast<StoreInst>(I))
+ if (Store->getPointerOperand() == Pair.Low || Store->getPointerOperand() == Pair.High)
+ return false;
+ return true;
+}
+
+// Finds the stores of both lanes of an MMA result into a register pair.
+static bool matchStoredPair(CallInst *Call, RegisterPair &Pair, StoreInst *Stores[2])
+{
+ Stores[0] = Stores[1] = nullptr;
+ for (User *U : Call->users())
+ {
+ auto *Extract = dyn_cast<ExtractElementInst>(U);
+ auto *Index = Extract != nullptr ? dyn_cast<ConstantInt>(Extract->getIndexOperand()) : nullptr;
+ if (Index == nullptr || Index->getZExtValue() > 1 || !Extract->hasOneUse())
+ return false;
+ auto *Store = dyn_cast<StoreInst>(Extract->user_back());
+ if (Store == nullptr || Store->getValueOperand() != Extract || Stores[Index->getZExtValue()] != nullptr)
+ return false;
+ Stores[Index->getZExtValue()] = Store;
+ }
+ if (Stores[0] == nullptr || Stores[1] == nullptr)
+ return false;
+ Pair.Low = dyn_cast<AllocaInst>(Stores[0]->getPointerOperand());
+ Pair.High = dyn_cast<AllocaInst>(Stores[1]->getPointerOperand());
+ return Pair.Low != nullptr && Pair.High != nullptr && Pair.Low != Pair.High;
+}
+
+// Matches an N-element vector built as a chain of insertelements (indices
+// 0..N-1 into undef) of loads from plain i32 registers; returns the registers.
+static bool matchLoadedVector(Value *Vector, unsigned Elements, SmallVectorImpl<AllocaInst *> &Registers)
+{
+ Registers.assign(Elements, nullptr);
+ for (unsigned Remaining = Elements; Remaining > 0; --Remaining)
+ {
+ auto *Insert = dyn_cast<InsertElementInst>(Vector);
+ auto *Index = Insert != nullptr ? dyn_cast<ConstantInt>(Insert->getOperand(2)) : nullptr;
+ if (Index == nullptr || Index->getZExtValue() != Remaining - 1)
+ return false;
+ auto *Load = dyn_cast<LoadInst>(Insert->getOperand(1));
+ auto *Register = Load != nullptr ? dyn_cast<AllocaInst>(Load->getPointerOperand()) : nullptr;
+ if (Register == nullptr || !Register->getAllocatedType()->isIntegerTy(32) || !isPlainRegister(Register))
+ return false;
+ Registers[Remaining - 1] = Register;
+ Vector = Insert->getOperand(0);
+ }
+ return isa<UndefValue>(Vector);
+}
+
+// The registers are plain allocas (only loaded and stored directly), so only
+// explicit stores can change them.
+static bool storesBetween(Instruction *From, Instruction *To, ArrayRef<AllocaInst *> Registers)
+{
+ for (Instruction *I = From->getNextNode(); I != nullptr && I != To; I = I->getNextNode())
+ if (auto *Store = dyn_cast<StoreInst>(I))
+ if (is_contained(Registers, Store->getPointerOperand()))
+ return true;
+ return false;
+}
+
+struct Helpers
+{
+ FunctionCallee AFromB, BFromA, Mma, FromNv, ToNv, AFromBPair, MmaPair;
+};
+
+static Value *loadPair(IRBuilder<> &Builder, const RegisterPair &Pair)
+{
+ Type *I32 = Builder.getInt32Ty();
+ Value *Vector = UndefValue::get(FixedVectorType::get(I32, 2));
+ Vector = Builder.CreateInsertElement(Vector, Builder.CreateLoad(I32, Pair.Low), Builder.getInt8(0));
+ return Builder.CreateInsertElement(Vector, Builder.CreateLoad(I32, Pair.High), Builder.getInt8(1));
+}
+
+static unsigned rewriteFunction(Function &F, Function *Mma, const Helpers &H)
+{
+ SmallVector<CallInst *, 64> Calls;
+ for (Instruction &I : instructions(F))
+ if (auto *Call = dyn_cast<CallInst>(&I))
+ if (Call->getCalledFunction() == Mma && Call->arg_size() == 3)
+ Calls.push_back(Call);
+ if (Calls.empty())
+ return 0;
+
+ struct Site
+ {
+ CallInst *Call;
+ bool HasC;
+ RegisterPair C;
+ bool HasD;
+ RegisterPair D;
+ StoreInst *DStores[2];
+ };
+ SmallVector<Site, 64> Sites;
+ for (CallInst *Call : Calls)
+ {
+ Site S{Call, false, {}, false, {}, {nullptr, nullptr}};
+ S.HasC = matchLoadedPair(Call->getArgOperand(2), Call, S.C);
+ S.HasD = matchStoredPair(Call, S.D, S.DStores);
+ Sites.push_back(S);
+ }
+
+ // Accumulator pairs whose registers belong to no other pair are replaced
+ // outright by a shadow in WMMA layout.
+ SmallVector<RegisterPair, 32> Shadowed;
+ DenseMap<AllocaInst *, int> Owner; // register -> shadow index, -1 if ambiguous
+ auto consider = [&](const RegisterPair &Pair) {
+ for (const RegisterPair &Known : Shadowed)
+ if (Known == Pair)
+ return;
+ int Index = static_cast<int>(Shadowed.size());
+ for (AllocaInst *Member : {Pair.Low, Pair.High})
+ {
+ auto Found = Owner.find(Member);
+ if (Found != Owner.end())
+ Found->second = -1;
+ else
+ Owner[Member] = Index;
+ }
+ Shadowed.push_back(Pair);
+ };
+ for (const Site &S : Sites)
+ if (S.HasC)
+ consider(S.C);
+ auto shadowOf = [&](const RegisterPair &Pair) -> int {
+ auto Low = Owner.find(Pair.Low);
+ auto High = Owner.find(Pair.High);
+ if (Low == Owner.end() || High == Owner.end() || Low->second < 0 || Low->second != High->second)
+ return -1;
+ return Shadowed[Low->second] == Pair ? Low->second : -1;
+ };
+
+ Type *Pair32 = FixedVectorType::get(Type::getInt32Ty(F.getContext()), 2);
+ SmallVector<AllocaInst *, 32> Shadows;
+ {
+ IRBuilder<> Builder(&*F.getEntryBlock().getFirstInsertionPt());
+ for (size_t Index = 0; Index < Shadowed.size(); ++Index)
+ Shadows.push_back(
+ shadowOf(Shadowed[Index]) == static_cast<int>(Index)
+ ? Builder.CreateAlloca(Pair32, Shadowed[Index].Low->getAddressSpace(), nullptr, "wmma.shadow")
+ : nullptr);
+ }
+ auto shadowFor = [&](const RegisterPair &Pair) -> AllocaInst * {
+ int Index = shadowOf(Pair);
+ return Index >= 0 ? Shadows[Index] : nullptr;
+ };
+
+ // Snapshot every access to shadowed registers before rewriting anything.
+ SmallVector<LoadInst *, 64> Loads;
+ SmallVector<StoreInst *, 64> Stores;
+ for (auto &[Register, Index] : Owner)
+ {
+ if (Index < 0 || Shadows[Index] == nullptr)
+ continue;
+ for (User *U : Register->users())
+ {
+ if (auto *Load = dyn_cast<LoadInst>(U))
+ Loads.push_back(Load);
+ else
+ Stores.push_back(cast<StoreInst>(U));
+ }
+ }
+ DenseSet<StoreInst *> ResultStores; // MMA result stores replaced by shadow stores
+
+ auto accumulator = [&](const Site &S, IRBuilder<> &Builder) -> Value * {
+ if (S.HasC)
+ if (AllocaInst *Shadow = shadowFor(S.C))
+ return Builder.CreateLoad(Pair32, Shadow);
+ return Builder.CreateCall(H.FromNv, {S.Call->getArgOperand(2)});
+ };
+ auto finish = [&](const Site &S, Value *D, IRBuilder<> &Builder) {
+ if (S.HasD)
+ if (AllocaInst *Shadow = shadowFor(S.D))
+ {
+ // The result goes to the shadow where its register stores were.
+ StoreInst *Last = S.DStores[0]->comesBefore(S.DStores[1]) ? S.DStores[1] : S.DStores[0];
+ IRBuilder<> At(Last);
+ At.CreateStore(D, Shadow);
+ ResultStores.insert(S.DStores[0]);
+ ResultStores.insert(S.DStores[1]);
+ }
+ return Builder.CreateCall(H.ToNv, {D});
+ };
+ // Two MMAs that share A (same registers, no store in between to anything
+ // the second one reads) run as one 16x16x16 WMMA placed at the first.
+ auto pairable = [&](const Site &First, const Site &Second) {
+ SmallVector<AllocaInst *, 4> AFirst, ASecond, BSecond;
+ if (First.Call->getParent() != Second.Call->getParent() || !Second.HasC ||
+ shadowFor(Second.C) == nullptr || !First.Call->comesBefore(Second.Call) ||
+ !matchLoadedVector(First.Call->getArgOperand(0), 4, AFirst) ||
+ !matchLoadedVector(Second.Call->getArgOperand(0), 4, ASecond) || AFirst != ASecond ||
+ !matchLoadedVector(Second.Call->getArgOperand(1), 2, BSecond))
+ return false;
+ // The second B is reloaded at the first MMA, so it must not live in a
+ // shadow (those registers are no longer kept up to date).
+ for (AllocaInst *Register : BSecond)
+ {
+ auto Found = Owner.find(Register);
+ if (Found != Owner.end() && Found->second >= 0 && Shadows[Found->second] != nullptr)
+ return false;
+ }
+ SmallVector<AllocaInst *, 8> Read(AFirst.begin(), AFirst.end());
+ Read.append(BSecond.begin(), BSecond.end());
+ Read.push_back(Second.C.Low);
+ Read.push_back(Second.C.High);
+ return !storesBetween(First.Call, Second.Call, Read);
+ };
+ for (size_t Index = 0; Index < Sites.size(); ++Index)
+ {
+ const Site &S = Sites[Index];
+ CallInst *Call = S.Call;
+ IRBuilder<> Builder(Call);
+ Value *BTransposed = Builder.CreateCall(H.BFromA, {Call->getArgOperand(0)});
+ if (Index + 1 < Sites.size() && pairable(S, Sites[Index + 1]))
+ {
+ const Site &Next = Sites[Index + 1];
+ SmallVector<AllocaInst *, 2> BNext;
+ matchLoadedVector(Next.Call->getArgOperand(1), 2, BNext);
+ Value *B1 = loadPair(Builder, RegisterPair{BNext[0], BNext[1]});
+ Value *ATransposed = Builder.CreateCall(H.AFromBPair, {Call->getArgOperand(1), B1});
+ Value *C0 = accumulator(S, Builder);
+ Value *C1 = accumulator(Next, Builder);
+ Value *D = Builder.CreateCall(H.MmaPair, {ATransposed, BTransposed, C0, C1});
+ Value *D0 = Builder.CreateShuffleVector(D, ArrayRef<int>{0, 1});
+ Value *D1 = Builder.CreateShuffleVector(D, ArrayRef<int>{2, 3});
+ Call->replaceAllUsesWith(finish(S, D0, Builder));
+ Next.Call->replaceAllUsesWith(finish(Next, D1, Builder));
+ Call->eraseFromParent();
+ Next.Call->eraseFromParent();
+ ++Index;
+ continue;
+ }
+ Value *ATransposed = Builder.CreateCall(H.AFromB, {Call->getArgOperand(1)});
+ Value *D = Builder.CreateCall(H.Mma, {ATransposed, BTransposed, accumulator(S, Builder)});
+ Call->replaceAllUsesWith(finish(S, D, Builder));
+ Call->eraseFromParent();
+ }
+
+ // Every other access now goes through the shadow.
+ for (LoadInst *Load : Loads)
+ {
+ auto *Register = cast<AllocaInst>(Load->getPointerOperand());
+ const RegisterPair &Pair = Shadowed[Owner[Register]];
+ IRBuilder<> Builder(Load);
+ Value *Nv = Builder.CreateCall(H.ToNv, {Builder.CreateLoad(Pair32, shadowFor(Pair))});
+ Load->replaceAllUsesWith(Builder.CreateExtractElement(Nv, Builder.getInt8(Register == Pair.High)));
+ Load->eraseFromParent();
+ }
+ for (StoreInst *Store : Stores)
+ {
+ if (!ResultStores.contains(Store))
+ {
+ auto *Register = cast<AllocaInst>(Store->getPointerOperand());
+ const RegisterPair &Pair = Shadowed[Owner[Register]];
+ AllocaInst *Shadow = shadowFor(Pair);
+ IRBuilder<> Builder(Store);
+ Value *Nv = Builder.CreateCall(H.ToNv, {Builder.CreateLoad(Pair32, Shadow)});
+ Nv = Builder.CreateInsertElement(Nv, Store->getValueOperand(), Builder.getInt8(Register == Pair.High));
+ Builder.CreateStore(Builder.CreateCall(H.FromNv, {Nv}), Shadow);
+ }
+ auto *Operand = dyn_cast<Instruction>(Store->getValueOperand());
+ Store->eraseFromParent();
+ if (Operand != nullptr && Operand->use_empty() && !Operand->mayHaveSideEffects())
+ Operand->eraseFromParent();
+ }
+ return Calls.size();
+}
+} // namespace
+
+unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef)
+{
+ Module *M = unwrap(ModuleRef);
+ Function *Mma = M->getFunction(kMmaF16);
+ if (Mma == nullptr || Mma->use_empty())
+ return 0;
+ LLVMContext &Context = M->getContext();
+ Type *I32 = Type::getInt32Ty(Context);
+ Type *V2 = FixedVectorType::get(I32, 2);
+ Type *V4 = FixedVectorType::get(I32, 4);
+ Type *V8 = FixedVectorType::get(I32, 8);
+ AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
+ AttrBuilder(Context, Mma->getAttributes().getFnAttrs()));
+ Helpers H;
+ H.AFromB = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b", Attributes, V8, V2);
+ H.BFromA = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_b_from_a", Attributes, V8, V4);
+ H.Mma = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16", Attributes, V2, V8, V8, V2);
+ H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
+ H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
+ H.AFromBPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b_pair", Attributes, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16_pair", Attributes, V4, V8, V8, V2, V2);
+ unsigned Rewritten = 0;
+ for (Function &F : *M)
+ if (!F.isDeclaration())
+ Rewritten += rewriteFunction(F, Mma, H);
+ return Rewritten;
+}
+
bool LLVMZludaParseCommandLineOptions(int argc, const char *const *argv,
char **ErrorMessage)
{
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index 69d1faa..c11e06a 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1235,6 +1235,184 @@ extern "C"
}
}
+// gfx11 WMMA lowering of mma.sync.aligned.m16n8k16.row.col.f16.f16.f16.f16.
+//
+// llvm_zluda's MMA rewrite (LLVMZludaRewriteMmaF16Wmma) replaces each such MMA
+// with these helpers. It computes D^T = B^T A^T + C^T with
+// v_wmma_f32_16x16x16_f16, so the 16x8 accumulator occupies WMMA result
+// rows 0..7, i.e. VGPRs 0..3 of the f32 result: lane l holds D[m][n] for
+// m = l % 16 and n = 2v + l / 16, v = 0..3 (the "transposed WMMA layout").
+// Those four values are kept as two f16x2 words (v0 | v1 << 16,
+// v2 | v3 << 16), the same register footprint as NVIDIA's own fragment.
+// Converting between the two layouts is pure 16-bit data movement, so the
+// rewrite keeps accumulators in WMMA layout across chained MMAs and converts
+// only where other code reads or writes them. Each MMA still rounds its result
+// to f16, as the .f16 accumulator requires; only the f32 summation order
+// within one MMA differs from the dot-product fallback.
+typedef _Float16 wmma_half16 __attribute__((ext_vector_type(16)));
+typedef float wmma_float8 __attribute__((ext_vector_type(8)));
+typedef uint32_t wmma_uint8 __attribute__((ext_vector_type(8)));
+__device__ wmma_float8 __zluda_wmma_f32_16x16x16_f16(wmma_half16 a, wmma_half16 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v16f16");
+
+__device__ static uint32_t wmma_lane()
+{
+ return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
+}
+
+__device__ static uint32_t wmma_from_lane(uint32_t lane, uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_ds_bpermute(int(lane << 2), int(value)));
+}
+
+// Gathers WMMA operand dword q for every lane from two NVIDIA fragment
+// registers: rows r < 8 take `low` from lane 4r+q, rows r >= 8 take `high`
+// from lane 4(r-8)+q (r = lane % 16). Swapping `high` between quad halves
+// (DPP quad_perm [2,3,0,1]) parks the value rows r >= 8 need in lane
+// 4(r-8)+(q^2), so one bpermute serves both halves.
+__device__ static uint32_t wmma_gather_rows(uint32_t low, uint32_t high_swapped, uint32_t q)
+{
+ uint32_t lane = wmma_lane();
+ uint32_t row = lane % 16;
+ uint32_t source = lane % 4 == q ? low : high_swapped;
+ uint32_t from = row < 8 ? row * 4 + q : (row - 8) * 4 + (q ^ 2);
+ return wmma_from_lane(from, source);
+}
+
+__device__ static uint32_t wmma_swap_quad_halves(uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_mov_dpp(int(value), 0b01'00'11'10, 0xf, 0xf, true));
+}
+
+extern "C"
+{
+ // WMMA A operand (lane l holds row l % 16 over k) = B^T, from an NVIDIA B
+ // fragment (lane 4n+q: B[2q..2q+1][n] in .x, B[2q+8..2q+9][n] in .y).
+ // Rows 8..15 (padding columns of the 16x8 B) are zero.
+ wmma_uint8 FUNC(d4r_wmma_t_a_from_b)(uint2::Native_vec_ b)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t n = wmma_lane() % 16;
+ uint32_t source = (n % 8) * 4;
+ wmma_uint8 result;
+ for (uint32_t q = 0; q < 4; q++)
+ {
+ uint32_t low = wmma_from_lane(source + q, b.x);
+ uint32_t high = wmma_from_lane(source + q, b.y);
+ result[q] = n < 8 ? low : 0u;
+ result[q + 4] = n < 8 ? high : 0u;
+ }
+ return result;
+ }
+
+ // WMMA B operand (lane l holds column l % 16 over k) = A^T, i.e. row
+ // m = l % 16 of the NVIDIA A fragment (lane 4g+t: rows g/g+8, k 2t..2t+1 in
+ // .x/.y and k 2t+8..2t+9 in .z/.w).
+ wmma_uint8 FUNC(d4r_wmma_t_b_from_a)(uint4::Native_vec_ a)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t ay = wmma_swap_quad_halves(a.y);
+ uint32_t aw = wmma_swap_quad_halves(a.w);
+ wmma_uint8 result;
+ for (uint32_t p = 0; p < 4; p++)
+ {
+ result[p] = wmma_gather_rows(a.x, ay, p);
+ result[p + 4] = wmma_gather_rows(a.z, aw, p);
+ }
+ return result;
+ }
+
+ // Paired form: the WMMA A operand rows 0..7 are B0^T and rows 8..15 are
+ // B1^T, so one WMMA computes two m16n8k16 MMAs that share A.
+ wmma_uint8 FUNC(d4r_wmma_t_a_from_b_pair)(uint2::Native_vec_ b0, uint2::Native_vec_ b1)
+ {
+ // Keep the scheduler from hoisting these shuffles (and the second
+ // candidate of each select) far ahead of the WMMA that consumes them,
+ // which otherwise spills them to scratch.
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t b1x = wmma_swap_quad_halves(b1.x);
+ uint32_t b1y = wmma_swap_quad_halves(b1.y);
+ wmma_uint8 result;
+ for (uint32_t q = 0; q < 4; q++)
+ {
+ result[q] = wmma_gather_rows(b0.x, b1x, q);
+ result[q + 4] = wmma_gather_rows(b0.y, b1y, q);
+ }
+ return result;
+ }
+
+ // Two accumulators (for B0 and B1) in transposed WMMA layout: .xy is the
+ // first, .zw the second; they are WMMA result VGPRs 0..3 and 4..7.
+ uint4::Native_vec_ FUNC(d4r_wmma_t_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ f16x2 c01 = std::bit_cast<f16x2>(c0_t.x);
+ f16x2 c23 = std::bit_cast<f16x2>(c0_t.y);
+ f16x2 c45 = std::bit_cast<f16x2>(c1_t.x);
+ f16x2 c67 = std::bit_cast<f16x2>(c1_t.y);
+ wmma_float8 c = {float(c01.x), float(c01.y), float(c23.x), float(c23.y),
+ float(c45.x), float(c45.y), float(c67.x), float(c67.y)};
+ wmma_float8 d = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a_t), std::bit_cast<wmma_half16>(b_t), c);
+ f16x2 d01{static_cast<f16>(d[0]), static_cast<f16>(d[1])};
+ f16x2 d23{static_cast<f16>(d[2]), static_cast<f16>(d[3])};
+ f16x2 d45{static_cast<f16>(d[4]), static_cast<f16>(d[5])};
+ f16x2 d67{static_cast<f16>(d[6]), static_cast<f16>(d[7])};
+ return {std::bit_cast<uint32_t>(d01), std::bit_cast<uint32_t>(d23),
+ std::bit_cast<uint32_t>(d45), std::bit_cast<uint32_t>(d67)};
+ }
+
+ // D = round_f16(A B + C) with C and D in the transposed WMMA layout.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ f16x2 c01 = std::bit_cast<f16x2>(c_t.x);
+ f16x2 c23 = std::bit_cast<f16x2>(c_t.y);
+ wmma_float8 c = {float(c01.x), float(c01.y), float(c23.x), float(c23.y), 0.0f, 0.0f, 0.0f, 0.0f};
+ wmma_float8 d = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a_t), std::bit_cast<wmma_half16>(b_t), c);
+ f16x2 d01{static_cast<f16>(d[0]), static_cast<f16>(d[1])};
+ f16x2 d23{static_cast<f16>(d[2]), static_cast<f16>(d[3])};
+ return {std::bit_cast<uint32_t>(d01), std::bit_cast<uint32_t>(d23)};
+ }
+
+ // NVIDIA accumulator fragment (lane 4g+t: D[g][2t..2t+1] in .x,
+ // D[g+8][2t..2t+1] in .y) -> transposed WMMA layout.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_from_nv)(uint2::Native_vec_ c)
+ {
+ uint32_t lane = wmma_lane();
+ uint32_t m = lane % 16;
+ uint32_t shift = (lane / 16) * 16;
+ uint32_t source = (m % 8) * 4;
+ uint32_t halves[4];
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ uint32_t low_rows = wmma_from_lane(source + v, c.x);
+ uint32_t high_rows = wmma_from_lane(source + v, c.y);
+ halves[v] = ((m >= 8 ? high_rows : low_rows) >> shift) & 0xffffu;
+ }
+ return {halves[0] | (halves[1] << 16), halves[2] | (halves[3] << 16)};
+ }
+
+ // Transposed WMMA layout -> NVIDIA accumulator fragment.
+ uint2::Native_vec_ FUNC(d4r_wmma_t_to_nv)(uint2::Native_vec_ d)
+ {
+ uint32_t lane = wmma_lane();
+ uint32_t g = lane / 4;
+ uint32_t t = lane % 4;
+ uint32_t sources[4] = {g, g + 16, g + 8, g + 24}; // D[g][2t], D[g][2t+1], D[g+8][2t], D[g+8][2t+1]
+ uint32_t halves[4];
+ for (uint32_t i = 0; i < 4; i++)
+ {
+ uint32_t v01 = wmma_from_lane(sources[i], d.x);
+ uint32_t v23 = wmma_from_lane(sources[i], d.y);
+ halves[i] = ((t >= 2 ? v23 : v01) >> ((t % 2) * 16)) & 0xffffu;
+ }
+ return {halves[0] | (halves[1] << 16), halves[2] | (halves[3] << 16)};
+ }
+}
+
template <typename T, typename A, typename B, int BOffset, typename OcklFunc>
__device__ T dp2a_impl(T a, T b, T c, OcklFunc ockl_func)
{
diff --git a/zluda_common/examples/d4r_fatbin_ptx.rs b/zluda_common/examples/d4r_fatbin_ptx.rs
new file mode 100644
index 0000000..50f7a09
--- /dev/null
+++ b/zluda_common/examples/d4r_fatbin_ptx.rs
@@ -0,0 +1,36 @@
+// Extracts (and decompresses) every PTX file from captured CUDA fatbin images.
+// usage: d4r_fatbin_ptx <outdir> <image>...
+use cuda_types::dark_api::FatbinFileHeader;
+use std::{env, fs, path::PathBuf};
+use zluda_common::{CodeLibraryRef, CodeModuleRef};
+
+fn main() {
+ let mut args = env::args().skip(1);
+ let outdir = PathBuf::from(args.next().expect("output directory"));
+ fs::create_dir_all(&outdir).expect("create output directory");
+ for path in args {
+ let mut bytes = fs::read(&path).expect("read image");
+ bytes.extend_from_slice(&[0; 16]);
+ let stem = PathBuf::from(&path).file_stem().unwrap().to_string_lossy().into_owned();
+ let library = unsafe { CodeLibraryRef::try_load(bytes.as_ptr().cast()) }.expect("load image");
+ let mut count = 0;
+ unsafe {
+ library.iterate_modules(|_, module| {
+ let text = match module {
+ Ok(CodeModuleRef::Text(ptx)) => Some(ptx.to_string()),
+ Ok(CodeModuleRef::File(file)) if file.header.kind == FatbinFileHeader::HEADER_KIND_PTX => file
+ .get_or_decompress_content(true)
+ .ok()
+ .map(|content| String::from_utf8_lossy(&content).trim_end_matches('\0').to_string()),
+ _ => None,
+ };
+ if let Some(text) = text {
+ let name = outdir.join(format!("{stem}-{count:02}.ptx"));
+ fs::write(&name, text).expect("write PTX");
+ count += 1;
+ }
+ })
+ };
+ println!("{path}: {count} PTX modules");
+ }
+}
patches/zluda/0004-native-overrides-and-gpu-handoffs.patch
-1672
View File
File diff suppressed because it is too large. Load diff
patches/zluda/0005-null-texture-objects-read-zero.patch
-40
View File
@@ -1,40 +0,0 @@
From: d4r
Subject: [PATCH] null texture objects read as zero
CUDA texture object 0 reads as zeros on NVIDIA GPUs, and DLSS samples absent optional inputs
(its exposure scales, with the flags Ghost of Tsushima uses) that way. A ZLUDA texture object is a
descriptor pointer, so handle 0 made the GPU read address 0 and fault. Handle 0 now points at an
all-zero descriptor, which AMD hardware treats as a null resource. Rebuild ptx/lib/*.bc after
applying (scripts/build_zluda_ptx_helpers.sh).
---
--- a/ptx/lib/zluda_ptx_impl.cpp 2026-09-27 16:47:02.815623944 -0400
+++ b/ptx/lib/zluda_ptx_impl.cpp 2026-09-27 16:47:02.815623944 -0400
@@ -1725,9 +1725,18 @@
c;
}
+ // d4r: CUDA texture object 0 reads as zeros on NVIDIA GPUs, and DLSS samples absent optional inputs
+ // (exposure scales) that way. A ZLUDA texture object is a descriptor pointer, so 0 is redirected to
+ // an all-zero descriptor, which AMD hardware treats as a null resource (reads return 0).
+ __device__ static unsigned int d4r_null_texture[32];
+ static uint64_t texture_or_null(uint64_t texobj) __device__
+ {
+ return texobj != 0 ? texobj : uint64_t(uintptr_t(d4r_null_texture));
+ }
+
static std::pair<GLOBAL_SPACE void *, GLOBAL_SPACE void *> get_image_and_sampler(uint64_t texobj) __device__
{
- unsigned int GLOBAL_SPACE *image = (unsigned int GLOBAL_SPACE *)texobj;
+ unsigned int GLOBAL_SPACE *image = (unsigned int GLOBAL_SPACE *)texture_or_null(texobj);
unsigned int GLOBAL_SPACE *sampler = image + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
return {image, sampler};
}
@@ -1868,7 +1877,7 @@
#define tex_2d_gather4(COMPONENT) \
v4f32 FUNC(texobj_gather4##COMPONENT##_2d_v4_f32_f32)(uint64_t texobj, v2f32 coord) \
{ \
- auto image = reinterpret_cast<unsigned int CONSTANT_SPACE *>(static_cast<uintptr_t>(texobj)); \
+ auto image = reinterpret_cast<unsigned int CONSTANT_SPACE *>(static_cast<uintptr_t>(texture_or_null(texobj))); \
auto sampler = image + HIP_SAMPLER_OBJECT_OFFSET_DWORD; \
float2 coordinates{coord.x, coord.y}; \
auto result = __ockl_image_gather4##COMPONENT##_2D(image, sampler, get_native_vector(coordinates)); \
patches/zluda/0006-k8-wmma-prep-slots-wave64.patch
-444
View File
@@ -1,444 +0,0 @@
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 5d8c0a5..b2870e5 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -70,8 +70,10 @@ fn create_oclc_constants(ctx: &Context, gcn_arch: &str) -> Result<Module, String
let module = Module::new(ctx, c"oclc_constants");
// used by ockl
- add_constant(ctx, &module, c"__oclc_wavefrontsize64", 0);
- add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", 5);
+ // d4r: D4R_ZLUDA_WAVE64=1 (see ptx::pass::wave64_requested)
+ let wave64 = std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1");
+ add_constant(ctx, &module, c"__oclc_wavefrontsize64", if wave64 { 1 } else { 0 });
+ add_constant(ctx, &module, c"__oclc_wavefrontsize_log2", if wave64 { 6 } else { 5 });
add_constant(ctx, &module, c"__oclc_ABI_version", 500);
add_constant(
ctx,
@@ -174,7 +176,13 @@ pub fn compile(
let triple = c"amdgcn-amd-amdhsa";
let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
- let features = c"-wavefrontsize64,+cumode";
+ // functions without their own target features (the linked helper libraries) take these, so under
+ // D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
+ let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
+ c"-wavefrontsize32,+wavefrontsize64,+cumode"
+ } else {
+ c"-wavefrontsize64,+cumode"
+ };
let mut target = unsafe { std::mem::zeroed() };
let mut err = std::ptr::null_mut();
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 88848b1..0d1f9cf 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -869,10 +869,111 @@ unsigned LLVMZludaSplitWmmaM16n16k16(LLVMModuleRef ModuleRef)
return static_cast<unsigned>(Calls.size());
}
+// d4r: an m16n8k8 f16 MMA is an m16n8k16 one whose second K half is zero (its A fragment is the k16
+// A's registers 0-1, its B the k16 B's register 0), and adding zero products is exact. Each k8 call
+// becomes a k16 call so the WMMA rewrite below lowers it too. Zero halves are loads of one zeroed
+// register, so the operands stay "loads of plain registers" and two k8 MMAs that share A still fuse
+// into one WMMA. Returns the number of calls converted.
+static unsigned widenMmaK8(Module *M)
+{
+ Function *K8 = M->getFunction("__zluda_ptx_impl_mma_sync_aligned_m16n8k8_row_col_f16_f16_f16_f16");
+ if (K8 == nullptr || K8->use_empty())
+ return 0;
+ LLVMContext &Context = M->getContext();
+ Type *I32 = Type::getInt32Ty(Context);
+ Type *V2 = FixedVectorType::get(I32, 2);
+ Type *V4 = FixedVectorType::get(I32, 4);
+ AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
+ AttrBuilder(Context, K8->getAttributes().getFnAttrs()));
+ FunctionCallee K16 = M->getOrInsertFunction(kMmaF16, Attributes, V2, V4, V2, V2);
+ SmallVector<CallInst *, 64> Calls;
+ for (User *U : K8->users())
+ if (auto *Call = dyn_cast<CallInst>(U); Call != nullptr && Call->getCalledFunction() == K8)
+ Calls.push_back(Call);
+ DenseMap<Function *, AllocaInst *> Zeros;
+ for (CallInst *Call : Calls)
+ {
+ Function *F = Call->getFunction();
+ AllocaInst *&Zero = Zeros[F];
+ if (Zero == nullptr)
+ {
+ // an alloca like ZLUDA's register allocas (same address space), stored once at entry
+ AllocaInst *Like = nullptr;
+ for (Instruction &I : F->getEntryBlock())
+ if ((Like = dyn_cast<AllocaInst>(&I)) != nullptr)
+ break;
+ IRBuilder<> Entry(&*F->getEntryBlock().getFirstInsertionPt());
+ Zero = Entry.CreateAlloca(I32, Like != nullptr ? Like->getAddressSpace() : 5, nullptr, "d4r.k8.zero");
+ Entry.CreateStore(Entry.getInt32(0), Zero);
+ }
+ IRBuilder<> Builder(Call);
+ SmallVector<AllocaInst *, 2> ARegisters, BRegisters;
+ auto build = [&](ArrayRef<Value *> Elements) {
+ Value *Vector = UndefValue::get(FixedVectorType::get(I32, Elements.size()));
+ for (unsigned Index = 0; Index < Elements.size(); ++Index)
+ Vector = Builder.CreateInsertElement(Vector, Elements[Index], Builder.getInt8(Index));
+ return Vector;
+ };
+ auto zero = [&]() { return Builder.CreateLoad(I32, Zero); };
+ // the registers may be reloaded at the call only if no store reaches them after their original loads
+ auto unchangedSinceLoads = [&](Value *Vector) {
+ for (Value *V = Vector; auto *Insert = dyn_cast<InsertElementInst>(V); V = Insert->getOperand(0))
+ {
+ auto *Load = cast<LoadInst>(Insert->getOperand(1));
+ if (Load->getParent() != Call->getParent() ||
+ storesBetween(Load, Call, {cast<AllocaInst>(Load->getPointerOperand())}))
+ return false;
+ }
+ return true;
+ };
+ // B is one register: a <1 x i32> vector or a plain i32
+ Value *BArg = Call->getArgOperand(1);
+ auto bRegister = [&]() {
+ if (BArg->getType()->isVectorTy())
+ return matchLoadedVector(BArg, 1, BRegisters) && unchangedSinceLoads(BArg);
+ auto *Load = dyn_cast<LoadInst>(BArg);
+ auto *Register = Load != nullptr ? dyn_cast<AllocaInst>(Load->getPointerOperand()) : nullptr;
+ if (Register == nullptr || !Register->getAllocatedType()->isIntegerTy(32) || !isPlainRegister(Register) ||
+ Load->getParent() != Call->getParent() || storesBetween(Load, Call, {Register}))
+ return false;
+ BRegisters.assign(1, Register);
+ return true;
+ };
+ auto element = [&](Value *V, unsigned Index) -> Value * {
+ return V->getType()->isVectorTy() ? Builder.CreateExtractElement(V, Builder.getInt8(Index)) : V;
+ };
+ Value *A, *B;
+ if (matchLoadedVector(Call->getArgOperand(0), 2, ARegisters) && bRegister() &&
+ unchangedSinceLoads(Call->getArgOperand(0)))
+ {
+ // reload the same registers right at the call (nothing lies between these loads and it)
+ Value *A0 = Builder.CreateLoad(I32, ARegisters[0]), *A1 = Builder.CreateLoad(I32, ARegisters[1]);
+ Value *B0 = Builder.CreateLoad(I32, BRegisters[0]);
+ A = build({A0, A1, zero(), zero()});
+ B = build({B0, zero()});
+ }
+ else
+ {
+ Value *A0 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(0));
+ Value *A1 = Builder.CreateExtractElement(Call->getArgOperand(0), Builder.getInt8(1));
+ Value *B0 = element(BArg, 0);
+ A = build({A0, A1, Builder.getInt32(0), Builder.getInt32(0)});
+ B = build({B0, Builder.getInt32(0)});
+ }
+ CallInst *Wide = Builder.CreateCall(K16, {A, B, Call->getArgOperand(2)});
+ Call->replaceAllUsesWith(Wide);
+ Call->eraseFromParent();
+ }
+ return static_cast<unsigned>(Calls.size());
+}
+
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
Module *M = unwrap(ModuleRef);
+ // D4R_ZLUDA_WMMA_K8=0 keeps m16n8k8 on the dot-product fallback
+ if (const char *K8Env = getenv("D4R_ZLUDA_WMMA_K8"); K8Env == nullptr || K8Env[0] != '0')
+ widenMmaK8(M);
Function *Mma = M->getFunction(kMmaF16);
Function *MmaE4m3 = (Flags & 1) != 0 ? M->getFunction(kMmaE4m3) : nullptr;
if (Mma != nullptr && Mma->use_empty())
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index cdefdf1..74cf36f 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -127,9 +127,32 @@ extern "C"
{
extern "C" __attribute__((constant)) uint32_t __oclc_ISA_version __device__;
+ // d4r: kernels may be compiled as wave64 (D4R_ZLUDA_WAVE64, offline texture-kernel builds, linked with
+ // the zluda_ptx_impl*_w64.bc helpers built with -DD4R_WAVE64), where each wave holds two CUDA warps:
+ // lanes 0-31 and 32-63. Warp-level helpers then act on the lane's own 32-lane half.
+#ifdef D4R_WAVE64
+ __device__ static inline uint32_t d4r_wave_lane()
+ {
+ return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u));
+ }
+ __device__ static inline uint32_t d4r_warp_base()
+ {
+ return d4r_wave_lane() & 32u;
+ }
+#else
+ __device__ static inline uint32_t d4r_warp_base()
+ {
+ return 0u;
+ }
+#endif
+
uint32_t FUNC(activemask)()
{
+#ifdef D4R_WAVE64
+ return (uint32_t)(__builtin_amdgcn_read_exec() >> d4r_warp_base());
+#else
return __builtin_amdgcn_read_exec_lo();
+#endif
}
size_t __ockl_get_local_id(uint32_t) __device__;
@@ -158,7 +181,11 @@ extern "C"
uint32_t FUNC(sreg_laneid)()
{
+#ifdef D4R_WAVE64
+ return d4r_wave_lane() & 31u;
+#else
return __lane_id();
+#endif
}
uint32_t FUNC(sreg_lanemask_le)()
@@ -322,7 +349,7 @@ extern "C"
int32_t section_mask = (opts >> 8) & 0b11111; \
int32_t warp_end = opts & 0b11111; \
delta &= 0b11111; \
- int32_t self = (int32_t)__lane_id(); \
+ int32_t self = (int32_t)__zluda_ptx_impl_sreg_laneid(); \
int32_t subsection = section_mask & self; \
int32_t subsection_end = subsection | (~section_mask & warp_end); \
int32_t idx = calculate_index; \
@@ -331,7 +358,7 @@ extern "C"
{ \
idx = self; \
} \
- int32_t output = __builtin_amdgcn_ds_bpermute(idx << 2, (int32_t)input); \
+ int32_t output = __builtin_amdgcn_ds_bpermute((int32_t)(d4r_warp_base() | (uint32_t)idx) << 2, (int32_t)input); \
return {(uint32_t)output, uint32_t(!out_of_bounds)}; \
} \
\
@@ -850,7 +877,11 @@ extern "C"
__device__ static inline uint32_t ballot(bool value, bool negate)
{
__builtin_amdgcn_wave_barrier();
+#ifdef D4R_WAVE64
+ return (uint32_t)(__builtin_amdgcn_ballot_w64(negate ? !value : value) >> d4r_warp_base());
+#else
return __builtin_amdgcn_ballot_w32(negate ? !value : value);
+#endif
}
bool FUNC(vote_sync_any_pred)(bool value, uint32_t membermask __attribute__((unused)))
@@ -866,7 +897,7 @@ extern "C"
// IMPORTANT: exec mask must be a subset of membermask, the behavior is undefined otherwise
bool FUNC(vote_sync_all_pred)(bool value, uint32_t membermask __attribute__((unused)))
{
- return ballot(value, false) == __builtin_amdgcn_read_exec_lo();
+ return ballot(value, false) == __zluda_ptx_impl_activemask();
}
// also known as "none"
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 8294bb4..6af2495 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -397,7 +397,12 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
fn emit_target_features(&mut self, fn_: LLVMValueRef) {
let value = format!(
- "+wavefrontsize32,-wavefrontsize64,+cumode{}",
+ "{},+cumode{}",
+ if crate::pass::wave64_requested() {
+ "-wavefrontsize32,+wavefrontsize64"
+ } else {
+ "+wavefrontsize32,-wavefrontsize64"
+ },
if cfg!(debug_assertions) {
",+precise-memory"
} else {
diff --git a/ptx/src/pass/mod.rs b/ptx/src/pass/mod.rs
index 63aa98c..e3baa72 100644
--- a/ptx/src/pass/mod.rs
+++ b/ptx/src/pass/mod.rs
@@ -37,6 +37,18 @@ mod test;
static ZLUDA_PTX_IMPL: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl.bc");
static ZLUDA_PTX_IMPL_CONSTRAINED: &'static [u8] =
include_bytes!("../../lib/zluda_ptx_impl_constrained.bc");
+static ZLUDA_PTX_IMPL_W64: &'static [u8] = include_bytes!("../../lib/zluda_ptx_impl_w64.bc");
+static ZLUDA_PTX_IMPL_CONSTRAINED_W64: &'static [u8] =
+ include_bytes!("../../lib/zluda_ptx_impl_constrained_w64.bc");
+
+/// d4r: D4R_ZLUDA_WAVE64=1 compiles kernels as wave64: each wave runs two CUDA warps (lanes 0-31 and
+/// 32-63; the warp-level helpers act on the lane's own half) and RDNA3 issues most FP32 VALU work for all
+/// 64 lanes at once. Meant for offline builds of texture kernels whose warp operations are limited to
+/// lane ids, shuffles and votes; not for MMA/matrix kernels (their helpers assume one warp per wave).
+pub fn wave64_requested() -> bool {
+ static VALUE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1"))
+}
const ZLUDA_PTX_PREFIX: &'static str = "__zluda_ptx_impl_";
quick_error! {
@@ -194,10 +206,11 @@ pub struct Module {
impl Module {
pub fn linked_bitcode(&self) -> &'static [u8] {
- if self.constrained_fp {
- ZLUDA_PTX_IMPL_CONSTRAINED
- } else {
- ZLUDA_PTX_IMPL
+ match (self.constrained_fp, wave64_requested()) {
+ (true, false) => ZLUDA_PTX_IMPL_CONSTRAINED,
+ (false, false) => ZLUDA_PTX_IMPL,
+ (true, true) => ZLUDA_PTX_IMPL_CONSTRAINED_W64,
+ (false, true) => ZLUDA_PTX_IMPL_W64,
}
}
}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index bacb93d..033d9a0 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -18,9 +18,14 @@ pub(crate) struct Function {
/// d4r native override: persistent kernels launch this many workgroups in x (0 = as launched)
pub(crate) grid_x: u32,
/// d4r native override: byte offset of the u64 parameter that keys the prep kernel's output
- /// (0 = run the prep before every launch) and the key of the last prep launched
- pub(crate) prep_key: u32,
+ /// (None = run the prep before every launch) and the key of the last prep launched
+ pub(crate) prep_key: Option<u32>,
pub(crate) prep_last: std::sync::atomic::AtomicU64,
+ /// d4r native override: with N > 0 prep slots the kernel keeps one prepared image per key for
+ /// up to N keys, so the prep runs only for keys not seen yet (and for every key beyond N, which
+ /// the kernel prepares into a shared overflow slot). 0 = only the last key is kept.
+ pub(crate) prep_slots: u32,
+ pub(crate) prep_seen: std::sync::Mutex<Vec<u64>>,
}
impl<'a, E: zluda_common::CudaErrorType> zluda_common::FromCuda<'a, CUfunction, E>
@@ -140,7 +145,7 @@ pub(crate) fn launch_kernel(
translated_extra_storage.as_mut_ptr()
};
// the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
- let prep_key = if f.prep.is_some() && f.prep_key != 0 {
+ let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
let mut block: *const u8 = std::ptr::null();
if !extra.is_null() {
let mut i = 0usize;
@@ -160,12 +165,27 @@ pub(crate) fn launch_kernel(
if block.is_null() {
0
} else {
- unsafe { std::ptr::read_unaligned(block.add(f.prep_key as usize) as *const u64) }
+ unsafe { std::ptr::read_unaligned(block.add(key_offset as usize) as *const u64) }
}
} else {
0
};
- let run_prep = prep_key == 0 || f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key;
+ let run_prep = if prep_key == 0 {
+ true
+ } else if f.prep_slots == 0 {
+ f.prep_last.load(std::sync::atomic::Ordering::Relaxed) != prep_key
+ } else {
+ // prepared once per key while slots last; keys beyond them are prepared on every launch
+ let mut seen = f.prep_seen.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+ if seen.contains(&prep_key) {
+ false
+ } else {
+ if seen.len() < f.prep_slots as usize {
+ seen.push(prep_key);
+ }
+ true
+ }
+ };
if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
unsafe {
hipModuleLaunchKernel(
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 5db5ace..c5a39c9 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -37,12 +37,12 @@ impl ModuleMutable {
Ok(match self.functions.entry(name) {
hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
hash_map::Entry::Vacant(entry) => {
- let (func_handle, prep, block_z, grid_x, prep_key) = match native_override(entry.key()) {
+ let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
Some(native) => native,
None => {
let mut func_handle = unsafe { std::mem::zeroed() };
unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
- (func_handle, None, 0, 0, 0)
+ (func_handle, None, 0, 0, None, 0)
}
};
let func = Box::new(Function {
@@ -54,6 +54,8 @@ impl ModuleMutable {
grid_x,
prep_key,
prep_last: std::sync::atomic::AtomicU64::new(0),
+ prep_slots,
+ prep_seen: std::sync::Mutex::new(Vec::new()),
});
&*entry.insert(func)
}
@@ -67,7 +69,9 @@ impl ModuleMutable {
/// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
/// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
/// (persistent kernels, which derive the original grid from their parameters) the grid.
-fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, u32)> {
+fn native_override(
+ name: &CStr,
+) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
let image = fs::read(&path).ok()?;
@@ -103,34 +107,38 @@ fn native_override(name: &CStr) -> Option<(hipFunction_t, Option<(hipFunction_t,
} else {
None
};
- let read_u32 = |symbol: &CStr| -> u32 {
+ let read_u32_opt = |symbol: &CStr| -> Option<u32> {
let mut ptr = unsafe { mem::zeroed() };
let mut size = 0usize;
let mut value = 0u32;
- if unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
+ (unsafe { hipModuleGetGlobal(&mut ptr, &mut size, native_module, symbol.as_ptr()) }.is_ok()
&& size == 4
- && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok()
- {
- value
- } else {
- 0
- }
+ && unsafe { hipMemcpyDtoH((&mut value as *mut u32).cast(), ptr, 4) }.is_ok())
+ .then_some(value)
};
+ let read_u32 = |symbol: &CStr| -> u32 { read_u32_opt(symbol).unwrap_or(0) };
let block_z = read_u32(c"d4r_block_z");
let grid_x = read_u32(c"d4r_grid_x");
// byte offset of a u64 in the parameter block that fully determines the prep kernel's output
- // (the weights pointer): the prep is skipped while it repeats (0 = prep before every launch)
- let prep_key = read_u32(c"d4r_prep_key_offset");
+ // (the weights pointer): the prep is skipped while it repeats. d4r_prep_key_at holds offset + 1
+ // (so a key at offset 0 is possible); the older d4r_prep_key_offset uses 0 for "no key".
+ let prep_key = match read_u32_opt(c"d4r_prep_key_at") {
+ Some(at) if at > 0 => Some(at - 1),
+ _ => Some(read_u32(c"d4r_prep_key_offset")).filter(|&offset| offset != 0),
+ };
+ // d4r_prep_key_slots: the kernel keeps one prepared image per key (see Function::prep_slots)
+ let prep_slots = read_u32(c"d4r_prep_key_slots");
eprintln!(
- "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {})",
+ "[d4r-native] {:?} -> {} (prep: {:?}, block_z: {}, grid_x: {}, prep_key: {:?}, prep_slots: {})",
name,
path.display(),
prep.map(|p| p.1),
block_z,
grid_x,
- prep_key
+ prep_key,
+ prep_slots
);
- Some((func, prep, block_z, grid_x, prep_key))
+ Some((func, prep, block_z, grid_x, prep_key, prep_slots))
}
impl ZludaObject for Module {
patches/zluda/0007-gfx12-wmma-native-fp8.patch
-551
View File
@@ -1,551 +0,0 @@
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -113,9 +113,57 @@
*VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_PARAM_PROVENANCE").map_or(true, |value| value.trim() != "0"))
}
-fn wmma_rewrite_enabled(gcn_arch: &str) -> bool {
- // The helpers assume RDNA3's wave32 WMMA register layout.
- wmma_rewrite_requested() && gcn_arch.starts_with("gfx11")
+/// D4R_ZLUDA_WMMA_FP8_NATIVE=1: on gfx12 (RDNA4), e4m3 MMAs use the native FP8 WMMA instead of widening
+/// to f16. Ignored on every other GPU (gfx11 has no FP8 WMMA).
+pub fn wmma_fp8_native_requested() -> bool {
+ static VALUE: OnceLock<bool> = OnceLock::new();
+ *VALUE.get_or_init(|| std::env::var("D4R_ZLUDA_WMMA_FP8_NATIVE").map_or(false, |value| value.trim() == "1"))
+}
+
+/// The WMMA register layout family of a GPU: 11 (RDNA3) or 12 (RDNA4); the MMA helpers exist for these.
+/// D4R_ZLUDA_WMMA_LAYOUT=12 on gfx11 is for tests only: it takes the gfx12 helper names, which then must
+/// come from a helper library built with D4R_WMMA12_SHIM (gfx12 data movement, gfx11 WMMAs).
+fn wmma_family(gcn_arch: &str) -> Option<u32> {
+ let family = if gcn_arch.starts_with("gfx11") {
+ 11
+ } else if gcn_arch.starts_with("gfx12") {
+ 12
+ } else {
+ return None;
+ };
+ if family == 11 && std::env::var("D4R_ZLUDA_WMMA_LAYOUT").map_or(false, |value| value.trim() == "12") {
+ return Some(12);
+ }
+ Some(family)
+}
+
+/// Whether e4m3 MMAs compiled for this GPU use native FP8 WMMA (the setting only acts on gfx12).
+pub fn wmma_fp8_native_effective(gcn_arch: &str) -> bool {
+ wmma_rewrite_flags(gcn_arch).map_or(false, |flags| flags & 4 != 0)
+}
+
+/// Flags for LLVMZludaRewriteMmaF16Wmma, or None to keep the dot-product fallback (logged once when the
+/// rewrite was requested but this GPU has no WMMA lowering).
+fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
+ if !wmma_rewrite_requested() {
+ return None;
+ }
+ let Some(family) = wmma_family(gcn_arch) else {
+ static LOGGED: OnceLock<()> = OnceLock::new();
+ LOGGED.get_or_init(|| {
+ eprintln!("[ZLUDA] D4R_ZLUDA_WMMA: no WMMA lowering for {gcn_arch}; MMAs use the dot-product fallback")
+ });
+ return None;
+ };
+ let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+ if family == 12 {
+ flags |= 2;
+ // native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
+ if flags & 1 != 0 && wmma_fp8_native_requested() {
+ flags |= 4;
+ }
+ }
+ Some(flags)
}
pub fn compile(
@@ -139,8 +187,7 @@
// wmma.mma m16n16k16 always becomes m16n8k16 MMAs (there is no helper for it).
unsafe { crate::ffi::LLVMZludaSplitWmmaM16n16k16(main.get()) };
- if wmma_rewrite_enabled(gcn_arch) {
- let flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
+ if let Some(flags) = wmma_rewrite_flags(gcn_arch) {
unsafe { crate::ffi::LLVMZludaRewriteMmaF16Wmma(main.get(), flags) };
}
linked.link(main)?;
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -968,6 +968,9 @@
}
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
+// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
+// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
+// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
Module *M = unwrap(ModuleRef);
@@ -990,33 +993,39 @@
Type *V8 = FixedVectorType::get(I32, 8);
AttributeList Attributes = AttributeList::get(Context, AttributeList::FunctionIndex,
AttrBuilder(Context, Reference->getAttributes().getFnAttrs()));
+ // arch-specific helpers (operand gathers, WMMAs), e4m3 MMAs, e4m3 fragment widening; the conversions
+ // between NVIDIA fragments, the transposed layout and f32 shadows are shared
+ const bool Gfx12 = (Flags & 2) != 0, NativeFp8 = Gfx12 && (Flags & 4) != 0;
+ const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
+ const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
+ const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
Helpers H;
- H.AFromB = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b", Attributes, V8, V2);
- H.BFromA = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_b_from_a", Attributes, V8, V4);
- H.Mma = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16", Attributes, V2, V8, V8, V2);
+ H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
+ H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
+ H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
- H.AFromBPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_a_from_b_pair", Attributes, V8, V2, V2);
- H.MmaPair = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_f16_pair", Attributes, V4, V8, V8, V2, V2);
+ H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
{
Type *F = Type::getFloatTy(Context);
Type *V4F = FixedVectorType::get(F, 4);
Type *V8F = FixedVectorType::get(F, 8);
H.SFromT = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_from_t", Attributes, V4F, V2, F);
H.TFromS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_s", Attributes, V2, V4F, F);
- H.MmaS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16", Attributes, V4F, V8, V8, V4F, F, F);
- H.MmaPairS = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_f16_pair", Attributes, V8F, V8, V8, V4F, V4F, F, F);
- H.MmaE4m3S = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3", Attributes, V4F, V8, V8, V8, V8, V4F);
+ H.MmaS = M->getOrInsertFunction((Arch + "s_mma_f16").c_str(), Attributes, V4F, V8, V8, V4F, F, F);
+ H.MmaPairS = M->getOrInsertFunction((Arch + "s_mma_f16_pair").c_str(), Attributes, V8F, V8, V8, V4F, V4F, F, F);
+ H.MmaE4m3S = M->getOrInsertFunction((E4m3 + "s_mma_e4m3").c_str(), Attributes, V4F, V8, V8, V8, V8, V4F);
H.MmaE4m3PairS =
- M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_s_mma_e4m3_pair", Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
+ M->getOrInsertFunction((E4m3 + "s_mma_e4m3_pair").c_str(), Attributes, V8F, V8, V8, V8, V8, V4F, V4F);
}
if (MmaE4m3 != nullptr)
{
- H.E4m3AToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_a_to_f16", Attributes, V8, V4);
- H.E4m3BToF16 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_e4m3_b_to_f16", Attributes, V4, V2);
- H.MmaE4m3 = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3", Attributes, V2, V8, V8, V8, V8, V2);
+ H.E4m3AToF16 = M->getOrInsertFunction((Widen + "e4m3_a_to_f16").c_str(), Attributes, V8, V4);
+ H.E4m3BToF16 = M->getOrInsertFunction((Widen + "e4m3_b_to_f16").c_str(), Attributes, V4, V2);
+ H.MmaE4m3 = M->getOrInsertFunction((E4m3 + "t_mma_e4m3").c_str(), Attributes, V2, V8, V8, V8, V8, V2);
H.MmaE4m3Pair =
- M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_mma_e4m3_pair", Attributes, V4, V8, V8, V8, V8, V2, V2);
+ M->getOrInsertFunction((E4m3 + "t_mma_e4m3_pair").c_str(), Attributes, V4, V8, V8, V8, V8, V2, V2);
}
unsigned Rewritten = 0;
for (Function &F : *M)
--- a/ptx/examples/d4r_emit.rs
+++ b/ptx/examples/d4r_emit.rs
@@ -1,9 +1,13 @@
use std::{env, fs, path::PathBuf};
use ptx_parser::parse_module_checked;
+// usage: d4r_emit PTX OUT_DIR [GPU_ARCH]: translates and compiles a PTX module offline (no GPU needed) for
+// GPU_ARCH (default gfx1101), with the same D4R_ZLUDA_* code-generation switches as the runtime, and writes
+// the IR stages, asm.s and module.hsaco to OUT_DIR
fn main() {
let input = env::args().nth(1).expect("PTX path");
let outdir = PathBuf::from(env::args().nth(2).unwrap_or_else(|| "/tmp/d4r-autoexposure-emitted".into()));
+ let arch = env::args().nth(3).unwrap_or_else(|| "gfx1101".into());
fs::create_dir_all(&outdir).expect("create output directory");
let bytes = fs::read(&input).expect("read PTX");
let text = std::str::from_utf8(&bytes).expect("PTX UTF-8").trim_end_matches('\0');
@@ -18,13 +22,14 @@
"opt.ll" => "opt.ll",
"asm" => "asm.s",
"o" => "object.o",
+ "elf" => "linked.elf", // the code object compile() returns (written again as module.hsaco)
other => panic!("unexpected stage: {other}"),
};
fs::write(outdir.join(filename), data).expect("write compiler stage");
};
let elf = llvm_zluda::compile(
&module.context,
- "gfx1101",
+ &arch,
module.llvm_ir,
linked_helpers,
module.attributes_ir,
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1696,6 +1696,349 @@
}
}
+// gfx12 (RDNA4) lowering of the same MMAs (LLVMZludaRewriteMmaF16Wmma with the gfx12 flag: helpers
+// d4r_wmma12_*, and d4r_wmma12f8_* for the e4m3 ones with native FP8 WMMA). The contract is the gfx11 one
+// above: NVIDIA fragments in, the transposed accumulator layout (lane l holds D[l % 16][2v + l / 16] as two
+// f16x2 words, or four floats for the f32 shadows), so the rewrite only switches helper names.
+//
+// A gfx12 WMMA operand holds 8 of the 16 K values per lane: half h = lane / 16 its own 8 (4 dwords, returned
+// in the low half of a wmma_uint8). With A' = B^T and B' = A^T as above, dword j of half h holds the k-pair of
+// NVIDIA thread t = 2h + (j >> 1): the low pair (k 2t, 2t + 1) for even j, the high pair (k 2t + 8, 2t + 9)
+// for odd j. Then every lane of a gather reads the same source register (4 bpermutes per operand; B' also
+// needs a DPP swap per dword for rows 8..15). For e4m3 fragments the same slots hold, after the widening
+// above, e4m3 k 8h .. 8h + 7 in order.
+// The result D' has VGPR i = row n = i + 8h (columns m = lane % 16): unpaired, rows 0..7 are the h = 0 lanes
+// and the h = 1 lanes are padding; paired (A' rows 8..15 = B1^T), D1 is in the h = 1 lanes. Converting from
+// and to the transposed layout takes 2 permlanex16 each way (4 for f32 shadows).
+//
+// D4R_WMMA12_SHIM (tests only): these helpers compiled for gfx11, every gfx12 WMMA done by the gfx11 one on
+// operands rebuilt in the gfx11 helpers' K order (and for e4m3 with the same scaled widening), so the results
+// are bit-identical to the gfx11 helpers and the gfx12 data movement can be checked on RDNA3.
+typedef _Float16 wmma_half8 __attribute__((ext_vector_type(8)));
+typedef uint32_t wmma_uint4 __attribute__((ext_vector_type(4)));
+typedef int wmma_int2 __attribute__((ext_vector_type(2)));
+#ifndef D4R_WMMA12_SHIM
+__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_f16(wmma_half8 a, wmma_half8 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.f16.v8f32.v8f16");
+__device__ wmma_float8 __zluda_wmma12_f32_16x16x16_fp8_fp8(wmma_int2 a, wmma_int2 b, wmma_float8 c) __asm("llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32");
+#endif
+
+__device__ static uint32_t wmma12_half()
+{
+ return wmma_lane() / 16;
+}
+
+__device__ static uint32_t wmma12_other_half(uint32_t value)
+{
+ return uint32_t(__builtin_amdgcn_permlanex16(int(value), int(value), 0x76543210, 0xfedcba98, false, false));
+}
+
+__device__ static float wmma12_other_half_f(float value)
+{
+ return std::bit_cast<float>(wmma12_other_half(std::bit_cast<uint32_t>(value)));
+}
+
+// dword j of an operand gathered from two candidate registers: rows r < 8 take `low` from lane 4r + t, rows
+// r >= 8 take `high` from lane 4(r - 8) + t (t = 2h + (j >> 1)). The source lane is in the first half-wave,
+// while t differs between the two destination half-waves, so the two candidates must be gathered independently.
+__device__ static uint32_t wmma12_gather2(uint32_t low, uint32_t high, uint32_t j)
+{
+ const uint32_t row = wmma_lane() % 16;
+ const uint32_t t = (j >> 1) + 2 * wmma12_half();
+ const uint32_t lo = wmma_from_lane(row * 4 + t, low);
+ const uint32_t hi = wmma_from_lane((row % 8) * 4 + t, high);
+ return row < 8 ? lo : hi;
+}
+
+__device__ static wmma_uint4 wmma12_low4(wmma_uint8 v)
+{
+ return {v[0], v[1], v[2], v[3]};
+}
+
+// f32 D' = A' B' + C' (f16 operands in the slot order above)
+__device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+{
+#ifndef D4R_WMMA12_SHIM
+ return __zluda_wmma12_f32_16x16x16_f16(std::bit_cast<wmma_half8>(wmma12_low4(a)), std::bit_cast<wmma_half8>(wmma12_low4(b)), c);
+#else
+ // gfx11 operand dword p (K 2p, 2p + 1 in the gfx11 helpers' order) is gfx12 dword jp of half hp
+ const bool high = wmma12_half() != 0;
+ wmma_uint8 a11, b11;
+ for (uint32_t p = 0; p < 8; p++)
+ {
+ const uint32_t hp = (p & 3) >> 1, jp = 2 * (p & 1) + (p >> 2);
+ const uint32_t ao = wmma12_other_half(a[jp]), bo = wmma12_other_half(b[jp]);
+ a11[p] = (hp != 0) == high ? a[jp] : ao;
+ b11[p] = (hp != 0) == high ? b[jp] : bo;
+ }
+ // accumulators: gfx12 VGPR i = row i + 8h, gfx11 VGPR i = row 2i + h
+ wmma_float8 co, c11;
+ for (uint32_t i = 0; i < 8; i++)
+ co[i] = wmma12_other_half_f(c[i]);
+ for (uint32_t i = 0; i < 8; i++)
+ c11[i] = high ? (i < 4 ? co[2 * i + 1] : c[2 * i - 7]) : (i < 4 ? c[2 * i] : co[2 * i - 8]);
+ wmma_float8 d11 = __zluda_wmma_f32_16x16x16_f16(std::bit_cast<wmma_half16>(a11), std::bit_cast<wmma_half16>(b11), c11);
+ wmma_float8 dout, d;
+ for (uint32_t i = 0; i < 8; i++)
+ dout[i] = wmma12_other_half_f(d11[i]);
+ for (uint32_t i = 0; i < 8; i++)
+ d[i] = high ? ((i & 1) ? d11[(i >> 1) + 4] : dout[(i >> 1) + 4]) : ((i & 1) ? dout[i >> 1] : d11[i >> 1]);
+ return d;
+#endif
+}
+
+// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
+// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
+__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
+{
+ const wmma_int2 a8 = {int(__builtin_amdgcn_perm(a[1], a[0], 0x05040100u)), int(__builtin_amdgcn_perm(a[3], a[2], 0x05040100u))};
+ const wmma_int2 b8 = {int(__builtin_amdgcn_perm(b[1], b[0], 0x05040100u)), int(__builtin_amdgcn_perm(b[3], b[2], 0x05040100u))};
+#ifndef D4R_WMMA12_SHIM
+ return __zluda_wmma12_f32_16x16x16_fp8_fp8(a8, b8, c);
+#else
+ // the gfx11 helpers' scaled widening (value * 2^-8 per operand, C * 2^-16): exact powers of two
+ auto widen = [](wmma_int2 q) {
+ auto lo = e4m3x4_to_scaled_f16x2x2(uint32_t(q.x)), hi = e4m3x4_to_scaled_f16x2x2(uint32_t(q.y));
+ return wmma_uint8{lo.x, lo.y, hi.x, hi.y, 0u, 0u, 0u, 0u};
+ };
+ return wmma12_f16(widen(a8), widen(b8), c * 0x1p-16f) * 0x1p16f;
+#endif
+}
+
+// transposed f16 accumulator(s) <-> D' / C'
+__device__ static wmma_float8 wmma12_c_from_t(uint2::Native_vec_ c_t)
+{
+ // lane h = 0 needs rows 0..7: its own rows 0, 2, 4, 6 and the other half's 1, 3, 5, 7 (h = 1 lanes: padding)
+ const uint32_t o0 = wmma12_other_half(c_t.x), o1 = wmma12_other_half(c_t.y);
+ f16x2 e01 = std::bit_cast<f16x2>(c_t.x), e23 = std::bit_cast<f16x2>(c_t.y);
+ f16x2 d01 = std::bit_cast<f16x2>(o0), d23 = std::bit_cast<f16x2>(o1);
+ return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
+}
+
+__device__ static wmma_float8 wmma12_c_from_t_pair(uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+{
+ // h = 0: C0 rows 0..7 (own even, other odd); h = 1: C1 rows 0..7 (other even, own odd)
+ const bool high = wmma12_half() != 0;
+ const uint2::Native_vec_ own = high ? c1_t : c0_t, send = high ? c0_t : c1_t;
+ const uint32_t r0 = wmma12_other_half(send.x), r1 = wmma12_other_half(send.y);
+ f16x2 o01 = std::bit_cast<f16x2>(own.x), o23 = std::bit_cast<f16x2>(own.y);
+ f16x2 p01 = std::bit_cast<f16x2>(r0), p23 = std::bit_cast<f16x2>(r1);
+ f16x2 e01 = high ? p01 : o01, e23 = high ? p23 : o23, d01 = high ? o01 : p01, d23 = high ? o23 : p23;
+ return {float(e01.x), float(d01.x), float(e01.y), float(d01.y), float(e23.x), float(d23.x), float(e23.y), float(d23.y)};
+}
+
+__device__ static void wmma12_rows_f16(wmma_float8 d, uint2::Native_vec_ &even, uint2::Native_vec_ &odd)
+{
+ f16x2 e01{static_cast<f16>(d[0]), static_cast<f16>(d[2])}, e23{static_cast<f16>(d[4]), static_cast<f16>(d[6])};
+ f16x2 o01{static_cast<f16>(d[1]), static_cast<f16>(d[3])}, o23{static_cast<f16>(d[5]), static_cast<f16>(d[7])};
+ even = {std::bit_cast<uint32_t>(e01), std::bit_cast<uint32_t>(e23)};
+ odd = {std::bit_cast<uint32_t>(o01), std::bit_cast<uint32_t>(o23)};
+}
+
+__device__ static uint2::Native_vec_ wmma12_t_from_d(wmma_float8 d)
+{
+ // h = 0 keeps rows 0, 2, 4, 6; h = 1 takes rows 1, 3, 5, 7 from the other half
+ uint2::Native_vec_ even, odd;
+ wmma12_rows_f16(d, even, odd);
+ const uint2::Native_vec_ other = {wmma12_other_half(odd.x), wmma12_other_half(odd.y)};
+ return wmma12_half() != 0 ? other : even;
+}
+
+__device__ static uint4::Native_vec_ wmma12_t_from_d_pair(wmma_float8 d)
+{
+ // h = 0 holds D0 rows 0..7, h = 1 D1 rows 0..7; lane h wants rows 2v + h of both
+ const bool high = wmma12_half() != 0;
+ uint2::Native_vec_ even, odd;
+ wmma12_rows_f16(d, even, odd);
+ const uint2::Native_vec_ send = high ? even : odd;
+ const uint2::Native_vec_ recv = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
+ return high ? uint4::Native_vec_{recv.x, recv.y, odd.x, odd.y} : uint4::Native_vec_{even.x, even.y, recv.x, recv.y};
+}
+
+// the same for f32 shadows (element v = row 2v + h)
+__device__ static wmma_float8 wmma12_c_from_s(wmma_float4 c)
+{
+ wmma_float4 o;
+ for (uint32_t v = 0; v < 4; v++)
+ o[v] = wmma12_other_half_f(c[v]);
+ return {c[0], o[0], c[1], o[1], c[2], o[2], c[3], o[3]};
+}
+
+__device__ static wmma_float8 wmma12_c_from_s_pair(wmma_float4 c0, wmma_float4 c1)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float8 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float own = high ? c1[v] : c0[v], recv = wmma12_other_half_f(high ? c0[v] : c1[v]);
+ r[2 * v] = high ? recv : own;
+ r[2 * v + 1] = high ? own : recv;
+ }
+ return r;
+}
+
+__device__ static wmma_float4 wmma12_s_from_d(wmma_float8 d)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float4 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float odd = wmma12_other_half_f(d[2 * v + 1]);
+ r[v] = high ? odd : d[2 * v];
+ }
+ return r;
+}
+
+__device__ static wmma_float8 wmma12_s_from_d_pair(wmma_float8 d)
+{
+ const bool high = wmma12_half() != 0;
+ wmma_float8 r;
+ for (uint32_t v = 0; v < 4; v++)
+ {
+ const float recv = wmma12_other_half_f(high ? d[2 * v] : d[2 * v + 1]);
+ r[v] = high ? recv : d[2 * v];
+ r[4 + v] = high ? d[2 * v + 1] : recv;
+ }
+ return r;
+}
+
+extern "C"
+{
+ wmma_uint8 FUNC(d4r_wmma12_t_a_from_b)(uint2::Native_vec_ b)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ uint32_t n = wmma_lane() % 16;
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ {
+ uint32_t value = wmma_from_lane((n % 8) * 4 + (j >> 1) + 2 * wmma12_half(), (j & 1) ? b.y : b.x);
+ result[j] = n < 8 ? value : 0u;
+ }
+ return result;
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12_t_b_from_a)(uint4::Native_vec_ a)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ result[j] = wmma12_gather2((j & 1) ? a.z : a.x, (j & 1) ? a.w : a.y, j);
+ return result;
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12_t_a_from_b_pair)(uint2::Native_vec_ b0, uint2::Native_vec_ b1)
+ {
+ __builtin_amdgcn_sched_barrier(0);
+ wmma_uint8 result = {};
+ for (uint32_t j = 0; j < 4; j++)
+ result[j] = wmma12_gather2((j & 1) ? b0.y : b0.x, (j & 1) ? b1.y : b1.x, j);
+ return result;
+ }
+
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ return wmma12_t_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_t(c_t)));
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
+ }
+
+ wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
+ {
+ return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
+ }
+
+ wmma_float8 FUNC(d4r_wmma12_s_mma_f16_pair)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c0, wmma_float4 c1, float up,
+ float down)
+ {
+ return wmma12_s_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_s_pair(c0 * up, c1 * up))) * down;
+ }
+
+ // e4m3 with the f16 widening (d4r_wmma_e4m3_a_to_f16 / b_to_f16 above): two chained f16 WMMAs in the
+ // 2^-16 scaled domain, like the gfx11 helpers
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
+ {
+ wmma_float8 c = wmma12_c_from_t(c_t) * 0x1p-16f;
+ c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
+ return wmma12_t_from_d(c * 0x1p16f);
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
+ uint2::Native_vec_ c1_t)
+ {
+ wmma_float8 c = wmma12_c_from_t_pair(c0_t, c1_t) * 0x1p-16f;
+ c = wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, c));
+ return wmma12_t_from_d_pair(c * 0x1p16f);
+ }
+
+ wmma_float4 FUNC(d4r_wmma12_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c)
+ {
+ return wmma12_s_from_d(wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s(c))));
+ }
+
+ wmma_float8 FUNC(d4r_wmma12_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
+ {
+ return wmma12_s_from_d_pair(
+ wmma12_f16(a_t_high, b_t_high, wmma12_f16(a_t_low, b_t_low, wmma12_c_from_s_pair(c0, c1))));
+ }
+
+ // Native FP8 (d4r_wmma12f8_*, D4R_ZLUDA_WMMA_FP8_NATIVE): the "widening" keeps the e4m3 bytes, two per
+ // dword in the low 16 bits and in the same slots the f16 widening fills, so the f16 gathers above move them;
+ // the MMA packs this half's 8 bytes (e4m3 k 8h .. 8h + 7 in order) into the FP8 operand. OCP e4m3 (gfx12)
+ // is NVIDIA's e4m3, so the bytes need no conversion. Shadows keep the 2^-16 scaled domain of the e4m3
+ // functions (exact power-of-two rescaling around the WMMA).
+ __device__ static uint2::Native_vec_ e4m3x4_split(uint32_t packed)
+ {
+ return {__builtin_amdgcn_perm(0u, packed, 0x0c0c0100u), __builtin_amdgcn_perm(0u, packed, 0x0c0c0302u)};
+ }
+
+ wmma_uint8 FUNC(d4r_wmma12f8_e4m3_a_to_f16)(uint4::Native_vec_ a)
+ {
+ auto x = e4m3x4_split(a.x), y = e4m3x4_split(a.y), z = e4m3x4_split(a.z), w = e4m3x4_split(a.w);
+ return {x.x, y.x, x.y, y.y, z.x, w.x, z.y, w.y};
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12f8_e4m3_b_to_f16)(uint2::Native_vec_ b)
+ {
+ auto x = e4m3x4_split(b.x), y = e4m3x4_split(b.y);
+ return {x.x, x.y, y.x, y.y};
+ }
+
+ uint2::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c_t)
+ {
+ return wmma12_t_from_d(wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t(c_t))));
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12f8_t_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, uint2::Native_vec_ c0_t,
+ uint2::Native_vec_ c1_t)
+ {
+ return wmma12_t_from_d_pair(
+ wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_t_pair(c0_t, c1_t))));
+ }
+
+ wmma_float4 FUNC(d4r_wmma12f8_s_mma_e4m3)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c)
+ {
+ wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high, wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s(c * 0x1p16f)));
+ return wmma12_s_from_d(d) * 0x1p-16f;
+ }
+
+ wmma_float8 FUNC(d4r_wmma12f8_s_mma_e4m3_pair)(wmma_uint8 a_t_low, wmma_uint8 b_t_low, wmma_uint8 a_t_high,
+ wmma_uint8 b_t_high, wmma_float4 c0, wmma_float4 c1)
+ {
+ wmma_float8 d = wmma12_e4m3(a_t_high, b_t_high,
+ wmma12_e4m3(a_t_low, b_t_low, wmma12_c_from_s_pair(c0 * 0x1p16f, c1 * 0x1p16f)));
+ return wmma12_s_from_d_pair(d) * 0x1p-16f;
+ }
+}
+
template <typename T, typename A, typename B, int BOffset, typename OcklFunc>
__device__ T dp2a_impl(T a, T b, T c, OcklFunc ockl_func)
{
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -356,6 +356,7 @@
wmma: llvm_zluda::compile::wmma_rewrite_requested(),
wmma_fp8: llvm_zluda::compile::wmma_fp8_rewrite_requested(),
param_provenance: llvm_zluda::compile::param_provenance_enabled(),
+ wmma_fp8_native: llvm_zluda::compile::wmma_fp8_native_effective(gcn_arch),
};
let mut cache_with_key = match (text, global_state.cache_path.as_ref()) {
(Some(text), Some(p)) => (|| {
@@ -397,6 +398,10 @@
wmma: bool,
wmma_fp8: bool,
param_provenance: bool,
+ // d4r: native FP8 WMMA where it takes effect (gfx12); absent from the key everywhere else, so the setting
+ // (d4r.ini NativeFp8) never forces a recompile on GPUs that ignore it
+ #[serde(skip_serializing_if = "std::ops::Not::not")]
+ wmma_fp8_native: bool,
}
fn get_hip_properties<'a>() -> Result<hipDeviceProp_tR0600, CUerror> {
@@ -430,6 +435,7 @@
&& !matches!(
key.as_str(),
"D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
+ | "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
.map(|(key, value)| format!("{key}={}", value.trim()))
patches/zluda/0008-native-windows-build.patch
-203
View File
@@ -1,203 +0,0 @@
Windows LLVM-MinGW build, portable helper assertions and optional diagnostic UI.
Apply after upstream d4r patches 0002 through 0007, on ZLUDA ee2f25a.
diff --git a/ext/highs-sys/build.rs b/ext/highs-sys/build.rs
--- a/ext/highs-sys/build.rs
+++ b/ext/highs-sys/build.rs
@@ -66,11 +66,12 @@
let linux = target.contains("linux");
let mingw = target.contains("pc-windows-gnu");
if apple {
println!("cargo:rustc-link-lib=dylib=c++");
} else if linux || mingw {
- println!("cargo:rustc-link-lib=dylib=stdc++");
+ let cxx_library = env::var("CXXSTDLIB").unwrap_or_else(|_| "stdc++".to_owned());
+ println!("cargo:rustc-link-lib=dylib={cxx_library}");
}
println!("cargo:rerun-if-changed=../HiGHS/highs/interfaces/highs_c_api.h");
true
}
diff --git a/ext/hip_runtime-sys/build.rs b/ext/hip_runtime-sys/build.rs
--- a/ext/hip_runtime-sys/build.rs
+++ b/ext/hip_runtime-sys/build.rs
@@ -1,8 +1,12 @@
fn main() {
if cfg!(windows) {
- println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
+ println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
+ } else {
+ println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ }
} else {
println!("cargo:rustc-link-lib=dylib=amdhip64");
println!("cargo:rustc-link-search=native=/opt/rocm/lib/");
}
}
diff --git a/llvm_zluda/build.rs b/llvm_zluda/build.rs
--- a/llvm_zluda/build.rs
+++ b/llvm_zluda/build.rs
@@ -19,10 +19,18 @@
fn main() {
let bc_path1 = "src/device-libs/ockl.bc";
println!("cargo:rerun-if-changed={}", bc_path1);
check_lfs_file(bc_path1);
+
+ println!("cargo:rerun-if-env-changed=ZLUDA_LLVM_BUILD_DIR");
+ if let Some(directory) = env::var_os("ZLUDA_LLVM_BUILD_DIR") {
+ let llvm_build_path = PathBuf::from(directory);
+ println!("cargo:build-path={}", llvm_build_path.display());
+ link_prebuilt_llvm(&llvm_build_path);
+ return;
+ }
let mut cmake = Config::new(r"../ext/llvm-project/llvm");
try_use_sccache(&mut cmake);
try_use_ninja(&mut cmake);
cmake
@@ -57,16 +65,31 @@
cmake.build();
}
let cmake_profile = cmake.get_profile();
let llvm_build_path = get_llvm_build_path(&llvm_dir, cmake_profile);
println!("cargo:build-path={}", llvm_build_path.display());
+ link_prebuilt_llvm(&llvm_build_path);
+}
+
+fn link_prebuilt_llvm(llvm_build_path: &PathBuf) {
let (cxxflags, ldflags, libdir, lib_names, system_libs) =
llvm_config(&llvm_build_path.join("bin/llvm-config")).unwrap();
println!("cargo:rustc-link-arg={ldflags}");
println!("cargo:rustc-link-search=native={libdir}");
for lib in system_libs.split_ascii_whitespace() {
- println!("cargo:rustc-link-arg={lib}");
+ if let Some(name) = lib.strip_prefix("-l") {
+ println!("cargo:rustc-link-lib={name}");
+ } else {
+ println!("cargo:rustc-link-arg={lib}");
+ }
+ }
+ // llvm-config omits system libraries on Windows. Propagate these through
+ // this rlib to its final cdylib consumer (link-arg alone cannot do that).
+ if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") {
+ for library in ["psapi", "shell32", "ole32", "uuid", "advapi32", "ws2_32", "ntdll"] {
+ println!("cargo:rustc-link-lib={library}");
+ }
}
link_lld_components();
link_llvm_components(lib_names);
compile_cxx_lib(cxxflags, &llvm_build_path);
}
diff --git a/zluda/build.rs b/zluda/build.rs
--- a/zluda/build.rs
+++ b/zluda/build.rs
@@ -1,11 +1,15 @@
use vergen_gix::{Emitter, Gix};
fn main() {
if cfg!(windows) {
- println!("cargo:rustc-link-arg=delayimp.lib");
- println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("gnu") {
+ println!("cargo:rustc-link-arg=-Wl,--delayload=amdhip64_7.dll");
+ } else {
+ println!("cargo:rustc-link-arg=delayimp.lib");
+ println!("cargo:rustc-link-arg=/DELAYLOAD:amdhip64_7.dll");
+ }
let dll_path = "bin/nvcudart_hybrid64.dll";
println!("cargo:rerun-if-changed={}", dll_path);
check_lfs_file(dll_path);
}
let git = Gix::builder().sha(false).build();
diff --git a/zluda_windows/src/lib.rs b/zluda_windows/src/lib.rs
--- a/zluda_windows/src/lib.rs
+++ b/zluda_windows/src/lib.rs
@@ -11,27 +11,27 @@
use uuid::uuid;
use widestring::{u16str, U16Str, U16String};
use windows::{
core::{w, HRESULT, PCWSTR},
Win32::{
- Foundation::{HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
+ Foundation::{FreeLibrary, HANDLE, HINSTANCE, HMODULE, HWND, LPARAM, S_OK, WPARAM},
System::{
JobObjects::{
AssignProcessToJobObject, JobObjectExtendedLimitInformation,
SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
},
LibraryLoader::{
- GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, LoadLibraryExW,
+ GetModuleFileNameA, GetModuleFileNameW, GetModuleHandleExW, GetProcAddress, LoadLibraryExW,
GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS,
- LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR,
+ LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR, LOAD_LIBRARY_SEARCH_SYSTEM32,
},
},
UI::{
Controls::{
- TaskDialogIndirect, TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
+ TASKDIALOGCONFIG, TASKDIALOGCONFIG_0, TASKDIALOGCONFIG_1,
TASKDIALOG_NOTIFICATIONS, TDCBF_CLOSE_BUTTON, TDF_ENABLE_HYPERLINKS,
TDN_HYPERLINK_CLICKED, TD_ERROR_ICON,
},
WindowsAndMessaging::IDCLOSE,
},
@@ -487,11 +487,23 @@
pszFooter: PCWSTR::null(),
pfCallback: Some(task_dialog_callback),
lpCallbackData: 0,
cxWidth: 0,
};
- TaskDialogIndirect(&config, None, None, None).ok();
+ // A CUDA DLL may be loaded by an executable with no Common Controls
+ // v6 activation manifest. Do not make its diagnostic UI an eager DLL
+ // import: comctl32 v5 has no TaskDialogIndirect export.
+ if let Ok(controls) = LoadLibraryExW(w!("comctl32.dll"), None, LOAD_LIBRARY_SEARCH_SYSTEM32) {
+ if let Some(raw) = GetProcAddress(controls, windows::core::s!("TaskDialogIndirect")) {
+ let show: unsafe extern "system" fn(*const TASKDIALOGCONFIG, *mut i32, *mut i32, *mut i32) -> HRESULT =
+ mem::transmute(raw);
+ let _ = show(&config, ptr::null_mut(), ptr::null_mut(), ptr::null_mut());
+ } else {
+ eprintln!("ZLUDA failed to load {}; set HIP_PATH to the selected HIP SDK", redirect_names[0]);
+ }
+ let _ = FreeLibrary(controls);
+ }
}
result
}
unsafe extern "system" fn task_dialog_callback(
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -405,11 +405,26 @@
uint32_t line,
uint64_t function,
uint64_t char_size)
{
(void)char_size;
+#if defined(_WIN32) || defined(_WIN64)
+ // HIP on Windows only supplies the wide _wassert entry point. Keep the
+ // CUDA assertion's ASCII diagnostics and trap through public OCKL APIs.
+ auto output = __ockl_fprintf_stderr_begin();
+ const char* parts[] = {"CUDA assertion: ", (const char*)message, " at ", (const char*)file,
+ " in ", (const char*)function, "\n"};
+ for (const char* part : parts) {
+ const char* end = part;
+ while (*end) ++end;
+ output = __ockl_fprintf_append_string_n(output, part, end - part, 0);
+ }
+ __ockl_fprintf_append_args(output, 1, line, 0, 0, 0, 0, 0, 0, 1);
+ __builtin_trap();
+#else
[[clang::noinline]] __assert_fail((const char *)message, (const char *)file, line, (const char *)function);
+#endif
}
// * Smallest denormal is 1.4 × 10^-45
// * Smallest normal is ~1.175494351 × 10^(-38)
// * Now, 1.175494351×10^-38 / 1.4 × 10^-45 = 8396388 + 31/140
patches/zluda/0009-resource-descriptor-queries.patch
-124
View File
@@ -1,124 +0,0 @@
Implement the CUDA texture resource descriptor query required by NGX Evaluate.
Regression: tools/windows/cuda_image_probe.cpp in the d4r Windows port.
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -67,12 +67,14 @@
cuda_macros::cuda_function_declarations!(
unimplemented,
implemented
<= [
cuArray3DCreate_v2,
+ cuArray3DGetDescriptor_v2,
cuArrayCreate_v2,
cuArrayDestroy,
+ cuArrayGetDescriptor_v2,
cuCtxCreate_v2,
cuCtxCreate_v4,
cuCtxDestroy_v2,
cuCtxGetApiVersion,
cuCtxGetCurrent,
@@ -203,10 +205,11 @@
cuSurfObjectCreate,
cuSurfObjectDestroy,
cuSurfObjectGetResourceDesc,
cuTexObjectCreate,
cuTexObjectDestroy,
+ cuTexObjectGetResourceDesc,
cuTexRefSetAddress_v2,
cuTexRefSetAddressMode,
cuTexRefSetArray,
cuTexRefSetFilterMode,
cuTexRefSetFlags,
diff --git a/zluda/src/impl/tex.rs b/zluda/src/impl/tex.rs
--- a/zluda/src/impl/tex.rs
+++ b/zluda/src/impl/tex.rs
@@ -1,19 +1,58 @@
use crate::r#impl::hipfix;
use hip_runtime_sys::*;
+use std::collections::HashMap;
+use std::sync::{Mutex, OnceLock};
+
+fn object_resources() -> &'static Mutex<HashMap<usize, Vec<u8>>> {
+ static RESOURCES: OnceLock<Mutex<HashMap<usize, Vec<u8>>>> = OnceLock::new();
+ RESOURCES.get_or_init(|| Mutex::new(HashMap::new()))
+}
pub(crate) unsafe fn object_create(
p_tex_object: *mut hipTextureObject_t,
p_res_desc: *const HIP_RESOURCE_DESC,
p_tex_desc: *const HIP_TEXTURE_DESC,
p_res_view_desc: *const HIP_RESOURCE_VIEW_DESC,
) -> hipError_t {
- hipTexObjectCreate(p_tex_object, p_res_desc, p_tex_desc, p_res_view_desc)
+ let output = p_tex_object.as_mut().ok_or(hipErrorCode_t::InvalidValue)?;
+ let descriptor = p_res_desc.as_ref().ok_or(hipErrorCode_t::InvalidValue)?;
+ hipTexObjectCreate(output, descriptor, p_tex_desc, p_res_view_desc)?;
+ let bytes = std::slice::from_raw_parts(p_res_desc.cast::<u8>(), std::mem::size_of::<HIP_RESOURCE_DESC>()).to_vec();
+ match object_resources().lock() {
+ Ok(mut resources) => {
+ resources.insert(*output as usize, bytes);
+ Ok(())
+ }
+ Err(_) => {
+ let _ = hipDestroyTextureObject(*output);
+ *output = std::ptr::null_mut();
+ Err(hipErrorCode_t::Unknown)
+ }
+ }
}
pub(crate) unsafe fn object_destroy(tex_object: hipTextureObject_t) -> hipError_t {
- hipDestroyTextureObject(tex_object)
+ let mut resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
+ if !resources.contains_key(&(tex_object as usize)) { return Err(hipErrorCode_t::InvalidValue); }
+ hipDestroyTextureObject(tex_object)?;
+ resources.remove(&(tex_object as usize));
+ Ok(())
+}
+
+pub(crate) unsafe fn object_get_resource_desc(
+ descriptor: &mut HIP_RESOURCE_DESC,
+ tex_object: hipTextureObject_t,
+) -> hipError_t {
+ // NGX probes surface handles as textures before trying the surface API.
+ // Passing that foreign handle to HIP dereferences invalid host metadata.
+ // Return a CUDA error for unknown handles and preserve the complete CUDA
+ // descriptor (HIP's query leaves the flags/reserved bytes unspecified).
+ let resources = object_resources().lock().map_err(|_| hipErrorCode_t::Unknown)?;
+ let bytes = resources.get(&(tex_object as usize)).ok_or(hipErrorCode_t::InvalidValue)?;
+ std::ptr::copy_nonoverlapping(bytes.as_ptr(), (descriptor as *mut HIP_RESOURCE_DESC).cast::<u8>(), bytes.len());
+ Ok(())
}
pub(crate) unsafe fn ref_set_array(
texref: *mut textureReference,
array: hipArray_t,
diff --git a/zluda/src/impl/array.rs b/zluda/src/impl/array.rs
--- a/zluda/src/impl/array.rs
+++ b/zluda/src/impl/array.rs
@@ -70,5 +70,23 @@
handle: *mut hipArray_t,
desc: *const HIP_ARRAY3D_DESCRIPTOR,
) -> hipError_t {
hipArray3DCreate(handle, desc)
}
+
+pub(crate) unsafe fn get_descriptor_v2(
+ output: &mut HIP_ARRAY_DESCRIPTOR,
+ array: hipArray_t,
+) -> hipError_t {
+ hipArrayGetDescriptor(output, array)?;
+ if let Some(format) = original_format(array)? {
+ output.Format = hipArray_Format(format);
+ }
+ Ok(())
+}
+
+pub(crate) unsafe fn _3d_get_descriptor_v2(
+ output: &mut HIP_ARRAY3D_DESCRIPTOR,
+ array: hipArray_t,
+) -> hipError_t {
+ hipArray3DGetDescriptor(output, array)
+}
patches/zluda/0010-native-identity-replay-windows-cache.patch
-440
View File
@@ -1,440 +0,0 @@
Subject: [PATCH] Guard and own native overrides; capture K replay on Windows
Native replacements require the upstream PTX identity manifest. Preserve native
module/image ownership, bypass redundant JIT for completely replaced modules,
include the loaded Windows DLL fingerprint in cache identity, and expose opt-in
private real-weight replay captures with explicit native/translated launch logs.
diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
new file mode 100644
index 0000000..875b2e3
--- /dev/null
+++ b/zluda/src/impl/d4r_replay.rs
@@ -0,0 +1,109 @@
+//! Opt-in, synchronous diagnostic captures. Never enabled in the game fast path.
+//! Captures can contain proprietary network weights; keep them out of releases.
+use hip_runtime_sys::*;
+use std::{collections::HashMap, ffi::{c_void, CStr}, fs, path::PathBuf, sync::{Mutex, OnceLock}};
+
+struct Config { directory: PathBuf, filter: String, limit: u32 }
+static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
+static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
+
+pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
+
+fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
+ eprintln!("[d4r-replay] capture failed: {error}");
+ hipErrorCode_t::r#Unknown
+}
+
+impl Capture {
+ fn save(&self, destination: &std::path::Path) -> hipError_t {
+ fs::create_dir_all(destination).map_err(io_error)?;
+ for (id, &(base, size)) in self.allocations.iter().enumerate() {
+ let mut bytes = vec![0u8; size];
+ unsafe { hipMemcpyDtoH(bytes.as_mut_ptr().cast(), hipDeviceptr_t(base as *mut c_void), size) }?;
+ fs::write(destination.join(format!("alloc-{id}.bin")), bytes).map_err(io_error)?;
+ }
+ Ok(())
+ }
+
+ pub(crate) fn finish(self, stream: hipStream_t) -> hipError_t {
+ unsafe { hipStreamSynchronize(stream) }?;
+ self.save(&self.directory.join("gpu-output"))?;
+ eprintln!("[d4r-replay] complete path={}", self.directory.display());
+ Ok(())
+ }
+}
+
+pub(crate) unsafe fn begin(
+ name: &CStr, stream: hipStream_t, parameters: *mut *mut c_void,
+ extra: *mut *mut c_void, launch: [u32; 7],
+) -> Result<Option<Capture>, hipErrorCode_t> {
+ let Some(config) = CONFIG.get_or_init(|| std::env::var_os("D4R_CUDA_REPLAY_DUMP_DIR").map(|directory| Config {
+ directory: directory.into(),
+ filter: std::env::var("D4R_CUDA_REPLAY_DUMP_FILTER").unwrap_or_default(),
+ limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
+ })) else { return Ok(None) };
+ let name = name.to_str().map_err(io_error)?;
+ // Only the audited, single 176-byte K argument ABI is captured here.
+ if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
+ return Ok(None);
+ }
+ let mut block = std::ptr::null::<u8>();
+ let mut bytes = None;
+ if !extra.is_null() {
+ for i in (0..6).step_by(2) {
+ match *extra.add(i) as usize {
+ 1 => block = (*extra.add(i + 1)).cast(),
+ 2 => bytes = Some(*(*extra.add(i + 1) as *const usize)),
+ 0 => break,
+ _ => return Err(hipErrorCode_t::r#InvalidValue),
+ }
+ }
+ } else if !parameters.is_null() {
+ block = (*parameters).cast();
+ bytes = Some(176);
+ }
+ if block.is_null() || bytes != Some(176) {
+ return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
+ }
+ let sequence = {
+ let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
+ let count = counts.entry(name.to_owned()).or_default();
+ if *count >= config.limit { return Ok(None); }
+ *count += 1;
+ *count
+ };
+ hipStreamSynchronize(stream)?;
+ let arguments = std::slice::from_raw_parts(block, 176);
+ let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
+ fs::create_dir_all(&directory).map_err(io_error)?;
+ fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
+ let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
+ launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
+ let mut allocations = Vec::<(usize, usize)>::new();
+ for offset in (0..176).step_by(8) {
+ let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
+ if pointer == 0 { continue; }
+ let mut base = hipDeviceptr_t(std::ptr::null_mut());
+ let mut size = 0;
+ if hipMemGetAddressRange(&mut base, &mut size, hipDeviceptr_t(pointer as *mut c_void)).is_err() { continue; }
+ let base = base.0 as usize;
+ if pointer < base || pointer - base >= size { continue; }
+ if size > 512 * 1024 * 1024 { return Err(io_error("allocation exceeds diagnostic 512 MiB limit")); }
+ let id = match allocations.iter().position(|&(b, _)| b == base) {
+ Some(id) => id,
+ None => {
+ let id = allocations.len();
+ allocations.push((base, size));
+ manifest.push_str(&format!("alloc {id} 0x{base:x} {size}\n"));
+ id
+ }
+ };
+ manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
+ }
+ if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
+ fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
+ let capture = Capture { directory, allocations };
+ capture.save(&capture.directory)?;
+ eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
+ Ok(Some(capture))
+}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index 033d9a0..7a7470a 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -7,6 +7,8 @@ use hip_runtime_sys::*;
use std::mem;
pub(crate) struct Function {
+ pub(crate) name: std::ffi::CString,
+ pub(crate) native_owner: Option<super::module::NativeModule>,
pub(crate) base: hipFunction_t,
pub(crate) sm_version: u32,
pub(crate) explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
@@ -144,6 +146,8 @@ pub(crate) fn launch_kernel(
}
translated_extra_storage.as_mut_ptr()
};
+ let replay = unsafe { super::d4r_replay::begin(&f.name, stream, kernel_params, extra,
+ [grid_dim_x, grid_dim_y, grid_dim_z, block_dim_x, block_dim_y, block_dim_z, shared_mem_bytes]) }?;
// the parameter block: kernel_params[0] (a single struct parameter) or the extra buffer
let prep_key = if let (Some(_), Some(key_offset)) = (f.prep, f.prep_key) {
let mut block: *const u8 = std::ptr::null();
@@ -206,7 +210,11 @@ pub(crate) fn launch_kernel(
}
let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
let (grid_dim_x, grid_dim_y, grid_dim_z) = if f.grid_x != 0 { (f.grid_x, 1, 1) } else { (grid_dim_x, grid_dim_y, grid_dim_z) };
- unsafe {
+ if std::env::var_os("D4R_ZLUDA_VERBOSE").is_some() {
+ eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
+ f.name, if f.native_owner.is_some() { "native" } else { "translated" });
+ }
+ let result = unsafe {
hipModuleLaunchKernel(
f.base,
grid_dim_x,
@@ -220,7 +228,10 @@ pub(crate) fn launch_kernel(
kernel_params,
extra_for_hip,
)
- }
+ };
+ result?;
+ if let Some(replay) = replay { replay.finish(stream)?; }
+ Ok(())
}
pub(crate) unsafe fn set_attribute(
diff --git a/zluda/src/impl/library.rs b/zluda/src/impl/library.rs
index 98be33e..1319792 100644
--- a/zluda/src/impl/library.rs
+++ b/zluda/src/impl/library.rs
@@ -21,8 +21,8 @@ impl Library {
let module_lock = self.modules.get(device).ok_or(CUerror::INVALID_DEVICE)?;
*module_lock.get_or_init(|| match self.data {
LibraryData::Lazy(lib) => {
- let (module, sm_version, meta32) = module::load_hip_module(lib)?;
- Ok(Module::new(module, sm_version, meta32).wrap())
+ let (module, sm_version, meta32, ptx_hash, native_image) = module::load_hip_module(lib)?;
+ Ok(Module::new(module, sm_version, meta32, ptx_hash, native_image).wrap())
}
LibraryData::Eager(()) => Err(CUerror::NOT_SUPPORTED),
})
diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
index 5329d32..14e0aee 100644
--- a/zluda/src/impl/mod.rs
+++ b/zluda/src/impl/mod.rs
@@ -5,6 +5,7 @@ pub(super) mod array;
pub(super) mod context;
pub(super) mod device;
pub(super) mod driver;
+pub(super) mod d4r_replay;
pub(super) mod event;
pub(super) mod function;
pub(super) mod graph;
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 9f72f2f..5a79368 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -19,6 +19,8 @@ pub(crate) struct Module {
pub(crate) base: hipModule_t,
pub(crate) sm_version: u32,
pub(crate) bit32: Option<Metadata32Bit>,
+ ptx_hash: Option<u64>,
+ _native_base_image: Option<Vec<u8>>,
mutable: Mutex<ModuleMutable>,
}
@@ -33,12 +35,14 @@ impl ModuleMutable {
sm_version: u32,
name: CString,
explicit_args_size_align: Option<Vec<FunctionArgInfo>>,
+ ptx_hash: Option<u64>,
) -> Result<&Function, CUerror> {
Ok(match self.functions.entry(name) {
hash_map::Entry::Occupied(entry) => &*entry.into_mut(),
hash_map::Entry::Vacant(entry) => {
- let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key()) {
- Some(native) => native,
+ let mut native_owner = None;
+ let (func_handle, prep, block_z, grid_x, prep_key, prep_slots) = match native_override(entry.key(), ptx_hash) {
+ Some((native, owner)) => { native_owner = Some(owner); native },
None => {
let mut func_handle = unsafe { std::mem::zeroed() };
unsafe { hipModuleGetFunction(&mut func_handle, module, entry.key().as_ptr()) }?;
@@ -46,6 +50,8 @@ impl ModuleMutable {
}
};
let func = Box::new(Function {
+ name: entry.key().clone(),
+ native_owner,
base: func_handle,
sm_version,
explicit_args_size_align,
@@ -69,19 +75,48 @@ impl ModuleMutable {
/// same stream with `d4r_prep_blocks` (a u32 global in the code object) blocks of 128 threads.
/// An optional u32 global `d4r_block_z` replaces the launch's block z dimension, and `d4r_grid_x`
/// (persistent kernels, which derive the original grid from their parameters) the grid.
+pub(crate) struct NativeModule {
+ module: hipModule_t,
+ _image: Vec<u8>,
+}
+
+impl Drop for NativeModule {
+ fn drop(&mut self) {
+ if self.module.0.is_null() { return; }
+ if let Err(error) = unsafe { hipModuleUnload(self.module) } {
+ eprintln!("[d4r-native] unload failed: {error:?}");
+ }
+ }
+}
+
fn native_override(
name: &CStr,
-) -> Option<(hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32)> {
+ ptx_hash: Option<u64>,
+) -> Option<((hipFunction_t, Option<(hipFunction_t, u32)>, u32, u32, Option<u32>, u32), NativeModule)> {
let dir = std::env::var_os("D4R_ZLUDA_NATIVE_DIR")?;
+ let manifest = std::path::Path::new(&dir).join("d4r-kernels.txt");
+ let allowed = fs::read_to_string(&manifest).ok().is_some_and(|text| text.lines().any(|line| {
+ let mut fields = line.split_whitespace();
+ fields.next() == name.to_str().ok()
+ && fields.next().and_then(|s| u64::from_str_radix(s, 16).ok()) == ptx_hash
+ && ptx_hash.is_some()
+ }));
+ if !allowed {
+ eprintln!("[d4r-native] miss kernel={name:?} reason=ptx_identity hash={ptx_hash:016x?} fallback=translated");
+ return None;
+ }
let path = std::path::Path::new(&dir).join(format!("{}.hsaco", name.to_str().ok()?));
- let image = fs::read(&path).ok()?;
- // HIP may reference the image after loading; the override lives for the whole process.
- let image: &'static [u8] = Box::leak(image.into_boxed_slice());
+ let image = match fs::read(&path) {
+ Ok(image) => image,
+ Err(error) => { eprintln!("[d4r-native] miss kernel={name:?} reason=object_missing error={error} fallback=translated"); return None; }
+ };
let mut native_module = unsafe { mem::zeroed() };
if unsafe { hipModuleLoadData(&mut native_module, image.as_ptr().cast()) }.is_err() {
eprintln!("[d4r-native] failed to load {}", path.display());
return None;
}
+ // Keep the image and module owned by the CUfunction; release at module unload.
+ let owner = NativeModule { module: native_module, _image: image };
let mut func = unsafe { mem::zeroed() };
if unsafe { hipModuleGetFunction(&mut func, native_module, name.as_ptr()) }.is_err() {
eprintln!("[d4r-native] {} has no kernel {:?}", path.display(), name);
@@ -138,7 +173,7 @@ fn native_override(
prep_key,
prep_slots
);
- Some((func, prep, block_z, grid_x, prep_key, prep_slots))
+ Some(((func, prep, block_z, grid_x, prep_key, prep_slots), owner))
}
impl ZludaObject for Module {
@@ -229,7 +264,7 @@ pub(crate) struct Global32Bit {
}
impl Module {
- pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>) -> Self {
+ pub(crate) fn new(base: hipModule_t, sm_version: u32, bit32: Option<Metadata32Bit>, ptx_hash: Option<u64>, native_base_image: Option<Vec<u8>>) -> Self {
Self {
base,
sm_version,
@@ -237,6 +272,8 @@ impl Module {
functions: FxHashMap::default(),
}),
bit32,
+ ptx_hash,
+ _native_base_image: native_base_image,
}
}
@@ -248,7 +285,7 @@ impl Module {
.map(Vec::clone)
});
mutable
- .get_function(self.base, self.sm_version, name.to_owned(), explicit_args)
+ .get_function(self.base, self.sm_version, name.to_owned(), explicit_args, self.ptx_hash)
.map(|f| unsafe { (f as *const Function).as_ref().unwrap() })
}
}
@@ -256,7 +293,7 @@ impl Module {
fn get_best_ptx_and_compile(
global_state: &GlobalState,
image: CodeLibraryRef<'_>,
-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
+) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
let mut ptx_modules = Vec::new();
unsafe {
CodeLibraryRef::iterate_modules(image, |_, module| match module {
@@ -280,7 +317,7 @@ fn get_best_ptx_and_compile(
let dump_dir = std::path::PathBuf::from(dump_dir);
let _ = fs::create_dir_all(&dump_dir);
for (index, source) in ptx_modules.iter().enumerate() {
- let stem = format!("module-{index:04}");
+ let stem = format!("module-{:016x}-{index:04}", ptx_identity(source));
let _ = fs::write(dump_dir.join(format!("{stem}.ptx")), source.as_bytes());
let diagnostic = match ptx_parser::parse_module_checked(source.as_ref()) {
Ok(ast) => format!(
@@ -345,6 +382,32 @@ fn get_best_ptx_and_compile(
)
}
};
+ let ptx_hash = text.map(|source| ptx_identity(source));
+ // A fully replaced, single-entry module needs no translated binary. LDS
+ // declarations belong to the replaced kernel; exported globals do not.
+ let only_kernel_and_lds = module.directives.iter().all(|directive| match directive {
+ ptx_parser::Directive::Method(_, method) => method.func_directive.name.is_kernel(),
+ ptx_parser::Directive::Variable(_, variable) => matches!(variable.info.state_space, ptx_parser::StateSpace::Shared),
+ });
+ let entries: Vec<_> = module.directives.iter().filter_map(|directive| match directive {
+ ptx_parser::Directive::Method(_, method) => Some(method), _ => None,
+ }).collect();
+ if module.address_size == 64 && only_kernel_and_lds && entries.len() == 1 {
+ let method = entries[0];
+ {
+ if method.func_directive.name.is_kernel() {
+ if let Ok(name) = CString::new(method.func_directive.name.text()) {
+ if let Some((_, mut owner)) = native_override(&name, ptx_hash) {
+ let base = owner.module;
+ owner.module = hipModule_t(std::ptr::null_mut());
+ let image = mem::take(&mut owner._image);
+ eprintln!("[d4r-native] module_direct kernel={name:?} translated_jit=skipped");
+ return Ok((base, module.sm_version, None, ptx_hash, Some(image)));
+ }
+ }
+ }
+ }
+ }
// TODO: get this information on initialization
let hip_properties = get_hip_properties()?;
let gcn_arch = get_gcn_arch(&hip_properties)?;
@@ -372,7 +435,12 @@ fn get_best_ptx_and_compile(
.or_else(|_| compile_and_cache(gcn_arch, attributes, module, &mut cache_with_key))?;
let mut hip_module = unsafe { mem::zeroed() };
unsafe { hipModuleLoadData(&mut hip_module, elf_module.as_ptr().cast()) }?;
- Ok((hip_module, sm_version, zluda32))
+ Ok((hip_module, sm_version, zluda32, ptx_hash, None))
+}
+
+fn ptx_identity(text: &str) -> u64 {
+ text.trim_end_matches('\0').bytes().fold(0xcbf29ce484222325u64, |hash, byte|
+ (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3))
}
fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
@@ -384,7 +452,7 @@ fn cow_bytes_to_str<'a>(data: Cow<'a, [u8]>) -> Option<Cow<'a, str>> {
pub(crate) fn load_hip_module(
library: CodeLibraryRef,
-) -> Result<(hipModule_t, u32, Option<Metadata32Bit>), CUerror> {
+) -> Result<(hipModule_t, u32, Option<Metadata32Bit>, Option<u64>, Option<Vec<u8>>), CUerror> {
let global_state = driver::global_state()?;
get_best_ptx_and_compile(global_state, library)
}
@@ -434,7 +502,7 @@ fn zluda_build_version() -> &'static str {
key.starts_with("D4R_ZLUDA_")
&& !matches!(
key.as_str(),
- "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA"
+ "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
@@ -469,7 +537,10 @@ fn loaded_library_fingerprint() -> Option<String> {
#[cfg(windows)]
fn loaded_library_fingerprint() -> Option<String> {
- None
+ let path = zluda_windows::get_module_path_for_function(zluda_build_version as *const () as usize)?;
+ let metadata = fs::metadata(path).ok()?;
+ let timestamp = metadata.modified().ok()?.duration_since(std::time::UNIX_EPOCH).ok()?;
+ Some(format!("{:x}.{:x}.{:x}", metadata.len(), timestamp.as_secs(), timestamp.subsec_nanos()))
}
fn get_cache_key<'a, 'b>(
@@ -577,16 +648,16 @@ pub(crate) fn load(module: &mut CUmodule, fname: &CStr) -> CUresult {
image.push(0);
let library = unsafe { CodeLibraryRef::try_load(image.as_ptr() as *const std::ffi::c_void) }
.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
- let (hip_module, sm_version, meta32) = load_hip_module(library)?;
- *module = Module::new(hip_module, sm_version, meta32).wrap();
+ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
+ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
Ok(())
}
pub(crate) fn load_data(module: &mut CUmodule, image: &std::ffi::c_void) -> CUresult {
let library =
unsafe { CodeLibraryRef::try_load(image) }.map_err(|_| CUerror::NO_BINARY_FOR_GPU)?;
- let (hip_module, sm_version, meta32) = load_hip_module(library)?;
- *module = Module::new(hip_module, sm_version, meta32).wrap();
+ let (hip_module, sm_version, meta32, ptx_hash, native_image) = load_hip_module(library)?;
+ *module = Module::new(hip_module, sm_version, meta32, ptx_hash, native_image).wrap();
Ok(())
}
patches/zluda/0011-audited-k-m-windows-replay.patch
-93
View File
@@ -1,93 +0,0 @@
diff --git a/zluda/src/impl/d4r_replay.rs b/zluda/src/impl/d4r_replay.rs
index 875b2e3..d491ae2 100644
--- a/zluda/src/impl/d4r_replay.rs
+++ b/zluda/src/impl/d4r_replay.rs
@@ -7,6 +7,18 @@ struct Config { directory: PathBuf, filter: String, limit: u32 }
static CONFIG: OnceLock<Option<Config>> = OnceLock::new();
static COUNTS: OnceLock<Mutex<HashMap<String, u32>>> = OnceLock::new();
+fn parameter_size(name: &str) -> Option<usize> {
+ match name {
+ "dltss_pwin_enc0_layer" | "dltss_pwin_enc1_layer" | "dltss_pwin_enc2_layer" |
+ "dltss_pwin_enc3_layer" | "dltss_pwin_enc4_layer" | "dltss_pwin_dec5_layer" |
+ "dltss_pwin_dec4_layer" | "dltss_pwin_dec3_layer" | "dltss_pwin_dec2_layer" |
+ "dltss_pwin_dec1_layer" | "dltss_pwin_dec0_layer" => Some(176),
+ "rrlite_enc1_4x4" | "rrlite_enc2_4x4" | "rrlite_dec2_4x4" | "rrlite_dec1_4x4" => Some(56),
+ "rrlite_enc3_tube_4x4" => Some(88),
+ _ => None,
+ }
+}
+
pub(crate) struct Capture { directory: PathBuf, allocations: Vec<(usize, usize)> }
fn io_error(error: impl std::fmt::Display) -> hipErrorCode_t {
@@ -43,8 +55,10 @@ pub(crate) unsafe fn begin(
limit: std::env::var("D4R_CUDA_REPLAY_DUMP_LIMIT").ok().and_then(|s| s.parse().ok()).unwrap_or(1),
})) else { return Ok(None) };
let name = name.to_str().map_err(io_error)?;
- // Only the audited, single 176-byte K argument ABI is captured here.
- if !name.starts_with("dltss_pwin_") || !name.ends_with("_layer") || !name.contains(&config.filter) {
+ // Only audited single-by-value K/M argument blocks. Never infer the size
+ // from a prefix: reading a different kernel's parameters can overrun host memory.
+ let Some(parameter_bytes) = parameter_size(name) else { return Ok(None); };
+ if !name.contains(&config.filter) {
return Ok(None);
}
let mut block = std::ptr::null::<u8>();
@@ -60,10 +74,10 @@ pub(crate) unsafe fn begin(
}
} else if !parameters.is_null() {
block = (*parameters).cast();
- bytes = Some(176);
+ bytes = Some(parameter_bytes);
}
- if block.is_null() || bytes != Some(176) {
- return Err(io_error(format!("unexpected K parameter ABI for {name}: {bytes:?}")));
+ if block.is_null() || bytes != Some(parameter_bytes) {
+ return Err(io_error(format!("unexpected parameter ABI for {name}: {bytes:?}, expected {parameter_bytes}")));
}
let sequence = {
let mut counts = COUNTS.get_or_init(|| Mutex::new(HashMap::new())).lock().map_err(io_error)?;
@@ -73,14 +87,14 @@ pub(crate) unsafe fn begin(
*count
};
hipStreamSynchronize(stream)?;
- let arguments = std::slice::from_raw_parts(block, 176);
+ let arguments = std::slice::from_raw_parts(block, parameter_bytes);
let directory = config.directory.join(format!("replay-{sequence:06}-{name}"));
fs::create_dir_all(&directory).map_err(io_error)?;
fs::write(directory.join("args.bin"), arguments).map_err(io_error)?;
- let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs 176\n",
+ let mut manifest = format!("kernel {name}\nlaunch {} {} {} {} {} {} {}\nargs {parameter_bytes}\n",
launch[0], launch[1], launch[2], launch[3], launch[4], launch[5], launch[6]);
let mut allocations = Vec::<(usize, usize)>::new();
- for offset in (0..176).step_by(8) {
+ for offset in (0..parameter_bytes).step_by(8) {
let pointer = u64::from_le_bytes(arguments[offset..offset + 8].try_into().unwrap()) as usize;
if pointer == 0 { continue; }
let mut base = hipDeviceptr_t(std::ptr::null_mut());
@@ -100,10 +114,23 @@ pub(crate) unsafe fn begin(
};
manifest.push_str(&format!("pointer {offset} {id} {}\n", pointer - base));
}
- if allocations.is_empty() { return Err(io_error("no K allocation pointers resolved")); }
+ if allocations.is_empty() { return Err(io_error("no allocation pointers resolved")); }
fs::write(directory.join("manifest.txt"), manifest).map_err(io_error)?;
let capture = Capture { directory, allocations };
capture.save(&capture.directory)?;
eprintln!("[d4r-replay] begin kernel={name} allocations={} path={}", capture.allocations.len(), capture.directory.display());
Ok(Some(capture))
}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn only_audited_argument_sizes() {
+ assert_eq!(super::parameter_size("dltss_pwin_enc0_layer"), Some(176));
+ assert_eq!(super::parameter_size("rrlite_dec2_4x4"), Some(56));
+ assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4"), Some(88));
+ assert_eq!(super::parameter_size("dltss_pwin_enc0_layer_prep"), None);
+ assert_eq!(super::parameter_size("rrlite_enc3_tube_4x4_prep"), None);
+ assert_eq!(super::parameter_size("dltss_pwin_unknown_layer"), None);
+ }
+}
patches/zluda/0012-configurable-windows-jit-cache.patch
-65
View File
@@ -1,65 +0,0 @@
diff --git a/zluda_cache/src/lib.rs b/zluda_cache/src/lib.rs
index acd262e..c3ac1be 100644
--- a/zluda_cache/src/lib.rs
+++ b/zluda_cache/src/lib.rs
@@ -22,18 +22,36 @@ pub struct ModuleCache(SqliteConnection);
impl ModuleCache {
pub fn create_cache_dir_and_get_path() -> Option<String> {
- let mut cache_dir = dirs::cache_dir()?;
- cache_dir.extend(["zluda", "ComputeCache"]);
+ let override_dir = std::env::var_os("ZLUDA_CACHE_DIR");
+ let cache_dir = match override_dir {
+ Some(path) => {
+ let path = std::path::PathBuf::from(path);
+ if !path.is_absolute() {
+ eprintln!("[zluda-cache] ZLUDA_CACHE_DIR must be absolute; cache disabled");
+ return None;
+ }
+ path
+ }
+ None => dirs::cache_dir()?.join("zluda").join("ComputeCache"),
+ };
+ Self::cache_path(&cache_dir)
+ }
+
+ fn cache_path(cache_dir: &std::path::Path) -> Option<String> {
// We ensure that the cache directory exists
- std::fs::create_dir_all(&cache_dir).ok()?;
+ if let Err(error) = std::fs::create_dir_all(cache_dir) {
+ eprintln!("[zluda-cache] cannot create {}: {error}; cache disabled", cache_dir.display());
+ return None;
+ }
// No need to create the file, it will be created by SQLite on first access
// zluda.db might be in use by older versions of ZLUDA
- cache_dir.push("zluda2.db");
- Some(cache_dir.to_string_lossy().into())
+ Some(cache_dir.join("zluda2.db").to_string_lossy().into())
}
pub fn open(file_path: &str) -> Option<Self> {
- let mut conn = SqliteConnection::establish(file_path).ok()?;
+ let mut conn = SqliteConnection::establish(file_path).map_err(|error| {
+ eprintln!("[zluda-cache] cannot open {file_path}: {error}; cache disabled");
+ }).ok()?;
conn.batch_execute("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")
.ok()?;
conn.run_pending_migrations(MIGRATIONS).ok()?;
@@ -86,6 +104,17 @@ mod tests {
use arrayvec::ArrayString;
use diesel::prelude::*;
+ #[test]
+ fn explicit_directory_creates_persistent_database() {
+ let directory = std::env::temp_dir().join(format!("zluda-cache-test-{}-{}", std::process::id(), ModuleCache::time_now()));
+ let path = ModuleCache::cache_path(&directory).unwrap();
+ assert_eq!(std::path::Path::new(&path), directory.join("zluda2.db"));
+ let cache = ModuleCache::open(&path).unwrap();
+ drop(cache);
+ assert!(std::path::Path::new(&path).is_file());
+ std::fs::remove_dir_all(directory).unwrap();
+ }
+
#[derive(Queryable, Selectable)]
#[diesel(table_name = crate::schema::modules)]
#[diesel(check_for_backend(diesel::sqlite::Sqlite))]
patches/zluda/0013-gfx12-f16-reference-accumulation.patch
-151
View File
@@ -1,151 +0,0 @@
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 5a54417..90c0034 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -158,6 +158,10 @@ fn wmma_rewrite_flags(gcn_arch: &str) -> Option<u32> {
let mut flags = if wmma_fp8_rewrite_requested() { 1 } else { 0 };
if family == 12 {
flags |= 2;
+ // Explicit diagnostic/correctness mode; retain upstream's fast default.
+ if std::env::var("D4R_ZLUDA_WMMA_F16_REFERENCE").is_ok_and(|value| value.trim() == "1") {
+ flags |= 8;
+ }
// native FP8 needs gfx12 hardware; the layout shim on gfx11 takes it only for tests (D4R_WMMA12_SHIM)
if flags & 1 != 0 && wmma_fp8_native_requested() {
flags |= 4;
diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
index 07238db..d262fc6 100644
--- a/llvm_zluda/src/lib.cpp
+++ b/llvm_zluda/src/lib.cpp
@@ -970,6 +970,7 @@ static unsigned widenMmaK8(Module *M)
// Flags bit 0: also lower m16n8k32 e4m3 MMAs (D4R_ZLUDA_WMMA_FP8=1).
// Flags bit 1: gfx12 (RDNA4) WMMA layout: the d4r_wmma12_* helpers.
// Flags bit 2 (with bit 1): e4m3 MMAs on native FP8 WMMA: the d4r_wmma12f8_* e4m3 helpers.
+// Flags bit 3 (with bit 1): f64 sum / one f16 round for f16 accumulator MMAs.
// The helper sets share one contract (see zluda_ptx_impl.cpp), so only their names change here.
unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
{
@@ -999,14 +1000,15 @@ unsigned LLVMZludaRewriteMmaF16Wmma(LLVMModuleRef ModuleRef, unsigned Flags)
const std::string Arch = Gfx12 ? "__zluda_ptx_impl_d4r_wmma12_" : "__zluda_ptx_impl_d4r_wmma_";
const std::string E4m3 = NativeFp8 ? "__zluda_ptx_impl_d4r_wmma12f8_" : Arch;
const std::string Widen = NativeFp8 ? E4m3 : "__zluda_ptx_impl_d4r_wmma_";
+ const std::string F16 = Arch + (Gfx12 && (Flags & 8) != 0 ? "t_mma_f16_reference" : "t_mma_f16");
Helpers H;
H.AFromB = M->getOrInsertFunction((Arch + "t_a_from_b").c_str(), Attributes, V8, V2);
H.BFromA = M->getOrInsertFunction((Arch + "t_b_from_a").c_str(), Attributes, V8, V4);
- H.Mma = M->getOrInsertFunction((Arch + "t_mma_f16").c_str(), Attributes, V2, V8, V8, V2);
+ H.Mma = M->getOrInsertFunction(F16.c_str(), Attributes, V2, V8, V8, V2);
H.FromNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_from_nv", Attributes, V2, V2);
H.ToNv = M->getOrInsertFunction("__zluda_ptx_impl_d4r_wmma_t_to_nv", Attributes, V2, V2);
H.AFromBPair = M->getOrInsertFunction((Arch + "t_a_from_b_pair").c_str(), Attributes, V8, V2, V2);
- H.MmaPair = M->getOrInsertFunction((Arch + "t_mma_f16_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
+ H.MmaPair = M->getOrInsertFunction((F16 + "_pair").c_str(), Attributes, V4, V8, V8, V2, V2);
{
Type *F = Type::getFloatTy(Context);
Type *V4F = FixedVectorType::get(F, 4);
diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
index 4ff25dd..36bd6b1 100644
--- a/ptx/lib/zluda_ptx_impl.cpp
+++ b/ptx/lib/zluda_ptx_impl.cpp
@@ -1801,6 +1801,78 @@ __device__ static wmma_float8 wmma12_f16(wmma_uint8 a, wmma_uint8 b, wmma_float8
#endif
}
+// Correctness baseline for f16-accumulator MMAs. f32 WMMA accumulation may
+// reach the other f16 neighbour around cancellation-heavy rounding midpoints.
+// Keep the existing gfx12 gathers/layout; sum their f16 operands in f64 and
+// round directly to f16 once, matching d4r's independent NumPy/PTX reference.
+// The e4m3 and f32-shadow paths continue using native WMMA.
+__device__ static double wmma12_half_value(uint32_t bits)
+{
+ const uint32_t magnitude = bits & 0x7fffu, exponent = magnitude >> 10;
+ const double value = exponent == 0 ? double(magnitude) * 0x1p-24 :
+ double(std::bit_cast<float>(exponent == 31 ? (0x7f800000u | ((magnitude & 1023u) << 13)) :
+ ((magnitude << 13) + 0x38000000u)));
+ return bits & 0x8000u ? -value : value;
+}
+
+// Preserve f16 subnormal result bits even when the target's default f16
+// arithmetic mode flushes them. No intermediate f32 rounding is involved.
+__device__ static uint32_t wmma12_half_round(double value)
+{
+ const uint64_t bits = std::bit_cast<uint64_t>(value);
+ const uint32_t sign = uint32_t(bits >> 48) & 0x8000u;
+ const uint32_t exponent = uint32_t((bits >> 52) & 2047u);
+ const uint64_t fraction = bits & 0xfffffffffffffu;
+ if (exponent == 2047u) return sign | (fraction ? 0x7e00u : 0x7c00u);
+ const int power = int(exponent) - 1023;
+ if (power < -25) return sign;
+ if (power > 15) return sign | 0x7c00u;
+ const unsigned shift = power < -14 ? unsigned(28 - power) : 42u;
+ const uint64_t significand = fraction | 0x10000000000000ull;
+ uint32_t code = uint32_t(significand >> shift);
+ const uint64_t tail = significand & ((uint64_t(1) << shift) - 1);
+ const uint64_t midpoint = uint64_t(1) << (shift - 1);
+ code += tail > midpoint || (tail == midpoint && (code & 1u));
+ if (power >= -14) code += uint32_t(power + 14) << 10;
+ return sign | code;
+}
+
+__device__ static wmma_uint4 wmma12_reference_c(uint2::Native_vec_ c0, uint2::Native_vec_ c1, bool paired)
+{
+ const bool high = wmma12_half() != 0;
+ const uint2::Native_vec_ own = paired && high ? c1 : c0;
+ const uint2::Native_vec_ send = paired && !high ? c1 : c0;
+ const uint2::Native_vec_ other = {wmma12_other_half(send.x), wmma12_other_half(send.y)};
+ const uint2::Native_vec_ even = paired && high ? other : own;
+ const uint2::Native_vec_ odd = paired && high ? own : other;
+ return {(even.x & 0xffffu) | (odd.x << 16), (even.x >> 16) | (odd.x & 0xffff0000u),
+ (even.y & 0xffffu) | (odd.y << 16), (even.y >> 16) | (odd.y & 0xffff0000u)};
+}
+
+__device__ static wmma_uint4 wmma12_f16_reference(wmma_uint8 a, wmma_uint8 b, wmma_uint4 c)
+{
+ // x/y: packed even rows, z/w: packed odd rows, in this half-wave.
+ wmma_uint4 result = {};
+ const uint32_t half = wmma12_half();
+ for (uint32_t i = 0; i < 8; ++i)
+ {
+ const uint32_t row = i + 8 * half;
+ double sum = wmma12_half_value((c[i / 2] >> (16 * (i & 1))) & 0xffffu);
+ for (uint32_t khalf = 0; khalf < 2; ++khalf)
+ for (uint32_t j = 0; j < 4; ++j)
+ {
+ const uint32_t av = wmma_from_lane(row + 16 * khalf, a[j]);
+ const uint32_t other = wmma12_other_half(b[j]);
+ const uint32_t bv = half == khalf ? b[j] : other;
+ sum += wmma12_half_value(av & 0xffffu) * wmma12_half_value(bv & 0xffffu);
+ sum += wmma12_half_value(av >> 16) * wmma12_half_value(bv >> 16);
+ }
+ // Do not round through f32: that can reintroduce the midpoint error.
+ result[(i & 1) * 2 + i / 4] |= wmma12_half_round(sum) << (16 * ((i / 2) & 1));
+ }
+ return result;
+}
+
// f32 D' = A' B' + C' for operands whose dwords hold two e4m3 bytes in their low 16 bits (the d4r_wmma12f8
// e4m3 "widening" below keeps the bytes): true e4m3 values, no scaling
__device__ static wmma_float8 wmma12_e4m3(wmma_uint8 a, wmma_uint8 b, wmma_float8 c)
@@ -1959,6 +2031,22 @@ extern "C"
return wmma12_t_from_d_pair(wmma12_f16(a_t, b_t, wmma12_c_from_t_pair(c0_t, c1_t)));
}
+ uint2::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c_t)
+ {
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c_t, c_t, false));
+ const uint2::Native_vec_ other = {wmma12_other_half(rows.z), wmma12_other_half(rows.w)};
+ return wmma12_half() ? other : uint2::Native_vec_{rows.x, rows.y};
+ }
+
+ uint4::Native_vec_ FUNC(d4r_wmma12_t_mma_f16_reference_pair)(wmma_uint8 a_t, wmma_uint8 b_t, uint2::Native_vec_ c0_t, uint2::Native_vec_ c1_t)
+ {
+ const wmma_uint4 rows = wmma12_f16_reference(a_t, b_t, wmma12_reference_c(c0_t, c1_t, true));
+ const bool high = wmma12_half() != 0;
+ const uint32_t r0 = wmma12_other_half(high ? rows.x : rows.z);
+ const uint32_t r1 = wmma12_other_half(high ? rows.y : rows.w);
+ return high ? uint4::Native_vec_{r0, r1, rows.z, rows.w} : uint4::Native_vec_{rows.x, rows.y, r0, r1};
+ }
+
wmma_float4 FUNC(d4r_wmma12_s_mma_f16)(wmma_uint8 a_t, wmma_uint8 b_t, wmma_float4 c, float up, float down)
{
return wmma12_s_from_d(wmma12_f16(a_t, b_t, wmma12_c_from_s(c * up))) * down;
patches/zluda/0014-windows-kernel-profiling.patch
-148
View File
@@ -1,148 +0,0 @@
From b0161a4166652eee86a8113a38d042b02403db6e Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 18:07:07 +0300
Subject: [PATCH] Add opt-in Windows HIP event and occupancy diagnostics
---
zluda/src/impl/d4r_profile.rs | 60 +++++++++++++++++++++++++++++++++++
zluda/src/impl/function.rs | 9 ++++++
zluda/src/impl/mod.rs | 1 +
zluda/src/impl/module.rs | 2 +-
4 files changed, 71 insertions(+), 1 deletion(-)
create mode 100644 zluda/src/impl/d4r_profile.rs
diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
new file mode 100644
index 0000000..33ba5f6
--- /dev/null
+++ b/zluda/src/impl/d4r_profile.rs
@@ -0,0 +1,60 @@
+//! Optional, deliberately serializing HIP-event diagnostics. Not a production
+//! scheduler or a claim about application FPS / measured hardware occupancy.
+use hip_runtime_sys::*;
+use std::{ffi::CStr, ptr, time::Instant};
+
+pub(super) struct Launch {
+ start: hipEvent_t,
+ end: hipEvent_t,
+ host: Instant,
+}
+
+impl Drop for Launch {
+ fn drop(&mut self) {
+ unsafe {
+ if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
+ if !self.start.is_null() { let _ = hipEventDestroy(self.start); }
+ }
+ }
+}
+
+impl Launch {
+ pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
+ if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
+ let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
+ hipEventCreateWithFlags(&mut result.start, 0)?;
+ hipEventCreateWithFlags(&mut result.end, 0)?;
+ hipEventRecord(result.start, stream)?;
+ result.host = Instant::now();
+ Ok(Some(result))
+ }
+
+ pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
+ phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
+ let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
+ hipEventRecord(self.end, stream)?;
+ hipEventSynchronize(self.end)?;
+ let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
+ let mut gpu_ms = 0.;
+ hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
+ // Module function handles use the Driver API attribute query. The
+ // plural Runtime API expects a registered host function address.
+ let mut lds = 0;
+ let mut regs = 0;
+ let mut private = 0;
+ let mut max_threads = 0;
+ hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
+ hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
+ let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
+ .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
+ let mut blocks_per_multiprocessor = 0;
+ hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
+ function, threads as i32, shared as usize)?;
+ eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
+ grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
+ regs, private, max_threads);
+ Ok(())
+ }
+}
diff --git a/zluda/src/impl/function.rs b/zluda/src/impl/function.rs
index 7a7470a..c456dfe 100644
--- a/zluda/src/impl/function.rs
+++ b/zluda/src/impl/function.rs
@@ -191,6 +191,7 @@ pub(crate) fn launch_kernel(
}
};
if let (Some((prep, blocks)), true) = (f.prep, run_prep) {
+ let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
unsafe {
hipModuleLaunchKernel(
prep,
@@ -206,6 +207,9 @@ pub(crate) fn launch_kernel(
extra_for_hip,
)
}?;
+ if let Some(profile) = profile {
+ unsafe { profile.finish(prep, &f.name, "native", "prep", [blocks,1,1], [128,1,1], 0, stream) }?;
+ }
f.prep_last.store(prep_key, std::sync::atomic::Ordering::Relaxed);
}
let block_dim_z = if f.block_z != 0 { f.block_z } else { block_dim_z };
@@ -214,6 +218,7 @@ pub(crate) fn launch_kernel(
eprintln!("[d4r-launch] kernel={:?} backend={} grid={grid_dim_x},{grid_dim_y},{grid_dim_z} block={block_dim_x},{block_dim_y},{block_dim_z}",
f.name, if f.native_owner.is_some() { "native" } else { "translated" });
}
+ let profile = unsafe { super::d4r_profile::Launch::begin(stream) }?;
let result = unsafe {
hipModuleLaunchKernel(
f.base,
@@ -230,6 +235,10 @@ pub(crate) fn launch_kernel(
)
};
result?;
+ if let Some(profile) = profile {
+ unsafe { profile.finish(f.base, &f.name, if f.native_owner.is_some() { "native" } else { "translated" },
+ "main", [grid_dim_x, grid_dim_y, grid_dim_z], [block_dim_x, block_dim_y, block_dim_z], shared_mem_bytes, stream) }?;
+ }
if let Some(replay) = replay { replay.finish(stream)?; }
Ok(())
}
diff --git a/zluda/src/impl/mod.rs b/zluda/src/impl/mod.rs
index 14e0aee..380eb6e 100644
--- a/zluda/src/impl/mod.rs
+++ b/zluda/src/impl/mod.rs
@@ -6,6 +6,7 @@ pub(super) mod context;
pub(super) mod device;
pub(super) mod driver;
pub(super) mod d4r_replay;
+pub(super) mod d4r_profile;
pub(super) mod event;
pub(super) mod function;
pub(super) mod graph;
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 5a79368..232f7d4 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -502,7 +502,7 @@ fn zluda_build_version() -> &'static str {
key.starts_with("D4R_ZLUDA_")
&& !matches!(
key.as_str(),
- "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE"
+ "D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
--
2.55.0.windows.5
patches/zluda/0015-nonserializing-cuda-api-profile.patch
-83
View File
@@ -1,83 +0,0 @@
From a1c506fc956cfc347c56c44311669700c0d500ac Mon Sep 17 00:00:00 2001
From: Codex <codex@localhost>
Date: Thu, 1 Oct 2026 22:16:19 +0300
Subject: [PATCH] Profile CUDA host API duration without serializing GPU
launches
---
zluda/src/d4r_api_profile.rs | 34 ++++++++++++++++++++++++++++++++++
zluda/src/lib.rs | 3 +++
2 files changed, 37 insertions(+)
create mode 100644 zluda/src/d4r_api_profile.rs
diff --git a/zluda/src/d4r_api_profile.rs b/zluda/src/d4r_api_profile.rs
new file mode 100644
index 0000000..ef2da1b
--- /dev/null
+++ b/zluda/src/d4r_api_profile.rs
@@ -0,0 +1,34 @@
+//! Host API timings only: no HIP events, stream waits, or altered scheduling.
+use std::{cell::RefCell, collections::BTreeMap, sync::OnceLock, time::Instant};
+
+#[derive(Default)]
+struct Row { calls: u64, total_ms: f64, max_ms: f64 }
+struct Samples { since: Instant, rows: BTreeMap<&'static str, Row> }
+thread_local! {
+ static SAMPLES: RefCell<Samples> = RefCell::new(Samples { since: Instant::now(), rows: BTreeMap::new() });
+}
+pub(crate) struct Call { name: &'static str, start: Instant }
+impl Call {
+ pub(crate) fn begin(name: &'static str) -> Option<Self> {
+ static ENABLED: OnceLock<bool> = OnceLock::new();
+ if !*ENABLED.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_API").is_some()) { return None; }
+ Some(Self { name, start: Instant::now() })
+ }
+}
+impl Drop for Call {
+ fn drop(&mut self) {
+ let ms = self.start.elapsed().as_secs_f64() * 1000.;
+ SAMPLES.with(|samples| {
+ let mut samples = samples.borrow_mut();
+ let row = samples.rows.entry(self.name).or_default();
+ row.calls += 1; row.total_ms += ms; row.max_ms = row.max_ms.max(ms);
+ let period_ms = samples.since.elapsed().as_secs_f64() * 1000.;
+ if period_ms < 2000. { return; }
+ for (name, row) in &samples.rows {
+ eprintln!("D4R_CUDA_API_PROFILE thread={:?} api={name} calls={} total_ms={:.6} max_ms={:.6} period_ms={period_ms:.6} serializing=0",
+ std::thread::current().id(), row.calls, row.total_ms, row.max_ms);
+ }
+ samples.rows.clear(); samples.since = Instant::now();
+ });
+ }
+}
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
index ecdeb3e..31bfa42 100644
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -2,6 +2,7 @@ use cuda_types::cuda::CUerror;
use std::sync::atomic::{AtomicBool, Ordering};
pub(crate) mod r#impl;
+mod d4r_api_profile;
#[cfg_attr(windows, path = "os_win.rs")]
#[cfg_attr(not(windows), path = "os_unix.rs")]
mod os;
@@ -40,6 +41,7 @@ macro_rules! implemented {
if !initialized() {
return Err(CUerror::DEINITIALIZED);
}
+ let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
cuda_macros::cuda_normalize_fn!( crate::r#impl::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
Ok(())
}
@@ -57,6 +59,7 @@ macro_rules! implemented_in_function {
if !initialized() {
return Err(CUerror::DEINITIALIZED);
}
+ let _profile = d4r_api_profile::Call::begin(stringify!($fn_name));
cuda_macros::cuda_normalize_fn!( crate::r#impl::function::$fn_name ) ($(zluda_common::FromCuda::<_, CUerror>::from_cuda(&$arg_id)?),*)?;
Ok(())
}
--
2.55.0.windows.5
patches/zluda/0016-deferred-kernel-profile.patch
-282
View File
@@ -1,282 +0,0 @@
From 26a43f7047ab933a4afb5ebdd26d5ba9e51e58a3 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 11:50:45 +0300
Subject: [PATCH 16/17] Sample HIP kernel events without per-launch completion
waits
---
zluda/src/impl/context.rs | 4 +-
zluda/src/impl/d4r_profile.rs | 204 ++++++++++++++++++++++++++++------
zluda/src/impl/module.rs | 1 +
3 files changed, 171 insertions(+), 38 deletions(-)
diff --git a/zluda/src/impl/context.rs b/zluda/src/impl/context.rs
index 9ccb372..374d383 100644
--- a/zluda/src/impl/context.rs
+++ b/zluda/src/impl/context.rs
@@ -165,7 +165,7 @@ pub(crate) fn set_cache_config(config: CUfunc_cache) -> CUresult {
}
pub(crate) fn synchronize() -> hipError_t {
- unsafe { hipDeviceSynchronize() }
+ unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
}
pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
@@ -177,7 +177,7 @@ pub(crate) fn synchronize_v2(ctx: &Context) -> hipError_t {
} else {
None
};
- unsafe { hipDeviceSynchronize() }
+ unsafe { hipDeviceSynchronize()?; super::d4r_profile::collect_completed() }
}
struct SetDeviceOnDrop(i32);
diff --git a/zluda/src/impl/d4r_profile.rs b/zluda/src/impl/d4r_profile.rs
index 33ba5f6..1b9d248 100644
--- a/zluda/src/impl/d4r_profile.rs
+++ b/zluda/src/impl/d4r_profile.rs
@@ -1,15 +1,29 @@
-//! Optional, deliberately serializing HIP-event diagnostics. Not a production
-//! scheduler or a claim about application FPS / measured hardware occupancy.
+//! Optional HIP-event diagnostics. Deferred mode never waits for its events,
+//! skips the legacy default stream and bounds outstanding events per thread.
+//! Timestamp insertion still adds overhead; this is not a production scheduler.
use hip_runtime_sys::*;
-use std::{ffi::CStr, ptr, time::Instant};
+use std::{cell::RefCell, ffi::{CStr, CString}, ptr, sync::OnceLock, time::Instant};
-pub(super) struct Launch {
- start: hipEvent_t,
- end: hipEvent_t,
- host: Instant,
+#[derive(Clone, Copy)]
+enum Mode { Off, Serial, Deferred(u64) }
+fn mode() -> Mode {
+ static MODE: OnceLock<Mode> = OnceLock::new();
+ *MODE.get_or_init(|| {
+ if std::env::var_os("D4R_ZLUDA_PROFILE_DEFERRED").is_some() {
+ let every = std::env::var("D4R_ZLUDA_PROFILE_EVERY").ok()
+ .and_then(|v| v.parse::<u64>().ok()).filter(|n| *n > 0).unwrap_or(17);
+ Mode::Deferred(every)
+ } else if std::env::var_os("D4R_ZLUDA_PROFILE").is_some() { Mode::Serial }
+ else { Mode::Off }
+ })
+}
+fn allow_legacy() -> bool {
+ static ALLOW: OnceLock<bool> = OnceLock::new();
+ *ALLOW.get_or_init(|| std::env::var_os("D4R_ZLUDA_PROFILE_ALLOW_LEGACY").is_some())
}
-impl Drop for Launch {
+struct Events { start: hipEvent_t, end: hipEvent_t }
+impl Drop for Events {
fn drop(&mut self) {
unsafe {
if !self.end.is_null() { let _ = hipEventDestroy(self.end); }
@@ -17,44 +31,162 @@ impl Drop for Launch {
}
}
}
+impl Events {
+ unsafe fn new() -> Result<Self, hipErrorCode_t> {
+ let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut() };
+ hipEventCreateWithFlags(&mut result.start, 0)?;
+ hipEventCreateWithFlags(&mut result.end, 0)?;
+ Ok(result)
+ }
+}
+
+struct Description {
+ name: CString, backend: String, phase: String, grid: [u32;3], block: [u32;3],
+ shared: u32, lds: i32, regs: i32, private: i32, max_threads: i32, blocks_per_multiprocessor: i32,
+}
+impl Description {
+ unsafe fn new(function: hipFunction_t, name: &CStr, backend: &str, phase: &str,
+ grid: [u32;3], block: [u32;3], shared: u32) -> Result<Self, hipErrorCode_t> {
+ let mut d = Self { name: name.to_owned(), backend: backend.to_owned(), phase: phase.to_owned(),
+ grid, block, shared, lds: 0, regs: 0, private: 0, max_threads: 0, blocks_per_multiprocessor: 0 };
+ // Copy metadata while the function/module is alive. Collection never
+ // dereferences a function handle that the application may have unloaded.
+ hipFuncGetAttribute(&mut d.lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut d.regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
+ hipFuncGetAttribute(&mut d.private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
+ hipFuncGetAttribute(&mut d.max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
+ let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
+ .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
+ hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut d.blocks_per_multiprocessor,
+ function, threads as i32, shared as usize)?;
+ Ok(d)
+ }
+ fn print(&self, gpu_ms: f32, enqueue_ms: f64, completion_ms: f64, serializing: bool, stream: usize) {
+ if !gpu_ms.is_finite() || gpu_ms < 0. {
+ eprintln!("D4R_KERNEL_PROFILE_INVALID kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} reason=invalid_elapsed_time serializing={} deferred={} stream=0x{stream:x}",
+ self.name, self.backend, self.phase, u32::from(serializing), u32::from(!serializing));
+ return;
+ }
+ eprintln!("D4R_KERNEL_PROFILE kernel={:?} backend={} phase={} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={} serializing={} deferred={} stream=0x{stream:x} legacy_stream={}",
+ self.name, self.backend, self.phase, self.grid[0], self.grid[1], self.grid[2],
+ self.block[0], self.block[1], self.block[2], self.lds, self.shared, self.regs,
+ self.private, self.max_threads, self.blocks_per_multiprocessor,
+ u32::from(serializing), u32::from(!serializing), u32::from(stream <= 1));
+ }
+}
+
+struct Pending { launch: Launch, description: Description, enqueue_ms: f64, stream: usize }
+const CAPACITY: usize = 64;
+struct Deferred {
+ calls: u64, skipped: u64, pending: Vec<Pending>, free: Vec<(i32, Events)>,
+}
+thread_local! {
+ static DEFERRED: RefCell<Deferred> = RefCell::new(Deferred {
+ calls: 0, skipped: 0, pending: Vec::new(), free: Vec::new(),
+ });
+}
+
+impl Deferred {
+ unsafe fn collect(&mut self, device: i32) -> hipError_t {
+ let mut index = 0;
+ while index < self.pending.len() {
+ let p = &self.pending[index];
+ if p.launch.device != device { index += 1; continue; }
+ match hipEventQuery(p.launch.events.end) {
+ Err(e) if e == hipErrorCode_t::NotReady => { index += 1; continue; }
+ result => result?,
+ }
+ let mut gpu_ms = 0.;
+ hipEventElapsedTime(&mut gpu_ms, p.launch.events.start, p.launch.events.end)?;
+ p.description.print(gpu_ms, p.enqueue_ms, p.launch.host.elapsed().as_secs_f64() * 1000., false, p.stream);
+ let p = self.pending.swap_remove(index);
+ // Re-record only after the old timestamps have been collected.
+ self.free.push((device, p.launch.events));
+ }
+ Ok(())
+ }
+}
+
+/// Called only after the application's existing context synchronization, or
+/// before a selected launch. No event/stream/device wait is added here.
+pub(super) unsafe fn collect_completed() -> hipError_t {
+ if !matches!(mode(), Mode::Deferred(_)) { return Ok(()); }
+ DEFERRED.with(|state| {
+ let mut state = state.borrow_mut();
+ if state.pending.is_empty() { return Ok(()); }
+ let mut device = 0; hipGetDevice(&mut device)?;
+ state.collect(device)
+ })
+}
+
+pub(super) struct Launch {
+ events: Events,
+ host: Instant,
+ device: i32,
+ deferred: bool,
+}
impl Launch {
pub(super) unsafe fn begin(stream: hipStream_t) -> Result<Option<Self>, hipErrorCode_t> {
- if std::env::var_os("D4R_ZLUDA_PROFILE").is_none() { return Ok(None); }
- let mut result = Self { start: ptr::null_mut(), end: ptr::null_mut(), host: Instant::now() };
- hipEventCreateWithFlags(&mut result.start, 0)?;
- hipEventCreateWithFlags(&mut result.end, 0)?;
- hipEventRecord(result.start, stream)?;
- result.host = Instant::now();
- Ok(Some(result))
+ let mode = mode();
+ if matches!(mode, Mode::Off) { return Ok(None); }
+ if let Mode::Deferred(every) = mode {
+ return DEFERRED.with(|state| {
+ let mut state = state.borrow_mut();
+ state.calls = state.calls.wrapping_add(1);
+ if (state.calls - 1) % every != 0 { return Ok(None); }
+ // HIP timing events on the legacy default stream can order
+ // other streams. Do not introduce that dependency in this mode.
+ if stream.0 as usize <= 1 && !allow_legacy() {
+ state.skipped += 1;
+ if state.skipped.is_power_of_two() {
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=legacy_default_stream count={} serializing=0", state.skipped);
+ }
+ return Ok(None);
+ }
+ let mut device = 0; hipGetDevice(&mut device)?;
+ state.collect(device)?;
+ let mut capture = hipStreamCaptureStatus::hipStreamCaptureStatusNone;
+ hipStreamIsCapturing(stream, &mut capture)?;
+ if capture != hipStreamCaptureStatus::hipStreamCaptureStatusNone {
+ state.skipped += 1;
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=stream_capture count={} serializing=0", state.skipped);
+ return Ok(None);
+ }
+ let events = if let Some(index) = state.free.iter().position(|(d, _)| *d == device) {
+ state.free.swap_remove(index).1
+ } else if state.pending.len() + state.free.len() < CAPACITY {
+ Events::new()?
+ } else {
+ state.skipped += 1;
+ eprintln!("D4R_KERNEL_PROFILE_SKIPPED reason=capacity count={} capacity={CAPACITY} serializing=0", state.skipped);
+ return Ok(None);
+ };
+ hipEventRecord(events.start, stream)?;
+ Ok(Some(Self { events, host: Instant::now(), device, deferred: true }))
+ });
+ }
+ let events = Events::new()?;
+ hipEventRecord(events.start, stream)?;
+ Ok(Some(Self { events, host: Instant::now(), device: -1, deferred: false }))
}
pub(super) unsafe fn finish(self, function: hipFunction_t, name: &CStr, backend: &str,
phase: &str, grid: [u32;3], block: [u32;3], shared: u32, stream: hipStream_t) -> hipError_t {
let enqueue_ms = self.host.elapsed().as_secs_f64() * 1000.;
- hipEventRecord(self.end, stream)?;
- hipEventSynchronize(self.end)?;
+ hipEventRecord(self.events.end, stream)?;
+ let description = Description::new(function, name, backend, phase, grid, block, shared)?;
+ if self.deferred {
+ DEFERRED.with(|state| state.borrow_mut().pending.push(Pending {
+ launch: self, description, enqueue_ms, stream: stream.0 as usize,
+ }));
+ return Ok(());
+ }
+ hipEventSynchronize(self.events.end)?;
let completion_ms = self.host.elapsed().as_secs_f64() * 1000.;
let mut gpu_ms = 0.;
- hipEventElapsedTime(&mut gpu_ms, self.start, self.end)?;
- // Module function handles use the Driver API attribute query. The
- // plural Runtime API expects a registered host function address.
- let mut lds = 0;
- let mut regs = 0;
- let mut private = 0;
- let mut max_threads = 0;
- hipFuncGetAttribute(&mut lds, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, function)?;
- hipFuncGetAttribute(&mut regs, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_NUM_REGS, function)?;
- hipFuncGetAttribute(&mut private, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, function)?;
- hipFuncGetAttribute(&mut max_threads, hipFunction_attribute::HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function)?;
- let threads = block.iter().try_fold(1u32, |a, b| a.checked_mul(*b))
- .filter(|n| *n <= i32::MAX as u32).ok_or(hipErrorCode_t::InvalidValue)?;
- let mut blocks_per_multiprocessor = 0;
- hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(&mut blocks_per_multiprocessor,
- function, threads as i32, shared as usize)?;
- eprintln!("D4R_KERNEL_PROFILE kernel={name:?} backend={backend} phase={phase} gpu_ms={gpu_ms:.6} enqueue_ms={enqueue_ms:.6} completion_ms={completion_ms:.6} grid={},{},{} block={},{},{} static_lds_bytes={} dynamic_lds_bytes={shared} regs={} private_bytes={} max_threads={} predicted_blocks_per_multiprocessor={blocks_per_multiprocessor} serializing=1",
- grid[0], grid[1], grid[2], block[0], block[1], block[2], lds,
- regs, private, max_threads);
+ hipEventElapsedTime(&mut gpu_ms, self.events.start, self.events.end)?;
+ description.print(gpu_ms, enqueue_ms, completion_ms, true, stream.0 as usize);
Ok(())
}
}
diff --git a/zluda/src/impl/module.rs b/zluda/src/impl/module.rs
index 232f7d4..8d43abf 100644
--- a/zluda/src/impl/module.rs
+++ b/zluda/src/impl/module.rs
@@ -503,6 +503,7 @@ fn zluda_build_version() -> &'static str {
&& !matches!(
key.as_str(),
"D4R_ZLUDA_NATIVE_DIR" | "D4R_ZLUDA_DIR" | "D4R_ZLUDA_DUMP_DIR" | "D4R_ZLUDA_DUMP_PTX_DIR" | "D4R_ZLUDA_LIBCUDA" | "D4R_ZLUDA_VERBOSE" | "D4R_ZLUDA_PROFILE"
+ | "D4R_ZLUDA_PROFILE_API" | "D4R_ZLUDA_PROFILE_DEFERRED" | "D4R_ZLUDA_PROFILE_EVERY" | "D4R_ZLUDA_PROFILE_ALLOW_LEGACY"
| "D4R_ZLUDA_WMMA_FP8_NATIVE" // keyed by effect (ExtraCacheAttributes)
)
})
--
2.55.0.windows.5
patches/zluda/0017-asynchronous-2d-array-copies.patch
-50
View File
@@ -1,50 +0,0 @@
From 67127dde599879ada022d6439383d682e09e0b36 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 11:50:45 +0300
Subject: [PATCH 17/17] Implement asynchronous 2D array copies for Windows
input batching
---
zluda/src/impl/memory.rs | 13 +++++++++++++
zluda/src/lib.rs | 1 +
2 files changed, 14 insertions(+)
diff --git a/zluda/src/impl/memory.rs b/zluda/src/impl/memory.rs
index b724942..42facb3 100644
--- a/zluda/src/impl/memory.rs
+++ b/zluda/src/impl/memory.rs
@@ -469,6 +469,19 @@ pub(crate) unsafe fn copy_2d_v2(memcpy: hip_Memcpy2D) -> CUresult {
Ok(())
}
+pub(crate) unsafe fn copy_2d_async_v2(memcpy: hip_Memcpy2D, stream: hipStream_t) -> CUresult {
+ // The format-80 emulation in copy_2d_v2 uses temporary CPU storage.
+ // Reject it explicitly here: an asynchronous copy cannot outlive those
+ // temporaries, and the Windows fast path uses canonical float arrays.
+ let src_packed = memcpy.srcMemoryType == hipMemoryType::hipMemoryTypeArray
+ && array::original_format(memcpy.srcArray)? == Some(80);
+ let dst_packed = memcpy.dstMemoryType == hipMemoryType::hipMemoryTypeArray
+ && array::original_format(memcpy.dstArray)? == Some(80);
+ if src_packed || dst_packed { return Err(CUerror::NOT_SUPPORTED); }
+ hipMemcpyParam2DAsync(&memcpy, stream)?;
+ Ok(())
+}
+
pub(crate) unsafe fn copy_2d_unaligned_v2(memcpy: hip_Memcpy2D) -> CUresult {
hipDrvMemcpy2DUnaligned(&memcpy)?;
Ok(())
diff --git a/zluda/src/lib.rs b/zluda/src/lib.rs
index 31bfa42..939981e 100644
--- a/zluda/src/lib.rs
+++ b/zluda/src/lib.rs
@@ -150,6 +150,7 @@ cuda_macros::cuda_function_declarations!(
cuMemAlloc_v2,
cuMemAllocPitch_v2,
cuMemcpy2D_v2,
+ cuMemcpy2DAsync_v2,
cuMemcpy2DUnaligned_v2,
cuMemcpy3D_v2,
cuMemcpyAsync,
--
2.55.0.windows.5
patches/zluda/0018-opt-in-rdna-wgp-placement.patch
-93
View File
@@ -1,93 +0,0 @@
From fb0adc8714f57647320f0601533ec50331e378b0 Mon Sep 17 00:00:00 2001
From: xdfnx-dev <[email protected]>
Date: Fri, 2 Oct 2026 14:40:56 +0300
Subject: [PATCH] Add opt-in RDNA CU/WGP placement for matched native
experiments
---
llvm_zluda/src/compile.rs | 42 ++++++++++++++++++++++++++++++++++-----
ptx/src/pass/llvm/emit.rs | 3 ++-
2 files changed, 39 insertions(+), 6 deletions(-)
diff --git a/llvm_zluda/src/compile.rs b/llvm_zluda/src/compile.rs
index 90c0034..6a2f452 100644
--- a/llvm_zluda/src/compile.rs
+++ b/llvm_zluda/src/compile.rs
@@ -24,6 +24,37 @@ const OCKL_MODULE: &[u8] = include_bytes!("device-libs/ockl.bc");
// https://llvm.org/docs/AMDGPUUsage.html#address-spaces
const CONSTANT_ADDRESS_SPACE: u32 = 4;
+/// Opt-in RDNA scheduling experiment; changes placement, not FP semantics.
+pub fn wgp_requested() -> bool {
+ std::env::var("D4R_ZLUDA_WGP").is_ok_and(|value| value.trim() == "1")
+}
+
+fn target_features(wave64: bool, wgp: bool) -> &'static CStr {
+ match (wave64, wgp) {
+ (false, false) => c"-wavefrontsize64,+cumode",
+ (true, false) => c"-wavefrontsize32,+wavefrontsize64,+cumode",
+ (false, true) => c"-wavefrontsize64,-cumode",
+ (true, true) => c"-wavefrontsize32,+wavefrontsize64,-cumode",
+ }
+}
+
+#[cfg(test)]
+mod shader_mode_tests {
+ use super::target_features;
+
+ #[test]
+ fn wgp_changes_only_cu_placement_in_both_wave_modes() {
+ for wave64 in [false, true] {
+ let cu = target_features(wave64, false).to_str().unwrap();
+ let wgp = target_features(wave64, true).to_str().unwrap();
+ assert_eq!(cu.replace("+cumode", "-cumode"), wgp);
+ assert_eq!(cu.contains("+wavefrontsize64"), wave64);
+ assert_eq!(wgp.contains("+wavefrontsize64"), wave64);
+ }
+ assert_eq!(target_features(false, false), c"-wavefrontsize64,+cumode");
+ }
+}
+
fn load_module(ctx: &Context, bc: &[u8], name: &std::ffi::CStr) -> Result<Module, String> {
let module = Module::try_from_bitcode(ctx, bc, Some(name))
.ok_or(("Failed to parse bitcode").to_string())?;
@@ -229,11 +260,12 @@ pub fn compile(
let cpu = std::ffi::CString::new(gcn_arch).map_err(|_| ("invalid gcn_arch").to_string())?;
// functions without their own target features (the linked helper libraries) take these, so under
// D4R_ZLUDA_WAVE64 they must be optimised as wave64 too (lane-id and ballot folding)
- let features = if std::env::var("D4R_ZLUDA_WAVE64").map_or(false, |v| v.trim() == "1") {
- c"-wavefrontsize32,+wavefrontsize64,+cumode"
- } else {
- c"-wavefrontsize64,+cumode"
- };
+ let wgp = wgp_requested();
+ if wgp && !gcn_arch.starts_with("gfx11") && !gcn_arch.starts_with("gfx12") {
+ return Err(format!("D4R_ZLUDA_WGP requires RDNA3/4, got {gcn_arch}"));
+ }
+ let features = target_features(
+ std::env::var("D4R_ZLUDA_WAVE64").is_ok_and(|v| v.trim() == "1"), wgp);
let mut target = unsafe { std::mem::zeroed() };
let mut err = std::ptr::null_mut();
diff --git a/ptx/src/pass/llvm/emit.rs b/ptx/src/pass/llvm/emit.rs
index 6af2495..8b67c93 100644
--- a/ptx/src/pass/llvm/emit.rs
+++ b/ptx/src/pass/llvm/emit.rs
@@ -397,12 +397,13 @@ impl<'a, 'input> ModuleEmitContext<'a, 'input> {
fn emit_target_features(&mut self, fn_: LLVMValueRef) {
let value = format!(
- "{},+cumode{}",
+ "{},{}{}",
if crate::pass::wave64_requested() {
"-wavefrontsize32,+wavefrontsize64"
} else {
"+wavefrontsize32,-wavefrontsize64"
},
+ if llvm_zluda::compile::wgp_requested() { "-cumode" } else { "+cumode" },
if cfg!(debug_assertions) {
",+precise-memory"
} else {
--
2.55.0.windows.5