zluda: coalesce pair shadow stores, signed native FP8 conversion

Patch 0011 merges a complete accumulator-register pair overwrite into one
shadow-layout conversion (reads, rewritten shadow accesses and block
boundaries prevent it; partial writes keep read/modify/write), and requests
eight waves per SIMD for gfx12 native-FP8 functions. Patch 0012 keeps signs
in the native conversion: finite overflow clamped, NaNs mapped to signed
infinity, post-conversion sign masks gone; plain and ReLU remain
exhaustive-table identical to the software codec.

Eighty-frame harness at 720p->4K, excluding the first twenty: control
24.764 ms, coalescing 23.340 ms, plus eight waves 20.135 ms, plus signed
conversion 16.234 ms (15.144 ms with the harness idle gap removed); all
eighty saved frames byte-identical to the matching control. Twelve- and
ten-wave budgets regress (25.56/24.61 ms); fast math changes the image
(18.92 ms) and is not selected.

patches/zluda/0011-gfx12-shadow-store-coalescing.patch

@@ -0,0 +1,92 @@
+diff --git a/llvm_zluda/src/lib.cpp b/llvm_zluda/src/lib.cpp
+--- a/llvm_zluda/src/lib.cpp
++++ b/llvm_zluda/src/lib.cpp
+@@ -485,6 +485,8 @@
+     bool AnyFp8 = false;
+     for (const Site &S : Sites)
+         AnyFp8 |= S.Fp8;
++    if (PackedFp8 && AnyFp8)
++        F.addFnAttr("amdgpu-waves-per-eu", "8");
+     Type *F32Ty = Type::getFloatTy(F.getContext());
+     Value *Scale = ConstantFP::get(F32Ty, AnyFp8 ? 0x1p-16 : 1.0);
+     Value *Inverse = ConstantFP::get(F32Ty, AnyFp8 ? 0x1p16 : 1.0);
+@@ -758,6 +760,70 @@
+         Call->eraseFromParent();
+     }
+ 
++    // A complete pair overwrite needs one conversion. Reads of either
++    // NVIDIA register, accesses to its rewritten shadow, and block boundaries
++    // prevent coalescing. Scan after MMA rewriting to include hoisted reads.
++    DenseMap<AllocaInst *, int> ShadowOwners;
++    for (size_t Index = 0; Index < Shadows.size(); ++Index)
++        if (Shadows[Index] != nullptr)
++            ShadowOwners[Shadows[Index]] = static_cast<int>(Index);
++    SmallVector<std::pair<StoreInst *, StoreInst *>, 64> StorePairs;
++    DenseMap<int, StoreInst *> PendingStores;
++    for (BasicBlock &Block : F)
++    {
++        PendingStores.clear();
++        for (Instruction &I : Block)
++        {
++            auto *Load = dyn_cast<LoadInst>(&I);
++            auto *Store = dyn_cast<StoreInst>(&I);
++            if (Load == nullptr && Store == nullptr)
++                continue;
++            Value *Pointer = Load != nullptr ? Load->getPointerOperand() : Store->getPointerOperand();
++            auto *Register = dyn_cast<AllocaInst>(Pointer);
++            auto Shadow = ShadowOwners.find(Register);
++            if (Shadow != ShadowOwners.end())
++            {
++                PendingStores.erase(Shadow->second);
++                continue;
++            }
++            auto Found = Owner.find(Register);
++            if (Found == Owner.end() || Found->second < 0 || Shadows[Found->second] == nullptr)
++                continue;
++            int Index = Found->second;
++            if (Load != nullptr)
++            {
++                PendingStores.erase(Index);
++                continue;
++            }
++            auto Previous = PendingStores.find(Index);
++            if (Previous != PendingStores.end() && Previous->second->getPointerOperand() != Pointer)
++            {
++                StorePairs.push_back({Previous->second, Store});
++                PendingStores.erase(Previous);
++            }
++            else
++                PendingStores[Index] = Store;
++        }
++    }
++    DenseSet<StoreInst *> CoalescedStores;
++    for (auto [First, Second] : StorePairs)
++    {
++        if (ResultStores.contains(First) || ResultStores.contains(Second))
++            continue;
++        auto *Register = cast<AllocaInst>(First->getPointerOperand());
++        const RegisterPair &Pair = Shadowed[Owner[Register]];
++        const bool FirstHigh = Register == Pair.High;
++        IRBuilder<> Builder(Second);
++        Value *Nv = UndefValue::get(Pair32);
++        Nv = Builder.CreateInsertElement(Nv, First->getValueOperand(), Builder.getInt8(FirstHigh));
++        Nv = Builder.CreateInsertElement(Nv, Second->getValueOperand(), Builder.getInt8(!FirstHigh));
++        Builder.CreateStore(fromNv(Builder, Nv), shadowFor(Pair));
++        CoalescedStores.insert(First);
++        CoalescedStores.insert(Second);
++        First->eraseFromParent();
++        Second->eraseFromParent();
++    }
++
+     // Every other access now goes through the shadow.
+     for (LoadInst *Load : Loads)
+     {
+@@ -770,6 +836,8 @@
+     }
+     for (StoreInst *Store : Stores)
+     {
++        if (CoalescedStores.contains(Store))
++            continue;
+         if (!ResultStores.contains(Store))
+         {
+             auto *Register = cast<AllocaInst>(Store->getPointerOperand());

patches/zluda/0012-gfx12-signed-fp8-conversion.patch

@@ -0,0 +1,34 @@
+diff --git a/ptx/lib/zluda_ptx_impl.cpp b/ptx/lib/zluda_ptx_impl.cpp
+--- a/ptx/lib/zluda_ptx_impl.cpp
++++ b/ptx/lib/zluda_ptx_impl.cpp
+@@ -877,21 +877,18 @@
+         typedef _Float16 half_vector __attribute__((ext_vector_type(2)));
+         typedef uint16_t magnitude_vector __attribute__((ext_vector_type(2)));
+         const uint32_t bits = std::bit_cast<uint32_t>(input);
+-        // Positive half encodings are ordered by magnitude. Clamp both lanes
+-        // with one packed integer min, then retain NaNs as positive quiet NaNs.
+-        // Restore original signs after conversion, including negative zero.
+         const magnitude_vector magnitude = std::bit_cast<magnitude_vector>(bits & 0x7fff7fffu);
+-        const magnitude_vector saturated =
++        // Clamp finite overflow and infinities explicitly. Map NaNs to signed
++        // infinity: native infinity encodes as 0x7f/0xff, whereas native NaN is
++        // always 0xff. This preserves the sign without post-conversion masks.
++        const magnitude_vector bounded =
+             __builtin_elementwise_min(magnitude, magnitude_vector{0x5f00u, 0x5f00u});
+         const magnitude_vector kept{
+-            magnitude[0] > 0x7c00u ? uint16_t(0x7e00u) : saturated[0],
+-            magnitude[1] > 0x7c00u ? uint16_t(0x7e00u) : saturated[1]};
+-        const half_vector halves = std::bit_cast<half_vector>(kept);
+-        const float x = float(halves[0]), y = float(halves[1]);
+-        const uint32_t encoded = d4r_cvt_pk_fp8_f32(x, y, 0, false);
+-        // AMD canonicalizes NaNs to 0xff. PTX retains each input's sign.
+-        const uint16_t packed = uint16_t((encoded & 0x7f7fu) |
+-                                        ((bits >> 8) & 0x80u) | ((bits >> 16) & 0x8000u));
++            magnitude[0] > 0x7c00u ? uint16_t(0x7c00u) : bounded[0],
++            magnitude[1] > 0x7c00u ? uint16_t(0x7c00u) : bounded[1]};
++        const uint32_t signed_bits = std::bit_cast<uint32_t>(kept) | (bits & 0x80008000u);
++        const half_vector halves = std::bit_cast<half_vector>(signed_bits);
++        const uint16_t packed = uint16_t(d4r_cvt_pk_fp8_f32(float(halves[0]), float(halves[1]), 0, false));
+         return std::bit_cast<Fp8x2>(packed);
+     }
+