From a4dd4275fb63f90f4a13c21e46b541de7d423113 Mon Sep 17 00:00:00 2001 From: Tk-Glitch Date: Sun, 18 Oct 2026 21:55:00 +0200 Subject: [PATCH 01/19] engine: accept R32_UINT depth gameDepthFormat now admits VK_FORMAT_R32_UINT alongside the existing float/UNORM depth formats. The RR engine's hardwareDepth path (DLSS.Use.HW.Depth) already handles the nonlinear interpretation; the format check was the only gate keeping the feature on the CUDA fallback. Fixes Cyberpunk 2077 with RR enabled falling back to the CUDA path. --- engine/game.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/game.cpp b/engine/game.cpp index c983b35..cc22d14 100644 --- a/engine/game.cpp +++ b/engine/game.cpp @@ -22,7 +22,7 @@ bool gameMotionFormat(VkFormat format) { } bool gameDepthFormat(VkFormat format, VkImageAspectFlags aspect) { if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT) return depthAspect(format); - return format == VK_FORMAT_R32_SFLOAT || format == VK_FORMAT_R16_SFLOAT || format == VK_FORMAT_R16_UNORM; + return format == VK_FORMAT_R32_SFLOAT || format == VK_FORMAT_R32_UINT || format == VK_FORMAT_R16_SFLOAT || format == VK_FORMAT_R16_UNORM; } bool gameOutputFormat(VkFormat format) { return gameColorFormat(format); } From 474ffad227f9fd59de67d2e48f41389064dd3db2 Mon Sep 17 00:00:00 2001 From: Tk-Glitch Date: Mon, 19 Oct 2026 05:05:23 +0200 Subject: [PATCH 02/19] engine/game_rr: per-input ring-buffer detection to prevent descriptor set cache overflow The Ray Reconstruction engine caches one immutable descriptor set per unique combination of input views (hard limit: 16). Games that rotate their intermediate textures through a ring buffer generate a new combination each frame; with K independent ring buffers of size N the number of distinct combinations grows as N^K, quickly exceeding the cache limit. Fix: track the number of distinct VkImageView handles seen per input (11 inputs: colour, motion, depth, 8 guides). When an input exceeds 4 distinct views it is flagged as rotating and its data is copied to a canonical plane (a stable internal texture) instead of being borrowed directly. The `direct` lambda now takes an index and checks the per-input flag, so only the inputs that actually rotate incur a blit. Stable inputs (single view, no ring buffer) continue to be borrowed directly with zero extra cost. The worst-case overhead is one blit per rotating input per frame (~0.05 ms each). The flag is one-way: once an input is marked as rotating it stays that way for the lifetime of the engine. If a game changes its rendering mode mid-session the flag persists, adding a negligible number of extra blits. --- engine/game_rr.cpp | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/engine/game_rr.cpp b/engine/game_rr.cpp index 79aa379..f3a9bac 100644 --- a/engine/game_rr.cpp +++ b/engine/game_rr.cpp @@ -1,5 +1,6 @@ #include "game_rr.h" #include "game_internal.h" +#include #include #include #include @@ -15,6 +16,12 @@ struct GameDenoiser::Impl { game_detail::CanonicalInputs fallback; // colour, motion, depth std::array guides; // the guides and optional inputs, when one cannot be sampled directly bool started = false; + // Per-input ring-buffer detection: tracks distinct views for each of the 11 inputs + // (color, motion, depth, 8 guides). When an input exceeds the threshold, it is + // copied to a canonical plane so the engine sees a stable set of views. + static constexpr int NUM_INPUTS = 11; + std::array, NUM_INPUTS> seenViews; + std::array needsCopy = {}; // debugging: D4R_ENGINE_RR_DUMP=DIR copies every engine image after each frame into a host-visible buffer and // writes it at the next frame's recording (by then the game has waited for the frame), as DIR/F-NAME.bin VkPhysicalDevice physical; @@ -95,10 +102,28 @@ void GameDenoiser::record(VkCommandBuffer cb, const GameFrameR& frame) { throw std::runtime_error("unsupported game texture format"); // The stages sample every input at its texel coordinates from the origin, so only views at origin zero are // borrowed; anything else is copied first. Every input must be in GENERAL when borrowed (one layout for all). - auto direct = [](const GameImage& image) { - return image.view != VK_NULL_HANDLE && !image.x && !image.y && image.layout == VK_IMAGE_LAYOUT_GENERAL; + // Per-input ring-buffer detection: track distinct views and flag inputs that rotate. + // The engine caches descriptor sets per unique combination of input views (limit 16). + // If K inputs each rotate through N views, there are up to N^K combinations. + // Copying any rotating input to a canonical plane removes it from the combination. + const VkImageView inputViews[11] = { + frame.color.view, frame.motion.view, frame.depth.view, + frame.normals.view, frame.roughness.view, frame.diffuseAlbedo.view, + frame.specularAlbedo.view, frame.specularHitDistance.view, + frame.disocclusionMask.view, frame.subsurfaceGuide.view, frame.colorBeforeParticles.view }; - const bool directColor = direct(frame.color), directMotion = direct(frame.motion), directDepth = direct(frame.depth); + for (int i = 0; i < 11; ++i) { + if (inputViews[i] == VK_NULL_HANDLE) continue; + auto& seen = g.seenViews[i]; + if (std::find(seen.begin(), seen.end(), inputViews[i]) == seen.end()) { + seen.push_back(inputViews[i]); + if (seen.size() > 4) g.needsCopy[i] = true; + } + } + auto direct = [&](const GameImage& image, int idx) { + return !g.needsCopy[idx] && image.view != VK_NULL_HANDLE && !image.x && !image.y && image.layout == VK_IMAGE_LAYOUT_GENERAL; + }; + const bool directColor = direct(frame.color, 0), directMotion = direct(frame.motion, 1), directDepth = direct(frame.depth, 2); // the four guides, then the two optional inputs (kept in their own one-channel formats when copied) const GameImage* guides[8] = {&frame.normals, &frame.roughness, &frame.diffuseAlbedo, &frame.specularAlbedo, &frame.specularHitDistance, &frame.disocclusionMask, &frame.subsurfaceGuide, @@ -114,7 +139,7 @@ void GameDenoiser::record(VkCommandBuffer cb, const GameFrameR& frame) { } g.fallback.record(cb, frame.color, frame.motion, frame.depth, w, h, w, h, directColor, directMotion, directDepth); for (int i = 0; i < 8; ++i) { - if (!present(i) || direct(*guides[i])) continue; + if (!present(i) || direct(*guides[i], 3 + i)) continue; if (!g.guides[i].image) { g.fallback.ensure(cb, g.guides[i], guideFormats[i], w, h); } @@ -132,7 +157,7 @@ void GameDenoiser::record(VkCommandBuffer cb, const GameFrameR& frame) { &images.specularHitDistance, &images.disocclusionMask, &images.subsurfaceGuide, &images.colorBeforeParticles}; for (int i = 0; i < 8; ++i) - if (present(i)) *guideViews[i] = direct(*guides[i]) ? guides[i]->view : g.guides[i].view; + if (present(i)) *guideViews[i] = direct(*guides[i], 3 + i) ? guides[i]->view : g.guides[i].view; const bool directOutput = frame.output.view != VK_NULL_HANDLE && frame.output.format == VK_FORMAT_R16G16B16A16_SFLOAT && frame.output.layout == VK_IMAGE_LAYOUT_GENERAL && !frame.output.x && !frame.output.y && frame.output.image != frame.color.image && frame.output.image != frame.motion.image && frame.output.image != frame.depth.image; From 0aab224003fa85a826ca08b6b5d47db724dac6fe Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:53:40 +1300 Subject: [PATCH 03/19] engine: hardware FP8 saturation where the SR layers' values cannot be infinite layer_m.comp (M / L, M_RELAXED) converts to FP8 with saturatedConvertEXT, which RADV lowers as a NaN-preserving clamp per element and the conversion. As in engine/rr/rr_exact.glsl (FINITE), a value that is never infinite can be written as "an infinity becomes NaN, then saturate", which RADV turns into the hardware conversion with FP16_OVFL (f2e4m3fn_satfn, Mesa >= 25.2): the same result for every finite value and NaN. Only two sources are provably finite and take it (SATF2_ / MK8_F8F): - the attention's f32 sums of FP8 products (o); - the normalised tokens (H1N): the norm sums the squares of the very values it scales, so an infinite channel makes the reciprocal 0 and the product NaN, never infinite. The binary16 accumulators (q, the MLP, the expansion, the residual) can overflow to infinity, or carry an infinite input pixel, and keep the clamp. Per network shader 352 -> 64 clamps (one 384 -> 160), 3-4 % fewer instructions. Output byte-identical (M and L, six temporal frames at 1280x720 -> 1440p and 1114x626 -> 1080p). RX 9070 XT, d4r-m 720p -> 1440p: M unchanged (1579 / 1583 us, its network is not issue-bound), L 2348 -> 2329 us (input 479 -> 471, expansion 400 -> 386). Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/layer_m.comp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/engine/layer_m.comp b/engine/layer_m.comp index 6617327..8ead4dc 100644 --- a/engine/layer_m.comp +++ b/engine/layer_m.comp @@ -213,6 +213,15 @@ HAC bias16(uint off, uint stride) { HAC b; coopMatLoad(b, w16.w, off / 4u, strid // straight from f32 into FP8, two elements per saturating conversion; the activation in f32 as well #define SAT2_(dst, i, v) { fe4m3vec2 t_; saturatedConvertEXT(t_, (v)); (dst)[i] = t_.x; (dst)[(i) + 1] = t_.y; } #define MK8_F8(dst, a) { SAT2_(dst, 0, vec2((a)[0], (a)[1])) SAT2_(dst, 2, vec2((a)[2], (a)[3])) SAT2_(dst, 4, vec2((a)[4], (a)[5])) SAT2_(dst, 6, vec2((a)[6], (a)[7])) } +// SATF2_: the same for a value that is never infinite, written as "an infinity becomes NaN, then saturate", which RADV +// turns into the hardware conversion with FP16_OVFL instead of a NaN-preserving clamp per element and the conversion +// (f2e4m3fn_satfn; the same result for every finite value and NaN, engine/rr/rr_exact.glsl FINITE). Used for the +// attention's f32 sums of FP8 products (MK8_F8F) and the normalised tokens: the norm sums the squares of the very +// values it scales, so an infinite channel makes the reciprocal 0 and the product NaN, never infinite. The binary16 +// accumulators (q, the MLP, the expansion, the residual) can overflow to infinity and keep SAT2_. +vec2 finite2_(vec2 v) { return vec2(isinf(v.x) ? uintBitsToFloat(0x7fc00000u) : v.x, isinf(v.y) ? uintBitsToFloat(0x7fc00000u) : v.y); } +#define SATF2_(dst, i, v) { fe4m3vec2 t_; saturatedConvertEXT(t_, finite2_(v)); (dst)[i] = t_.x; (dst)[(i) + 1] = t_.y; } +#define MK8_F8F(dst, a) { SATF2_(dst, 0, vec2((a)[0], (a)[1])) SATF2_(dst, 2, vec2((a)[2], (a)[3])) SATF2_(dst, 4, vec2((a)[4], (a)[5])) SATF2_(dst, 6, vec2((a)[6], (a)[7])) } #define GELU2_(dst, i, h) { const vec2 x_ = vec2((h)[i], (h)[(i) + 1]); const vec2 y_ = clamp(x_, vec2(-2.0), vec2(2.0)); \ const vec2 g_ = fma(y_, fma(-abs(y_), vec2(0.08108133), vec2(0.41216981)), vec2(0.5)); SAT2_(dst, i, x_ * g_) } #define GELU8(dst, h) { GELU2_(dst, 0, h) GELU2_(dst, 2, h) GELU2_(dst, 4, h) GELU2_(dst, 6, h) } @@ -222,12 +231,13 @@ HAC bias16(uint off, uint stride) { HAC b; coopMatLoad(b, w16.w, off / 4u, strid const float16_t q4_ = CL8(Q16((a)[4])), q5_ = CL8(Q16((a)[5])), q6_ = CL8(Q16((a)[6])), q7_ = CL8(Q16((a)[7])); \ (dst)[0] = floate4m3_t(q0_); (dst)[1] = floate4m3_t(q1_); (dst)[2] = floate4m3_t(q2_); (dst)[3] = floate4m3_t(q3_); \ (dst)[4] = floate4m3_t(q4_); (dst)[5] = floate4m3_t(q5_); (dst)[6] = floate4m3_t(q6_); (dst)[7] = floate4m3_t(q7_); } +#define MK8_F8F MK8_F8 #endif // the normalised tokens into FP8 (inside NM1 / N2: x0..x7 the token, g0 / g1 the gains) #ifdef M_RELAXED #define H1N(N, r_) { const float f_ = float(r_); \ - SAT2_(h1[N], 0, vec2(x0, x1) * (f_ * vec2(g0.xy))) SAT2_(h1[N], 2, vec2(x2, x3) * (f_ * vec2(g0.zw))) \ - SAT2_(h1[N], 4, vec2(x4, x5) * (f_ * vec2(g1.xy))) SAT2_(h1[N], 6, vec2(x6, x7) * (f_ * vec2(g1.zw))) } + SATF2_(h1[N], 0, vec2(x0, x1) * (f_ * vec2(g0.xy))) SATF2_(h1[N], 2, vec2(x2, x3) * (f_ * vec2(g0.zw))) \ + SATF2_(h1[N], 4, vec2(x4, x5) * (f_ * vec2(g1.xy))) SATF2_(h1[N], 6, vec2(x6, x7) * (f_ * vec2(g1.zw))) } #else #define H1N(N, r_) { \ h1[N][0] = floate4m3_t(CL8(x0 * float16_t(r_ * g0.x))); h1[N][1] = floate4m3_t(CL8(x1 * float16_t(r_ * g0.y))); \ @@ -635,7 +645,7 @@ void main() o[0] = coopMatMulAdd(vt0, pb, ACC(0.0)); o[1] = coopMatMulAdd(vt1, pb, ACC(0.0)); R16(o[0]); R16(o[1]); - MK8_F8(OB_[w][0], o[0]); MK8_F8(OB_[w][1], o[1]); + MK8_F8F(OB_[w][0], o[0]); MK8_F8F(OB_[w][1], o[1]); } // the output projection of this head's 32 attention channels, straight into the residual stream #define WO_N(N) { \ From 912bef3fb806d4e19395947ceaf139d3fe9a1800 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:02:04 +1300 Subject: [PATCH 04/19] engine: SR layers keep their V tiles in a buffer that is dead during the attention layer_m.comp's V tiles (VSH, 4 KiB, per wave and head, accessed only as matrices and only during the attention) had their own shared memory. Shared memory caps how many waves this GPU runs, and ACO gives a shader the registers that many waves leave, so the network layers ran at 3 - 6 waves per SIMD with registers to spare. VSH now takes the place of a buffer that is dead during the attention: - the merge rows NB (enc1, enc2, L's input stage), written after it; - the expansion NEX (dec1, dec2), read before it; - HO (L's expansion stage), written after it. One barrier each before the buffer's own use (none was needed while the buffers were apart). The tube keeps its own 4 KiB. | layer | shared memory | vector registers | waves per SIMD | | enc1 | 13 -> 9 KiB | 256 -> 216 | 4 -> 7 | | enc2 | 17 -> 13 KiB | 256 | 3 -> 4 | | dec2 | 13 -> 9 KiB | 256 -> 240 | 4 -> 6 | | dec1 | 10 -> 6 KiB | 240 -> 192 | 6 -> 8 | | L input | 16 -> 12 KiB | 192 -> 144 | 8 -> 10 | | L expansion | 13 -> 9 KiB | 192 -> 96 | 8 -> 14 | Output byte-identical (M and L, six temporal frames at 1280x720 -> 1440p and 1114x626 -> 1080p). RX 9070 XT, d4r-m 720p -> 1440p, with the previous commit: M 1583 -> 1566 us (network 967 -> 951), L 2349 -> 2293 us (input 479 -> 453, expansion 401 -> 382). The tube on one window per wave instead of two (192 registers, 8 waves) is 9 % slower and stays as it is. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/layer_m.comp | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/engine/layer_m.comp b/engine/layer_m.comp index 8ead4dc..36bd42c 100644 --- a/engine/layer_m.comp +++ b/engine/layer_m.comp @@ -366,7 +366,6 @@ float16_t gelu1(float16_t x) precise f16vec2 C_ = f16vec2(EV(T, 4), EV(T, 5)), D_ = f16vec2(EV(T, 6), EV(T, 7)); \ P0_[T] = (A_ * A_) + (B_ * B_); P1_[T] = (C_ * C_) + (D_ * D_); } -shared f16vec4 VSH[4 * 2 * 16 * VWD / 4]; // per wave and head: the V tiles, transposed through shared memory #define MERGE (KIND == 1 || KIND == 3) #define EXPAND (KIND == 2 || KIND == 4) #if MERGE @@ -385,6 +384,34 @@ shared float16_t X0S[64 * XS]; // the embedding's output x0, the bl const uint HS = 40u; shared float16_t HO[64 * HS]; // the block's f16 output, window-major, for the per-pixel head #endif +// VSH: per wave and head the V tiles, transposed through shared memory (4 KiB, 512 bytes a tile). It is only accessed +// as matrices, and only during the attention, so it takes the place of a buffer that is dead then: the merge rows +// (written after the attention), the expansion (read before it) or, for KIND 4, the head's input (written after it). +// Shared memory caps how many waves this GPU runs, and ACO gives a shader the registers that number of waves leaves. +#if MERGE +#define VSH NB +#define VSH_UNIT 1u +#if 32 * (4 * C + 8) < 4096 +#error layer_m.comp: the V tiles no longer fit in NB +#endif +#elif KIND == 2 +#define VSH NEX +#define VSH_UNIT 1u +#if 16 * (4 * C + 8) < 4096 +#error layer_m.comp: the V tiles no longer fit in NEX +#endif +#elif KIND == 4 +#define VSH HO +#define VSH_UNIT 2u +#if 64 * 40 * 2 < 4096 +#error layer_m.comp: the V tiles no longer fit in HO +#endif +#else +shared floate4m3_t VSH[4 * 2 * 16 * VWD * 2]; +#define VSH_UNIT 1u +#endif +#define VSH_AT(t) ((t) * (512u / VSH_UNIT)) // tile t, in the buffer's elements +#define VSH_STRIDE (VWD * 2u / VSH_UNIT) // a row of VWD binary16 #define SYNC memoryBarrierShared(); barrier(); #if KIND == 4 // NVIDIA's f32 activations of the head (rrlite_dec0_4x4's output code): a sigmoid rounded to binary16, and a @@ -552,6 +579,9 @@ void main() H1N(N, rn) } EACH_T(NM1) } +#if KIND == 2 + SYNC // every wave has read the block's input from NEX: the V tiles take its place +#endif // ------------------------------------------------------------ attention, one head at a time for (uint hd = 0u; hd < uint(NH); ++hd) { @@ -603,8 +633,8 @@ void main() // V of every window goes to its shared buffer now, so it does not occupy registers during the scores FORW { const uint mt = mt0 + w; - coopMatStore(v[w][0], VSH, (mt * 2u + 0u) * 64u, VWD / 4u, RM); - coopMatStore(v[w][1], VSH, (mt * 2u + 1u) * 64u, VWD / 4u, RM); + coopMatStore(v[w][0], VSH, VSH_AT(mt * 2u + 0u), VSH_STRIDE, RM); + coopMatStore(v[w][1], VSH, VSH_AT(mt * 2u + 1u), VSH_STRIDE, RM); } } // scores: q8(Q) against itself, rows are the queries and columns the keys, plus the relative position bias @@ -634,8 +664,8 @@ void main() // V through this wave's shared buffer, then O^T = V^T x P^T: rows are the head channels, columns the tokens ACC o[2]; A16 vt0, vt1; - coopMatLoad(vt0, VSH, (mt * 2u + 0u) * 64u, VWD / 4u, RM); - coopMatLoad(vt1, VSH, (mt * 2u + 1u) * 64u, VWD / 4u, RM); + coopMatLoad(vt0, VSH, VSH_AT(mt * 2u + 0u), VSH_STRIDE, RM); + coopMatLoad(vt1, VSH, VSH_AT(mt * 2u + 1u), VSH_STRIDE, RM); #if TEXL // P V is the block's one product of two f16 operands, and its f32 sum over the 16 keys is not exact: ZLUDA's // lowering of m16n8k16 puts key pair (j & 1) * 4 + 2 h + (j >> 1) in K dword j of half h (swin_tok.h @@ -732,6 +762,7 @@ void main() } #if KIND == 4 // ------------------------------------------------------------ L's expansion head: per-pixel reconstruction inputs + SYNC // every wave is done with the V tiles in HO FORW { WINV const uint hb = (mt * 16u + row) * HS + ch8; @@ -773,6 +804,7 @@ void main() #endif // ------------------------------------------------------------ patch merge: 2x2 tokens (4C) -> XC, half resolution #if MERGE + SYNC // every wave is done with the V tiles in NB FORW { WINV // the 16 merged rows are gathered from the four waves: the row is the merged token of the lane's pixel From 0ba235a05b7d123d337321c8b6313b58070fb784 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:10:27 +1300 Subject: [PATCH 05/19] engine: per-layer GPU timestamps for the M / L network (D4R_ENGINE_M_LAYER_STAMPS) Timing only: with D4R_ENGINE_M_LAYER_STAMPS=1 the runtime writes a timestamp after each of the ten network layers (queries first + 7 on) and d4r-m prints the per-layer medians. RX 9070 XT, 1280x720 -> 1440p: enc1 210, enc2 115, tube 6 x 50, dec2 114, dec1 209 us. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/replay_m.cpp | 17 +++++++++++++---- engine/runtime_m.inc | 3 +++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/replay_m.cpp b/engine/replay_m.cpp index 817e22b..d9ff6f7 100644 --- a/engine/replay_m.cpp +++ b/engine/replay_m.cpp @@ -107,7 +107,7 @@ int main(int argc, char** argv) try { CK(vkCreateCommandPool(c.device, &pi, nullptr, &c.pool)); VkCommandBufferAllocateInfo cai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; cai.commandPool = c.pool; cai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; cai.commandBufferCount = 1; VkCommandBuffer cb; CK(vkAllocateCommandBuffers(c.device, &cai, &cb)); - VkQueryPoolCreateInfo qpi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qpi.queryType = VK_QUERY_TYPE_TIMESTAMP; qpi.queryCount = 7; + VkQueryPoolCreateInfo qpi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qpi.queryType = VK_QUERY_TYPE_TIMESTAMP; qpi.queryCount = 17; VkQueryPool qp; CK(vkCreateQueryPool(c.device, &qpi, nullptr, &qp)); VkPhysicalDeviceProperties props; vkGetPhysicalDeviceProperties(c.physical, &props); auto submit = [&] { @@ -151,7 +151,7 @@ int main(int argc, char** argv) try { if (data.size() != sizes[k]) throw std::runtime_error("wrong input size"); frameInputs.push_back(std::move(data)); } - std::vector times, walls; std::vector> stage(6); + std::vector times, walls; std::vector> stage(6), layer(10); const auto now = [] { return std::chrono::duration(std::chrono::steady_clock::now().time_since_epoch()).count(); }; // D4R_M_RESET_ALL=1: every frame is a reset frame, as the D3D12 harness passes them without its temporal option const bool resetAll = std::getenv("D4R_M_RESET_ALL") != nullptr; @@ -169,7 +169,7 @@ int main(int argc, char** argv) try { s.renderWidth = rw; s.renderHeight = rh; const double t0 = now(); engine.setFrame({color.view, motion.view, depth.view}, s); - vkCmdResetQueryPool(cb, qp, 0, 7); + vkCmdResetQueryPool(cb, qp, 0, 17); double networkUs = 0; if (hip) { #ifdef D4R_ENGINE_HIP @@ -186,12 +186,16 @@ int main(int argc, char** argv) try { if (pass == 0) vkCmdCopyImageToBuffer(cb, engine.outputImage(), VK_IMAGE_LAYOUT_GENERAL, out.buffer, 1, ®ion); submit(); if (pass) walls.push_back(now() - t0); - uint64_t t[7]; CK(vkGetQueryPoolResults(c.device, qp, 0, 7, sizeof(t), t, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT)); + uint64_t t[17] = {}; + // D4R_ENGINE_M_LAYER_STAMPS=1: the ten network layers' stamps as well (queries 7 - 16) + static const bool layerStamps = std::getenv("D4R_ENGINE_M_LAYER_STAMPS") != nullptr && !hip; + CK(vkGetQueryPoolResults(c.device, qp, 0, layerStamps ? 17 : 7, sizeof(t), t, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT)); const double tick = props.limits.timestampPeriod / 1000.0; double d[6]; for (int k = 0; k < 6; ++k) d[k] = double(t[k + 1] - t[k]) * tick; if (hip) d[2] = networkUs; // measured HIP network time; the complete span includes host handoffs const double us = double(t[6] - t[0]) * tick; if (pass) for (int k = 0; k < 6; ++k) stage[k].push_back(d[k]); + if (pass && layerStamps) for (int l = 0; l < 10; ++l) layer[l].push_back(double(t[7 + l] - (l ? t[6 + l] : t[2])) * tick); if (pass == 0 && std::getenv("D4R_M_DUMP")) { // development: every intermediate of this frame, to PREFIX-NAME-FRAME.bin for (const auto& d : engine.debugResources()) { @@ -216,6 +220,11 @@ int main(int argc, char** argv) try { std::sort(walls.begin(), walls.end()); std::printf(" wall clock, record to completion: median %.1f us, min %.1f us\n", walls[walls.size() / 2], walls[0]); const char* names[6] = {"exposure", "input", "network", "expansion", "reconstruction", "downsample"}; for (int k = 0; k < 6; ++k) { std::sort(stage[k].begin(), stage[k].end()); std::printf(" %-14s median %.1f us\n", names[k], stage[k][stage[k].size() / 2]); } + if (!layer[0].empty()) { + std::printf(" layers (us):"); + for (auto& v : layer) { std::sort(v.begin(), v.end()); std::printf(" %.1f", v[v.size() / 2]); } + std::printf("\n"); + } } CK(vkDeviceWaitIdle(c.device)); } diff --git a/engine/runtime_m.inc b/engine/runtime_m.inc index 803f18d..12a2e18 100644 --- a/engine/runtime_m.inc +++ b/engine/runtime_m.inc @@ -512,9 +512,12 @@ void MEngine::recordFrame(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first auto& e = *impl; auto& m = e.model; if (e.external) throw std::runtime_error("this engine's network is external: record the front and the back"); recordFront(cb, stamps, first); + // timing only: D4R_ENGINE_M_LAYER_STAMPS=1 also stamps after every network layer, from query first + 7 on + static const bool layerStamps = std::getenv("D4R_ENGINE_M_LAYER_STAMPS") != nullptr; for (unsigned l = 0; l < MLayers; ++l) { const uint32_t w = e.gw / 2 / MScale[l], h = e.gh / 2 / MScale[l]; dispatch(cb, m.network[l], e.layerSets[l], (w + (MShift[l] ^ ((e.frame & 1u) * 2u)) + 7) / 8, (h + MShift[l] + 7) / 8); emptyStage(cb); + if (stamps && layerStamps) vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, stamps, first + 7 + l); } recordBack(cb, stamps, stamps ? first + 3 : 0); } From b7b559ff6abd8c6d58db7bd0fa98a64bbfa6c743 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:34:19 +1300 Subject: [PATCH 06/19] engine: D4R_M_DUMP also writes the M / L network's activations layer by layer d4r-m's D4R_M_DUMP wrote the network's input (tokens) and output (head); it now also writes skip1, merged1, skip2, merged2, tubeA, tubeB and dec2, so that a difference between two builds can be found at the first layer it appears. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/runtime_m.inc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/engine/runtime_m.inc b/engine/runtime_m.inc index 12a2e18..0a55b1c 100644 --- a/engine/runtime_m.inc +++ b/engine/runtime_m.inc @@ -528,6 +528,14 @@ std::vector MEngine::debugResources() const { std::vector r{{"enc0-params", e.enc0P.handle, VK_NULL_HANDLE, e.enc0P.size, 0, 0}, {"tokens", e.tokens.handle, VK_NULL_HANDLE, e.tokens.size, 0, 0}, {"head", e.dec1Out.handle, VK_NULL_HANDLE, e.dec1Out.size, 0, 0}, + // the network's own activations, layer by layer + {"skip1", e.skip1.handle, VK_NULL_HANDLE, e.skip1.size, 0, 0}, + {"merged1", e.merged1.handle, VK_NULL_HANDLE, e.merged1.size, 0, 0}, + {"skip2", e.skip2.handle, VK_NULL_HANDLE, e.skip2.size, 0, 0}, + {"merged2", e.merged2.handle, VK_NULL_HANDLE, e.merged2.size, 0, 0}, + {"tubeA", e.tubeA.handle, VK_NULL_HANDLE, e.tubeA.size, 0, 0}, + {"tubeB", e.tubeB.handle, VK_NULL_HANDLE, e.tubeB.size, 0, 0}, + {"dec2", e.dec2Out.handle, VK_NULL_HANDLE, e.dec2Out.size, 0, 0}, {"gates", VK_NULL_HANDLE, e.gates.handle, VkDeviceSize(w) * h * 4, w, h}, {"mix", VK_NULL_HANDLE, e.mix.handle, VkDeviceSize(w) * h * 4, w, h}, {"cov", VK_NULL_HANDLE, e.cov.handle, VkDeviceSize(w) * h * 8, w, h}, From 9aa402e70fddb55b6d0098a1e5ee39d11656b656 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:08:52 +1300 Subject: [PATCH 07/19] engine/rr: OCML's tanh in f32 where f64 was not needed ptx2rr.py's tanh_ocml (tanh.approx.f32; only preset F's dec0_1 has it) ran every fused operation through f64, which this GPU runs at a small fraction of the f32 rate. They are f32 fma now, with two exceptions that keep every result: - exp's error term fma(w, c, -ph) next to its own product ph = w * c comes out of the driver as that product and a sum, exactly 0, even with precise or opaque operands. It is Dekker's exact product error: w split into halves of 12 bits (w * 4097), log2(e) into 1.4423828125 + 0.0003122091293334961, every partial product exact. - the polynomial below 0.625 stays in f64: the driver always folded the f32 rounding between its four nested f64 steps away (f2f64(f2f32(x)) -> x, the conversions are not NoContraction; seen in the NIR), so the old code computed it in f64 with one rounding at the end. A true f32 polynomial rounds 24,836 arguments one f32 step differently (2 in binary16). A NaN argument keeps its payload, quieted and positive, as the f64 conversion left it. A GPU probe of the old against the new function: identical on all 2^32 arguments, NaN payloads included (with a plain fma for the error term instead: 3,976,308 differ, 490 in binary16). Only dec0_1.spv changes, at every level and with D4R_RR2_SPLIT=1; presets D and E have no tanh.approx.f32 and their modules are byte-identical. Every image of every frame byte-identical to the previous model at levels 0 to 3 (1280x720 -> 1440p, a finite stress sequence with random motion, a reset and a moving camera, and at level 3 also 1114x626 -> 1080p, the NaN / Inf stress inputs and D4R_RR_TEST_ROBUST2=1); a model with tanh's constants perturbed differs in 50 of 90 images, so the comparison sees it. RX 9070 XT, RADV 26.2.2, d4r-rr 720p -> 1440p, interleaved: dec0_1 475 -> 446 us, frame 3.568 -> 3.537 ms; exact build (D4R_RR_RELAXED=0) 4.411 -> 4.388 ms. The polynomial in f32 as well would be 419 us. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/rr/ptx2rr.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/engine/rr/ptx2rr.py b/engine/rr/ptx2rr.py index f269c49..515982d 100755 --- a/engine/rr/ptx2rr.py +++ b/engine/rr/ptx2rr.py @@ -121,31 +121,49 @@ void sust_p1(writeonly image2D s, int x, int y, float v) { sust_p4(s, x, y, vec4 // UNORM 10:10:10:2 surfaces (the pooled histories) are RGBA32F images holding the quantised values, like ZLUDA's. // tanh.approx.f32 as the reference computes it: ROCm's OCML tanh (device-libs ocml/src/tanhF.cl) with LLVM's AMDGPU // lowering of exp (lowerFEXP, fast-FMA path): the same fused operations, hardware exp2, ldexp and reciprocal. -// (a fused f32 a * b + c through f64: the product is exact there and the sum is rounded to 53 bits before the f32 -// rounding. The driver's own f32 fma is a product and a sum whenever its product also exists on its own, as in -// exp's ph / fma(x, c, -ph) pair.) -float fmad(float a, float b, float c) { precise double s = double(a) * double(b) + double(c); return float(s); } +// Up to 2026-10-19 every fused operation went through f64 (the product exact there, the sum rounded to 53 bits, then +// to f32), which this GPU runs at a small fraction of the f32 rate. They are the driver's f32 fma now, with two +// exceptions that keep every result bit for bit what it was, on all 2^32 arguments (RX 9070 XT, RADV; NaN payloads +// included): +// - the polynomial stays in f64: the driver drops the f32 rounding between nested f64 operations (it folds +// f2f64(f2f32(x)) to x; the conversions are not NoContraction), so its four nested steps always ran in f64 with one +// rounding at the end. In f32 (true fma), 24,836 arguments round one f32 step differently, 2 of them in binary16. +// - exp's error term fma(w, c, -ph) next to ph = w * c: the driver makes that the product it already has and a sum, +// exactly 0, even with every operand precise or opaque. It is Dekker's exact product error: w split into two halves +// of 12 bits (w * 4097 = w * (2^12 + 1)), log2(e) into 1.4423828125 + 0.0003122091293334961 (11 and 12 bits), so +// every partial product is exact and so is each sum. +// A NaN argument keeps its payload, quieted and positive, as the f64 conversion left it. float tanh_ocml(float x) { const float y = abs(x); precise float z; if (y < 0.625) { precise float y2 = y * y; - precise float p = fmad(y2, fmad(y2, fmad(y2, fmad(y2, -0.005700020585209131, 0.02063407190144062), -0.053737930953502655), 0.13331416249275208), -0.3333328068256378); + precise double y2d = double(y2); + precise double pd = (((y2d * -0.005700020585209131 + 0.02063407190144062) * y2d - 0.053737930953502655) * y2d + + 0.13331416249275208) * y2d - 0.3333328068256378; + precise float p = float(pd); precise float yp = y * p; - z = fmad(y2, yp, y); + z = fma(y2, yp, y); } else { const float w = 2.0 * y; precise float ph = w * 1.4426950216293335; - precise float f0 = fmad(w, 1.4426950216293335, -ph); - precise float pl = fmad(w, 1.925962855864327e-08, f0); + precise float wt = w * 4097.0; + precise float wh = wt - (wt - w); + precise float wl = w - wh; + precise float f1 = wh * 1.4423828125 - ph; + precise float f2 = f1 + wh * 0.0003122091293334961; + precise float f3 = f2 + wl * 1.4423828125; + precise float f0 = f3 + wl * 0.0003122091293334961; + precise float pl = fma(w, 1.925962855864327e-08, f0); const float e = roundEven(ph); precise float d = ph - e; precise float a = d + pl; float t = ldexp(exp2(a), int(e)); if (w > 88.72283172607422) t = uintBitsToFloat(0x7f800000u); precise float s1 = t + 1.0; - z = fmad(-2.0, rcp_a(s1), 1.0); + z = fma(-2.0, rcp_a(s1), 1.0); + if (isnan(z)) z = uintBitsToFloat(floatBitsToUint(y) | 0x00400000u); } return x < 0.0 || floatBitsToUint(x) == 0x80000000u ? -z : z; } From 9c9145ac02b7cdb3d5ce77acdf972e0d00807a31 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:26:21 +1300 Subject: [PATCH 08/19] engine: D4R_M_DUMP also writes the reconstruction's outputs d4r-m's D4R_M_DUMP now also writes what the reconstruction (post_m.comp) writes: the history (hist, RGBA16F at the reconstruction resolution), its half-resolution mean (half, RGBA32F) and the packed 11-11-10 image the downsample reads (packed). The presented image is the packed one filtered and rounded again, so a change in the history shows up there a frame later, if at all; the dump shows it in the frame it happens. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/runtime_m.inc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/engine/runtime_m.inc b/engine/runtime_m.inc index 0a55b1c..ca63544 100644 --- a/engine/runtime_m.inc +++ b/engine/runtime_m.inc @@ -540,7 +540,12 @@ std::vector MEngine::debugResources() const { {"mix", VK_NULL_HANDLE, e.mix.handle, VkDeviceSize(w) * h * 4, w, h}, {"cov", VK_NULL_HANDLE, e.cov.handle, VkDeviceSize(w) * h * 8, w, h}, {"mvA", VK_NULL_HANDLE, e.mvA[1 - e.current].handle, VkDeviceSize(w) * h * 4, w, h}, - {"feat", VK_NULL_HANDLE, e.feat[1 - e.current].handle, VkDeviceSize(w) * h * 4, w, h}}; + {"feat", VK_NULL_HANDLE, e.feat[1 - e.current].handle, VkDeviceSize(w) * h * 4, w, h}, + // the reconstruction's outputs: the history, its half-resolution mean and the packed + // image the downsample reads + {"hist", VK_NULL_HANDLE, e.hist[1 - e.current].handle, VkDeviceSize(e.hw) * e.hh * 8, e.hw, e.hh}, + {"half", VK_NULL_HANDLE, e.half[1 - e.current].handle, VkDeviceSize(e.hw / 2) * (e.hh / 2) * 16, e.hw / 2, e.hh / 2}, + {"packed", e.packed.handle, VK_NULL_HANDLE, e.packed.size, 0, 0}}; if (e.model.presetL) r.push_back({"skip", e.skipL.handle, VK_NULL_HANDLE, e.skipL.size, 0, 0}); return r; } From 8c0635242b9023c96724354cff59f7cedbc1d405 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:31:40 +1300 Subject: [PATCH 09/19] engine: SR reconstruction samples the history once when both candidates have one vector post_m.comp (presets M and L) blends the history along the two most likely of three motion candidates and sampled it once per candidate (five bilinear taps; L also expands each sample to linear light). Where the motion is uniform the two candidates usually have the same vector: the same position, so the same bicubic sample or the same fallback colour (a NaN vector never compares equal, +0 and -0 give the same position). That case now takes the first candidate's value. Every output frame byte-identical (M and L, HDR and LDR; 1280x720 -> 1440p, 1114x626 -> 1080p for the compact tile tier, 1920x1080 -> 1440p for the 15x15 tier, the stress inputs), and so is everything D4R_M_DUMP writes, the history, half history and packed image included. Taking 0.99 times the first sample there instead changes every frame of every input, the reset frames too. Only post_m*.spv / post_l*.spv change. RX 9070 XT, d4r-m 720p -> 1440p, interleaved, reconstruction stage: uniform motion 325 -> 288 us (L 401 -> 354), compact tier at 1114x626 -> 1080p 219 -> 193 (L 275 -> 244); random motion unchanged (377 -> 374, L 404 -> 401). Frame 1570 -> 1541 us (L 2299 -> 2249) on the uniform-motion frames. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/post_m.comp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/engine/post_m.comp b/engine/post_m.comp index ab02dc6..f0243e2 100644 --- a/engine/post_m.comp +++ b/engine/post_m.comp @@ -214,6 +214,9 @@ void main() // the fallback colour is only fetched where the history sample leaves the image or the history was reset: // the value is the same either way, so computing it in the branch saves the fetch on ordinary pixels const bool out1 = q1.x < 0.0 || q1.y < 0.0 || q1.x > size.x || q1.y > size.y || p.reset != 0u; + // the second candidate often has the first one's vector (where the motion is uniform): the same position, + // so the first one's sample or fallback (a NaN is never equal, and +0 and -0 give the same position) + const bool same = all(equal(second.xy, big.xy)); #if PRESET_L // linear light: a sample's expansion (when the previous exposure is positive) or the colour texel const vec3 h1 = out1 ? texelFetch(colorTex, colourTexel(near), 0).rgb : historyLinear(q1, p.expoPrev == 0.0 ? 1.0 : p.expoPrev); @@ -221,7 +224,9 @@ void main() if (b2 > 0.0) { const vec2 q2 = centre + second.xy; - if (q2.x < 0.0 || q2.y < 0.0 || q2.x > size.x || q2.y > size.y || p.reset != 0u) + if (same) + h2 = h1; + else if (q2.x < 0.0 || q2.y < 0.0 || q2.x > size.x || q2.y > size.y || p.reset != 0u) h2 = out1 ? h1 : texelFetch(colorTex, colourTexel(near), 0).rgb; else h2 = historyLinear(q2, p.expoPrev == 0.0 ? 1.0 : p.expoPrev); } @@ -232,7 +237,9 @@ void main() if (b2 > 0.0) { const vec2 q2 = centre + second.xy; - if (q2.x < 0.0 || q2.y < 0.0 || q2.x > size.x || q2.y > size.y || p.reset != 0u) + if (same) + h2 = h1; + else if (q2.x < 0.0 || q2.y < 0.0 || q2.x > size.x || q2.y > size.y || p.reset != 0u) h2 = out1 ? h1 : compress(e2 * texelFetch(colorTex, colourTexel(near), 0).rgb); else h2 = history(q2); } From 8ffb0aee45e9bf32907cfe0f728e8cd28b129be8 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:39:37 +1300 Subject: [PATCH 10/19] engine: SR reconstruction computes both rows' Gaussians in one pass post_m.comp's invocations own a vertical pixel pair and ran the 3x3 (L 4x4) Gaussian taps once per row, reading the three shared tile entries of every tap twice. The two rows have the same column and texels and differ only in dy, so both are now accumulated in one loop: one set of tile reads per tap, dy, the exponent and the third accumulator as binary16 pairs (element 0 the even row), and every operation of a row the one it had on its own, lane by lane. The odd row keeps its own-cell residual rounding. Every output frame byte-identical (M and L, HDR and LDR, the three tile tiers: 1280x720 -> 1440p, 1114x626 -> 1080p, 1920x1080 -> 1440p, and the stress inputs), and so is everything D4R_M_DUMP writes. Controls that must differ: the odd row's weight sum times 1.01 (every frame of every input), the rows' third accumulators swapped (every frame; in the dump only the packed image, as g.z only enters it), the odd row's residual cast once (1114x626 -> 1080p, the one input where the rows' cells differ). Shader: 64 -> 37 shared memory reads, 2105 -> 1969 instructions, 60 vector registers as before. RX 9070 XT, d4r-m 720p -> 1440p, interleaved, reconstruction stage: 287 -> 265 us (L 355 -> 320), compact tier 193 -> 177 (L 244 -> 223), random motion 360 -> 347 (L 400 -> 367), 15x15 tier at 1920x1080 -> 1440p 424 -> 423 (L 427 -> 421). On the uniform-motion frames M's downsample after it measures 105 -> 117 us with the same shader (L's does not move), so the frame gains 7 us there (1527 -> 1520), L 36 (2249 -> 2212). Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/post_m.comp | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/engine/post_m.comp b/engine/post_m.comp index f0243e2..a347ae2 100644 --- a/engine/post_m.comp +++ b/engine/post_m.comp @@ -154,6 +154,42 @@ void main() // three render texel rows and its own-cell residual rounding exactly as in the 16x16 shader const int y0 = 2 * gid.y; const bool inside0 = gid.x < p.hiSize.x && y0 < p.hiSize.y; + const vec2 pos0 = fma(vec2(gid.x, y0) + 0.5, p.hi2rnd, p.rndOffs), pos1 = fma(vec2(gid.x, y0 + 1) + 0.5, p.hi2rnd, p.rndOffs); + // The Gaussians of both rows at once: they have the same column (dx) and texels and differ only in dy, so one set + // of tile reads serves both, and each row's operations are the ones it had on its own, lane by lane (dy, the + // exponents and the third accumulator are binary16 pairs, element 0 the even row). + f16vec4 a0[2] = {f16vec4(0.0), f16vec4(0.0)}, a1[2] = {f16vec4(0.0), f16vec4(0.0)}; + f16vec2 a2 = f16vec2(0.0); + float W[2] = {0.0, 0.0}; + if (inside0) + { + // NVIDIA's kernel handles rows in pairs and gives the odd row the even row's three texel rows + const ivec2 base = ivec2(floor(pos0 - 0.5)); + const f16vec2 d0 = f16vec2((vec2(base) + 0.5) - pos0); + // Native paired rows round the odd row's own-cell residual first, then adjust it in f16. + // Casting the even-cell residual once changes Gaussian weights at half-rounding boundaries. + const int ownBaseY = int(floor(pos1.y - 0.5)); + const float16_t d1y = ownBaseY != base.y ? float16_t((float(ownBaseY) + 0.5) - pos1.y) - float16_t(1.0) + : float16_t((float(base.y) + 0.5) - pos1.y); + const f16vec2 dy0 = f16vec2(d0.y, d1y); + for (int i = TAP_LO; i <= TAP_HI; ++i) + { + const float16_t dx = d0.x + float16_t(i), sq = dx * dx; + for (int j = TAP_LO; j <= TAP_HI; ++j) + { + const int k = (base.y - origin.y + j) * TILE_W + (base.x - origin.x + i); + const f16vec4 t0 = tile0[k], t1 = tile1[k], t2 = tile2[k]; + const f16vec2 dy = dy0 + float16_t(j); + precise f16vec2 e = (f16vec2(sq * t2.y) + (dy * dy) * t2.z) + (dx * dy) * t2.w; + const float w0 = exp2(float(e.x)), w1 = exp2(float(e.y)); + const float16_t wh0 = float16_t(w0), wh1 = float16_t(w1); + W[0] += w0; W[1] += w1; + a0[0] = fma(f16vec4(wh0), t0, a0[0]); a1[0] = fma(f16vec4(wh0), t1, a1[0]); + a0[1] = fma(f16vec4(wh1), t0, a0[1]); a1[1] = fma(f16vec4(wh1), t1, a1[1]); + a2 = fma(f16vec2(wh0, wh1), f16vec2(t2.x), a2); + } + } + } vec3 outA0 = vec3(0.0), outA1 = vec3(0.0); for (int r = 0; r < 2; ++r) { @@ -163,39 +199,10 @@ void main() if (inside) { const vec2 centre = vec2(pix) + 0.5; - const vec2 pos = fma(centre, p.hi2rnd, p.rndOffs); - // NVIDIA's kernel handles rows in pairs and gives the odd row the even row's three texel rows - const float evenY = fma(float(pix.y & ~1) + 0.5, p.hi2rnd.y, p.rndOffs.y); - const ivec2 near = ivec2(floor(pos)), base = ivec2(floor(pos.x - 0.5), floor(evenY - 0.5)); - f16vec2 d0 = f16vec2((vec2(base) + 0.5) - pos); - if ((pix.y & 1) != 0) - { - // Native paired rows round the odd row's own-cell residual first, then adjust it in f16. - // Casting the even-cell residual once changes Gaussian weights at half-rounding boundaries. - const int ownBaseY = int(floor(pos.y - 0.5)); - if (ownBaseY != base.y) - d0.y = float16_t((float(ownBaseY) + 0.5) - pos.y) - float16_t(1.0); - } - f16vec4 a0 = f16vec4(0.0), a1 = f16vec4(0.0); - float16_t a2 = float16_t(0.0); - float W = 0.0; - for (int i = TAP_LO; i <= TAP_HI; ++i) - { - const float16_t dx = d0.x + float16_t(i), sq = dx * dx; - for (int j = TAP_LO; j <= TAP_HI; ++j) - { - const int k = (base.y - origin.y + j) * TILE_W + (base.x - origin.x + i); - const f16vec4 t2 = tile2[k]; - const float16_t dy = d0.y + float16_t(j); - precise float16_t e = (sq * t2.y + (dy * dy) * t2.z) + (dx * dy) * t2.w; - const float w = exp2(float(e)); - const float16_t wh = float16_t(w); - W += w; - a0 = fma(f16vec4(wh), tile0[k], a0); a1 = fma(f16vec4(wh), tile1[k], a1); a2 = fma(wh, t2.x, a2); - } - } - const float inv = 1.0 / W; - const vec3 cur = inv * vec3(a0.rgb), m = inv * vec3(a0.w, a1.x, a1.y), g = inv * vec3(a1.z, a1.w, a2); + const vec2 pos = r == 0 ? pos0 : pos1; + const ivec2 near = ivec2(floor(pos)); + const float inv = 1.0 / W[r]; + const vec3 cur = inv * vec3(a0[r].rgb), m = inv * vec3(a0[r].w, a1[r].x, a1[r].y), g = inv * vec3(a1[r].z, a1[r].w, a2[r]); // the two most likely motion candidates const vec2 mvPos = any(notEqual(textureSize(mvCTex, 0), p.rndSize)) ? clamp(pos, vec2(0.5), vec2(p.rndSize) - 0.5) : pos; @@ -246,7 +253,7 @@ void main() const vec3 hist = e1 * (expand(fma(vec3(b1), h1, b2 * h2)) * (1.0 / e2)); #endif const vec3 lin = expand(cur); - const float g0 = clamp(p.gain * W * g.x, 0.0, 1.0); + const float g0 = clamp(p.gain * W[r] * g.x, 0.0, 1.0); const float ga = fma(g0, 1.0 - g.y, g.y), gb = fma(g0, 1.0 - g.z, g.z); outA = fma(lin - hist, vec3(ga), hist); if (isinf(outA.x + outA.y + outA.z) || isnan(outA.x + outA.y + outA.z)) outA = lin; From c90dc621120b8ce640b8e379785c16a3fb8ca89c Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:45:18 +1300 Subject: [PATCH 11/19] engine: SR downsample's horizontal pass reads its texels into registers down_m.comp (presets M and L) unpacked the block's 16x100 source texels into a shared tile, waited for it, filtered the rows from there and waited again. Each invocation of the horizontal pass now loads the ten packed texels of four output columns of one source row straight into registers and filters them (200 such items for the block's 512 invocations): no shared source tile (12.5 KiB) and one barrier fewer. 15 -> 5 KiB of shared memory, 36 -> 17 shared-memory instructions, 36 -> 48 vector registers (32 waves as before). An output column's parity stays a run-time value (gl_NumWorkGroups.z - 1 added, the dispatch's z count is 1): with constant weights the compiler shares products between neighbouring filters and splits their fused multiply-adds, and 30% of the output pixels change (62 dB) - that build is the positive control. ACO now unpacks with a fused multiply-convert (v_fma_mix*_f16: one rounding instead of f32, then binary16), which gives the same value for each of the 2048 / 1024 codes of a channel (checked exhaustively, HDR and LDR constants). Every output frame byte-identical (M and L, HDR and LDR, and the RGB10 output variant bound in the plain one's place; 1280x720 -> 1440p, 1114x626 -> 1080p, 1920x1080 -> 1440p, the stress inputs). Only down_m.spv and down_m_rgb10.spv change. RX 9070 XT, d4r-m, interleaved, downsample stage: 720p -> 1440p 102 -> 91 us (L; M 117 -> 113), 1114x626 -> 1080p 62 -> 47 (L 61 -> 47); noise (the stress inputs, whose output does not compress) unchanged, 144 -> 144 (L 139 -> 138). Frame (L) 2213 -> 2202 us and 1963 -> 1951. Two or eight columns per invocation instead of four: 92 / 93 us. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/down_m.comp | 49 +++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/engine/down_m.comp b/engine/down_m.comp index d445b0b..2a675af 100644 --- a/engine/down_m.comp +++ b/engine/down_m.comp @@ -4,6 +4,7 @@ // five-tap filter whose weights depend on the output pixel's parity, limits each pass to the range of its taps // (no ringing), and undoes the range compression and the exposure. #extension GL_EXT_shader_explicit_arithmetic_types : require +#extension GL_EXT_control_flow_attributes : require #extension GL_GOOGLE_include_directive : require #include "output_store.glsl" // LDR (NGX's *_ldr_* kernel): the packed image is the display-referred colour itself. @@ -13,8 +14,7 @@ // D4RM0003: the runtime dispatches 8x64 output pixels per workgroup. #define BLOCK_X 8 #define BLOCK_Y 64 -#define TILE_W (3 * BLOCK_X / 2 + 4) -#define TILE_H (3 * BLOCK_Y / 2 + 4) +#define TILE_H (3 * BLOCK_Y / 2 + 4) // source rows of the block layout(local_size_x = BLOCK_X, local_size_y = BLOCK_Y) in; layout(set = 0, binding = 0, std430) readonly buffer Params { @@ -27,8 +27,19 @@ layout(set = 0, binding = 0, std430) readonly buffer Params layout(set = 0, binding = 1) uniform sampler2D colorTex; // render resolution; its alpha is passed through layout(set = 0, binding = 2, FINAL_OUTPUT_FORMAT) uniform writeonly image2D outImg; layout(set = 0, binding = 3, std430) readonly buffer Src { uint w[]; } src; -shared f16vec4 tile[TILE_W * TILE_H]; // source texels around the block shared f16vec4 rows[TILE_H * BLOCK_X]; // after the horizontal pass +// The horizontal pass reads the source texels straight into registers, without a shared tile: an invocation filters +// SEG output columns of one source row from the 3 SEG / 2 + 4 texels around them. +#define SEG 4 +#define SPAN (3 * SEG / 2 + 4) +f16vec3 unpack(uint w) +{ +#if LDR + return f16vec3(vec3(float(w & 2047u) * (1.0 / 2047.0), float((w >> 11) & 2047u) * (1.0 / 2047.0), float(w >> 22) * (1.0 / 1023.0))); +#else + return f16vec3(vec3(float(w & 2047u) * 0.0011968734906986356, float((w >> 11) & 2047u) * 0.0011968734906986356, float(w >> 22) * 0.002394916955381632)); +#endif +} // weights of taps -2 .. +2 for an even output coordinate; an odd one uses them mirrored // as binary16 bit patterns: NGX rounds its f32 constants to the nearest binary16 (cvt.rn), while glslang folds // float16_t() toward zero (W0 became -0.041321 instead of -0.041351: 6% of the captured frame's outputs @@ -49,25 +60,27 @@ void main() { const ivec2 lid = ivec2(gl_LocalInvocationID.xy), pix = ivec2(gl_GlobalInvocationID.xy); const ivec2 base = ivec2(3 * BLOCK_X / 2, 3 * BLOCK_Y / 2) * ivec2(gl_WorkGroupID.xy) - 2; - for (int i = lid.y * BLOCK_X + lid.x; i < TILE_W * TILE_H; i += BLOCK_X * BLOCK_Y) + // the horizontal pass, SEG output columns x0 .. x0 + SEG - 1 of source row y per item + const int zero = int(gl_NumWorkGroups.z) - 1; + for (int item = lid.y * BLOCK_X + lid.x; item < TILE_H * (BLOCK_X / SEG); item += BLOCK_X * BLOCK_Y) { - const ivec2 at = clamp(base + ivec2(i % TILE_W, i / TILE_W), ivec2(0), p.srcSize - 1); - const uint w = src.w[at.y * p.srcSize.x + at.x]; -#if LDR - tile[i] = f16vec4(vec4(float(w & 2047u) * (1.0 / 2047.0), float((w >> 11) & 2047u) * (1.0 / 2047.0), float(w >> 22) * (1.0 / 1023.0), 0.0)); -#else - tile[i] = f16vec4(vec4(float(w & 2047u) * 0.0011968734906986356, float((w >> 11) & 2047u) * 0.0011968734906986356, float(w >> 22) * 0.002394916955381632, 0.0)); -#endif + const int y = item / (BLOCK_X / SEG), x0 = item % (BLOCK_X / SEG) * SEG; + const int row = clamp(base.y + y, 0, p.srcSize.y - 1) * p.srcSize.x; + f16vec3 t[SPAN]; + [[unroll]] for (int k = 0; k < SPAN; ++k) + t[k] = unpack(src.w[row + clamp(base.x + 3 * x0 / 2 + k, 0, p.srcSize.x - 1)]); + [[unroll]] for (int q = 0; q < SEG; ++q) + { + // column x0 + q (as even as q) is centred on texel 3 (q / 2) + 2 (q & 1) + 2 of the item's. Its parity stays a + // run-time value: with constant weights the compiler shares products between neighbouring filters and + // splits their fused multiply-adds (30% of the outputs change) + const int o = 3 * (q >> 1) + 2 * (q & 1) + 2; + rows[y * BLOCK_X + x0 + q].rgb = filter5(t[o - 2], t[o - 1], t[o], t[o + 1], t[o + 2], ((q + zero) & 1) != 0); + } } memoryBarrierShared(); barrier(); const ivec2 c = ivec2(floor((vec2(pix) + 0.5) * 1.5)) - base; - for (int y = lid.y; y < TILE_H; y += BLOCK_Y) - { - const int o = y * TILE_W + c.x; - rows[y * BLOCK_X + lid.x].rgb = filter5(tile[o - 2].rgb, tile[o - 1].rgb, tile[o].rgb, tile[o + 1].rgb, tile[o + 2].rgb, (pix.x & 1) != 0); - } - memoryBarrierShared(); barrier(); - // After both shared barriers: borrowed images may extend beyond the feature's true output rectangle. + // After the shared barrier: borrowed images may extend beyond the feature's true output rectangle. // M accepts even output dimensions and srcSize = outputSize/2*3, so this bounds check is exact. if (3 * pix.x >= 2 * p.srcSize.x || 3 * pix.y >= 2 * p.srcSize.y) return; const int o = c.y * BLOCK_X + lid.x; From f70adc8202e32449dccfee30edf6118250d833ce Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:50:41 +1300 Subject: [PATCH 12/19] engine: preset L's expansion stage at two windows per wave layer_m.comp's L expansion (KIND 4) wrote a window's block output to shared memory and ran the per-pixel head after a workgroup barrier, both inside the windows' loop, so it could not walk two windows per wave (#error for KIND >= 3). Every window now writes its output first and the head runs once after the barrier; L's input stage (KIND 3) gathers its features once, before its windows' loop, the same way. At one window per wave RADV emits the same machine code for both stages as before. WPW > 2 stays an error: the head and the gather need the block's 64 pixels on 64 invocations. compile_m.py builds dec0_l (and dec0_l_planes) with two windows per wave, which load each weight tile once for both: 96 -> 216 vector registers, 14 -> 7 waves per SIMD. Not under M_EXACT=1, like the network layers (it would gain 7 us there, 516 -> 509); M_DEFS_DEC0L=-DWPW=1 builds the previous layout. The input stage at two windows per wave is identical too but slower (452 -> 465 us; 144 -> 256 registers, 10 -> 5 waves), so it stays at one. Every output frame byte-identical (L, HDR and LDR, 1280x720 -> 1440p, 1114x626 -> 1080p, 1920x1080 -> 1440p, the stress inputs) and so is everything D4R_M_DUMP writes - with the new default, with both stages at one window and at two, and in the exact build (one window, and the expansion at two). The windows' outputs swapped in the head's staging changes every frame. Preset M's shaders are unchanged. RX 9070 XT, d4r-m L 720p -> 1440p, interleaved: expansion 383 -> 366 us (1114x626 -> 1080p: 342 -> 326), frame 2205 -> 2191 us. Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/compile_m.py | 4 +++- engine/layer_m.comp | 18 +++++++++++------- engine/test_m.py | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/engine/compile_m.py b/engine/compile_m.py index b35e38c..66b4ac7 100644 --- a/engine/compile_m.py +++ b/engine/compile_m.py @@ -32,6 +32,8 @@ from test_m import pack_layer, defines EXACT = os.environ.get('M_EXACT') == '1' FAST = [] if EXACT or '-DM_RELAXED' in os.environ.get('M_DEFS', '').split() else ['-DM_RELAXED'] FAST_WPW = [] if EXACT or os.environ.get('M_WPW') or 'WPW=' in os.environ.get('M_DEFS', '') else ['-DWPW=2'] +# preset L's expansion stage (dec0l) two windows per wave as well; its input stage (enc0l) is slower that way +L_WPW = [] if EXACT or 'WPW=' in os.environ.get('M_DEFS', '') + os.environ.get('M_DEFS_DEC0L', '') else ['-DWPW=2'] SHADERS = {'enc1': 'm_enc1', 'enc2': 'm_enc2', 'enc3_tube': 'm_tube', 'dec2': 'm_dec2', 'dec1': 'm_dec1'} @@ -159,7 +161,7 @@ def compile_l(a, one, R): for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW) for planes, suffix in ((0, ''), (1, '_planes')): glsl('layer_m.comp', f'enc0_l{suffix}.spv', defines('enc0l') + ldr + [f'-DPLANES={planes}'] + FAST) - glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST) + glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST + L_WPW) # the reconstruction one render texel wider per tier than M's (4x4 taps): 11 (reconstruction/render >= 3), # 16 (the whole post_3_2 range), 12 (the compact tier: render/reconstruction <= 0.45, jitter within half a texel) for out, tile in (('post_l', 11), ('post_l_q', 16), ('post_l_c', 12)): diff --git a/engine/layer_m.comp b/engine/layer_m.comp index 36bd42c..84e673c 100644 --- a/engine/layer_m.comp +++ b/engine/layer_m.comp @@ -64,8 +64,8 @@ #ifndef WPW #define WPW 1 #endif -#if WPW > 1 && KIND >= 3 -#error layer_m.comp: L's input and expansion stages stage their pixels per thread (WPW = 1) +#if WPW > 2 && KIND >= 3 +#error layer_m.comp: L's input and expansion stages handle their 64 pixels one per invocation (WPW <= 2) #endif layout(local_size_x = 128 / WPW) in; // o: 0 g1, 1 bo, 2 g2, 3 b2 (f16); per head h at 4 + 4h: q, v (FP8), bias (f16), wo (FP8); @@ -496,8 +496,7 @@ void main() #endif #if KIND == 3 // ------------------------------------------------------------ L's input: features, embedding -> x0 - FORW { - WINV + { const float e1 = ep.expo == 0.0 ? 1.0 : ep.expo, e2 = ep.expoPrev == 0.0 ? 1.0 : ep.expoPrev; const uint li = gl_LocalInvocationID.x; if (li < 64u) @@ -510,7 +509,10 @@ void main() const uint at = (((py >> 2) * 2u + (px >> 2)) * 16u + (py & 3u) * 4u + (px & 3u)) * FS; for (uint k = 0u; k < 32u; ++k) FEW[at + k] = fe[k]; } - SYNC + } + SYNC + FORW { + WINV // the embedding has the shape of an MLP chunk: 32 features -> 32 hidden (ReLU, FP8) -> C B8 f0, f1; coopMatLoad(f0, FEW, mt * 16u * FS, FS, CM); @@ -532,8 +534,8 @@ void main() for (int i = 0; i < 8; ++i) X0S[xb + 16u * (N) + uint(i)] = float16_t(x_[i]); } EACH_T(EMB_N) #undef EMB_N - SYNC } + SYNC #endif // ------------------------------------------------------------ norm 1: sum of squares, residual and h1 FORW { @@ -768,7 +770,9 @@ void main() const uint hb = (mt * 16u + row) * HS + ch8; for (uint nt = 0u; nt < NT; ++nt) for (int i = 0; i < 8; ++i) HO[hb + 16u * nt + uint(i)] = float16_t(res[nt][i]); - SYNC + } + SYNC + { const uint li = gl_LocalInvocationID.x; const uint px = li & 7u, py = li >> 3; const ivec2 at = ivec2(X0 + int(px), Y0 + int(py)); diff --git a/engine/test_m.py b/engine/test_m.py index 257e4c0..ffa273e 100644 --- a/engine/test_m.py +++ b/engine/test_m.py @@ -90,7 +90,7 @@ def defines(name): # compile_m.py's arithmetic (ZLUDA's lowering); M_TEXL=0 gives the native accuracy kernels' for the network layers. # M_DEFS: extra defines for every layer_m.comp stage; M_DEFS_ENC1, M_DEFS_ENC3_TUBE, ...: for one layer texl = ['-DTEXL=1'] if kind >= 3 or os.environ.get('M_TEXL', '1') != '0' else [] - # M_WPW=2: two 4x4-token windows per wave in the network layers (layer_m.comp WPW; not L's input / expansion) + # M_WPW=2: two 4x4-token windows per wave in the network layers (layer_m.comp WPW; L's input / expansion: M_DEFS_*) wpw = [f"-DWPW={os.environ['M_WPW']}"] if kind < 3 and os.environ.get('M_WPW') else [] return [f'-DC={C}', f'-DNH={heads}', f'-DKIND={kind}', f'-DXC={NPM or CIN or 16}'] + texl + wpw + os.environ.get('M_DEFS', '').split() + os.environ.get('M_DEFS_' + name.upper(), '').split() From 6c7f881d9c52b2f426103bc4ed5f46feae085147 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:02:22 +1300 Subject: [PATCH 13/19] engine/rr: preset F's exposure in one pass with 64-bit atomics exposure_f.comp ran NGX's auto exposure in two passes: the 16x16 blocks wrote their fixed-point luma sums, then a single workgroup added them after a barrier and finished (11 us of a 25 us stage). PASS 2 does what NVIDIA's kernel does: each block adds its sum to a 64-bit total with an atomic add and counts itself in with an acquire-release add; the last block reads the total, finishes and leaves both words zero for the next frame. The total is an integer, the same in any order, so the exposure image and the values are the two passes' bit for bit. compile_rr2.py builds exposure_f2.spv next to the two passes' shaders (the block sum and the finish are shared functions now; their arithmetic is unchanged). 64-bit buffer atomics are a device feature (shaderBufferInt64Atomics), so RModel takes whether the device has it enabled (default no) and preset F uses the one pass only then and only with a model that has exposure_f2.spv; otherwise the two passes as before. vkd3d-proton creates its device with every feature the GPU reports except a list it turns off, which does not include this one (it backs D3D12's 64-bit atomics with it); the shim reads the device's features through the interop interface (GetDeviceFeatures) and logs the result. d4r-rr enables every feature the GPU has, and D4R_RR_TEST_NO_INT64_ATOMICS=1 turns this one off to run the two passes. Ray Reconstruction never runs on the native Vulkan frontend's device. Every image of every frame byte-identical to the previous build (1280x720 -> 1440p, 1114x626 -> 1080p, 960x540 -> 1080p, the stress inputs, with D4R_RR_TEST_ROBUST2=1, at levels 0 and 3, with D4R_RR2_SPLIT=1), and so is the two-pass fallback of the new build; presets D and E unchanged. Controls: the last block finishing with twice the total changes 64 of the 90 images of the 1280x720 sequence (and nothing under the fallback), finishing without its own block's sum 58 of 90. RX 9070 XT, d4r-rr 720p -> 1440p, interleaved: exposure 25.2 -> 14.9 us, frame 3.536 -> 3.526 ms (the fallback 25.1 us). Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/compile_rr2.py | 4 +- engine/replay_rr.cpp | 4 +- engine/rr/exposure_f.comp | 93 ++++++++++++++++++++++---------- engine/runtime.h | 5 +- engine/runtime_rr.inc | 4 +- engine/runtime_rr2.inc | 55 +++++++++++++------ packaging/build_engine_models.sh | 2 +- tools/d4r_nvngx_shim.cpp | 16 +++++- 8 files changed, 131 insertions(+), 52 deletions(-) diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 90e07a5..455ea45 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -12,7 +12,7 @@ D4R_CUDA_REPLAY_DUMP_FILTER=rr2 D4R_CUDA_LAUNCH_STATS_FILTER=rr2 and preset hint table of its PTX module; enc0-aux.bin is encoder 0's second table (argument 104) .args the argument block of the capture's frame, pointers and handles cleared: the runtime keeps its constants and fills what depends on the frame or the geometry (engine/runtime_rr2.inc) - exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp + exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp (exposure_f2.spv: both in one, 64-bit atomics) rr.txt the manifest RModel reads ("D4RR0002") With --native, the stages engine/rr/rr2_m.comp implements (enc1 ... enc5, dec4 ... dec1, dec0's first kernel) are that shader with weights read from the DLL itself (engine/rr/rr2_pack.py) instead of the translation: bit-identical to @@ -249,7 +249,7 @@ def main(): if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3') print(f'{stage}: {kernel}, {len(args)} argument bytes' + (' (native block)' if hybrid else '')) - for p in (0, 1): + for p in (0, 1, 2): subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}', str(HERE / 'rr' / 'exposure_f.comp'), '-o', str(out / f'exposure_f{p}.spv')], check=True, stdout=subprocess.DEVNULL) diff --git a/engine/replay_rr.cpp b/engine/replay_rr.cpp index 4e3d605..c591d79 100644 --- a/engine/replay_rr.cpp +++ b/engine/replay_rr.cpp @@ -158,6 +158,8 @@ int main(int argc, char** argv) try { if (std::getenv("D4R_RR_TEST_NO_ROBUST")) { f2.features.robustBufferAccess = VK_FALSE; v13.robustImageAccess = VK_FALSE; } // D4R_RR_TEST_NO_COOPMAT=1: a device like vkd3d-proton's, without the cooperative-matrix and float8 features if (std::getenv("D4R_RR_TEST_NO_COOPMAT")) { di.enabledExtensionCount = 0; v13.pNext = nullptr; } + // D4R_RR_TEST_NO_INT64_ATOMICS=1: no 64-bit buffer atomics (preset F's exposure then takes its two passes) + if (std::getenv("D4R_RR_TEST_NO_INT64_ATOMICS")) v12.shaderBufferInt64Atomics = v12.shaderSharedInt64Atomics = VK_FALSE; CK(vkCreateDevice(c.physical, &di, nullptr, &c.device)); vkGetDeviceQueue(c.device, family, 0, &c.queue); VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pi.queueFamilyIndex = family; @@ -169,7 +171,7 @@ int main(int argc, char** argv) try { std::string key; uint32_t rw, rh, ow, oh; frames >> key >> rw >> rh >> ow >> oh; - d4r::RModel model(c.physical, c.device, modelDir); + d4r::RModel model(c.physical, c.device, modelDir, VK_NULL_HANDLE, v12.shaderBufferInt64Atomics == VK_TRUE); d4r::REngine engine(model, rw, rh, ow, oh); const char* names[] = {"color", "motion", "depth", "normals", "roughness", "diffuse", "specular"}; const VkFormat formats[] = {VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R32_SFLOAT, diff --git a/engine/rr/exposure_f.comp b/engine/rr/exposure_f.comp index fb4e308..fa456c6 100644 --- a/engine/rr/exposure_f.comp +++ b/engine/rr/exposure_f.comp @@ -1,33 +1,39 @@ #version 460 -// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel, in two passes. +// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel. // // NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block // sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish -// turns the total into the exposure. The integer total does not depend on the order of the blocks, so PASS 0 writes -// every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last block does: +// turns the total into the exposure: // exposure = 0.23547088 / max(mean luma, 0.005) -> the 1x1 exposure image of this frame // values = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 } -> the 16-byte buffer encoder 0 and the output passes read // with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero). +// PASS 2 is that kernel (64-bit atomics on a storage buffer: shaderBufferInt64Atomics). Without them the runtime runs +// two passes: PASS 0 writes every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last +// block does. The total is an integer: it does not depend on the order of the blocks, so both give the same values. #extension GL_EXT_shader_explicit_arithmetic_types : require #extension GL_EXT_control_flow_attributes : require #ifndef PASS #define PASS 0 #endif +#if PASS == 2 +#extension GL_EXT_shader_atomic_int64 : require +#extension GL_KHR_memory_scope_semantics : require +#endif layout(set = 0, binding = 0, std430) readonly buffer Params { ivec2 last; // render size - 1 uint blocksX, blocks; float invCount; // 10000 / (2^32 * render pixels) uint reset; // the kernel's argument 32: no log ratio when set } p; -#if PASS == 0 +#if PASS != 1 // One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128, // then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the // same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1). layout(local_size_x = 16) in; layout(set = 0, binding = 1) uniform sampler2D color; -layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums; shared float col[16]; -void main() +// the block's fixed-point sum, in thread 0 +uint64_t blockSum() { const uint tx = gl_LocalInvocationID.x; const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16; @@ -49,33 +55,37 @@ void main() precise float c = w1 + w0; col[tx] = c; barrier(); - if (tx == 0u) { - precise float d[8]; - [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k]; - precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3]; - precise float f0 = e2 + e0, f1 = e3 + e1; - precise float g = f1 + f0; - sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = uint64_t(roundEven(g * 429496.71875)); - } + if (tx != 0u) return 0ul; + precise float d[8]; + [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k]; + precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3]; + precise float f0 = e2 + e0, f1 = e3 + e1; + precise float g = f1 + f0; + return uint64_t(roundEven(g * 429496.71875)); +} +#endif +#if PASS == 0 +layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums; +void main() +{ + const uint64_t s = blockSum(); + if (gl_LocalInvocationID.x == 0u) sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = s; } #else +#if PASS == 1 layout(local_size_x = 64) in; shared uint64_t part[64]; layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums; -layout(set = 0, binding = 2, std430) writeonly buffer Values { vec4 v; } values; -layout(set = 0, binding = 3) uniform sampler2D previous; -layout(set = 0, binding = 4, r16f) uniform writeonly image2D current; -void main() +#else +// s[0]: the total, s[1]: the blocks done; the last block leaves both 0 for the next frame +layout(set = 0, binding = 2, std430) buffer Sums { uint64_t s[]; } sums; +#endif +layout(set = 0, binding = PASS + 1, std430) writeonly buffer Values { vec4 v; } values; +layout(set = 0, binding = PASS + 2) uniform sampler2D previous; +layout(set = 0, binding = PASS + 3, r16f) uniform writeonly image2D current; +// the last block's work, from the frame's total +void finish(uint64_t total) { - // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes - const uint tid = gl_LocalInvocationID.x; - uint64_t mine = 0ul; - for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b]; - part[tid] = mine; - barrier(); - if (tid != 0u) return; - uint64_t total = 0ul; - for (uint b = 0u; b < 64u; ++b) total += part[b]; precise float mean = p.invCount * float(total); precise float e = 0.23547088 / max(mean, 0.005); imageStore(current, ivec2(0), vec4(e)); @@ -90,4 +100,33 @@ void main() if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667; values.v = vec4(cur, prev, 1.0 / cur, ratio); } +#if PASS == 1 +void main() +{ + // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes + const uint tid = gl_LocalInvocationID.x; + uint64_t mine = 0ul; + for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b]; + part[tid] = mine; + barrier(); + if (tid != 0u) return; + uint64_t total = 0ul; + for (uint b = 0u; b < 64u; ++b) total += part[b]; + finish(total); +} +#else +void main() +{ + const uint64_t s = blockSum(); + if (gl_LocalInvocationID.x != 0u) return; + atomicAdd(sums.s[0], s, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + // the release orders this block's add before its count, and the last block's acquire every block's add before + // its read of the total + if (atomicAdd(sums.s[1], 1ul, gl_ScopeDevice, gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease) != uint64_t(p.blocks - 1u)) + return; + const uint64_t total = atomicExchange(sums.s[0], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + atomicExchange(sums.s[1], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + finish(total); +} +#endif #endif diff --git a/engine/runtime.h b/engine/runtime.h index 4ab8ed5..7c3d38e 100644 --- a/engine/runtime.h +++ b/engine/runtime.h @@ -273,7 +273,10 @@ struct RFrameSettings { }; class RModel { public: - RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE); + // int64Atomics: the device has shaderBufferInt64Atomics enabled; preset F then measures the exposure in one pass + // (engine/rr/exposure_f.comp PASS 2) instead of two + RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE, + bool int64Atomics = false); ~RModel(); RModel(const RModel&) = delete; RModel& operator=(const RModel&) = delete; diff --git a/engine/runtime_rr.inc b/engine/runtime_rr.inc index 825035e..04908c6 100644 --- a/engine/runtime_rr.inc +++ b/engine/runtime_rr.inc @@ -90,7 +90,7 @@ struct RModel::Impl { bool uploaded = false; Impl(VkPhysicalDevice physical, VkDevice dev) : device(physical, dev) {} }; -RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache) +RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache, bool int64Atomics) : impl(std::make_unique(physical, device)) { auto& m = *impl; // rr.txt: "D4RR0001", "preset N", one "stage NAME SHADER.spv SUBGROUP WEIGHTS TEXTURES SURFACES" line per stage in @@ -103,7 +103,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di if (line.rfind("D4RR0002", 0) == 0) { m.preset = 6; m.v2 = std::make_unique(); - m.v2->load(m.device, directory, lines, cache); + m.v2->load(m.device, directory, lines, cache, int64Atomics); return; } if (line.rfind("D4RR0001", 0) != 0) throw std::runtime_error("not a Ray Reconstruction model (rr.txt D4RR0001)"); diff --git a/engine/runtime_rr2.inc b/engine/runtime_rr2.inc index 85e8fcc..dfdf963 100644 --- a/engine/runtime_rr2.inc +++ b/engine/runtime_rr2.inc @@ -44,9 +44,10 @@ struct R2Model { Buffer aux; // encoder 0's second table (argument 104) std::vector auxPending; std::array exposure; + Pipeline exposureOne; // both exposure passes in one (exposure_f2.spv, 64-bit atomics); else null bool particles = false; // the model has R2DownP bool gather = false; // the model has R2Gather - void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache) { + void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache, bool int64Atomics) { const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; auto split = [](const std::string& list) { std::vector items; @@ -108,6 +109,8 @@ struct R2Model { blob(aux, auxPending, read(directory + "/enc0-aux.bin")); exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache); exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache); + if (int64Atomics && exists(directory + "/exposure_f2.spv")) + exposureOne.create(d.handle, directory + "/exposure_f2.spv", {b, t, b, b, t, s}, 64, cache); } void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents) for (auto& st : stages) recordContents(cb, st.weights, st.pending); @@ -172,7 +175,7 @@ struct R2Engine { std::vector down; std::vector> downP; VkDescriptorSet pooled = VK_NULL_HANDLE, exposure0 = VK_NULL_HANDLE; - std::array exposure1{}; + std::array exposure1{}, exposureOne{}; }; std::vector bindings; const Bindings* bound = nullptr; @@ -337,7 +340,7 @@ struct R2Engine { exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16); exposure.last[0] = static_cast(rw - 1); exposure.last[1] = static_cast(rh - 1); exposure.invCount = static_cast(10000.0 / (4294967296.0 * rw * rh)); - sums.create(d, exposure.blocks * 8, storage); + sums.create(d, std::max(exposure.blocks, 2u) * 8, storage); // the one-pass exposure: the total and the count const size_t tiles = src.size(); auto place = [&](size_t bytes) { const VkDeviceSize at = paramBytes; paramBytes += (bytes + 255) & ~VkDeviceSize(255); return at; }; for (unsigned i = 0; i < R2Network; ++i) { blocks[i] = model.stages[i].args; paramAt[i] = place(blocks[i].size()); } @@ -593,17 +596,29 @@ struct R2Engine { b.downP.push_back({stageSet(b, R2DownP, downPAt[i], 0, output), stageSet(b, R2DownP, downPAt[i], 1, output)}); } b.pooled = stageSet(b, R2Pooled, pooledAt, 0, output); - b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors); - bindRange(d.handle, b.exposure0, 0, paramAll, expAt); - bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout); - bindBuffer(d.handle, b.exposure0, 2, sums); - for (uint32_t par = 0; par < 2; ++par) { - VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors); - bindRange(d.handle, set, 0, paramAll, expAt); - bindBuffer(d.handle, set, 1, sums); - bindBuffer(d.handle, set, 2, values); - bindImage(d.handle, set, 3, e[par ^ 1].view, nearest); - bindImage(d.handle, set, 4, e[par].view); + if (model.exposureOne.handle) + for (uint32_t par = 0; par < 2; ++par) { + VkDescriptorSet set = b.exposureOne[par] = descriptorSet(d.handle, b.pool, model.exposureOne.descriptors); + bindRange(d.handle, set, 0, paramAll, expAt); + bindImage(d.handle, set, 1, images.color, nearest, images.layout); + bindBuffer(d.handle, set, 2, sums); + bindBuffer(d.handle, set, 3, values); + bindImage(d.handle, set, 4, e[par ^ 1].view, nearest); + bindImage(d.handle, set, 5, e[par].view); + } + else { + b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors); + bindRange(d.handle, b.exposure0, 0, paramAll, expAt); + bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout); + bindBuffer(d.handle, b.exposure0, 2, sums); + for (uint32_t par = 0; par < 2; ++par) { + VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors); + bindRange(d.handle, set, 0, paramAll, expAt); + bindBuffer(d.handle, set, 1, sums); + bindBuffer(d.handle, set, 2, values); + bindImage(d.handle, set, 3, e[par ^ 1].view, nearest); + bindImage(d.handle, set, 4, e[par].view); + } } bindings.push_back(std::move(b)); found = bindings.end() - 1; @@ -675,9 +690,15 @@ struct R2Engine { vkCmdUpdateBuffer(cb, paramAll.handle, at, std::min(65536, paramBytes - at), all.data() + at); memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX); - step(); - dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1); + // the exposure: one pass with 64-bit atomics (the last block finishes, as NGX's kernel), or the block sums and + // one workgroup that adds them + if (model.exposureOne.handle) + dispatch(cb, model.exposureOne, b.exposureOne[cur], exposure.blocksX, exposure.blocks / exposure.blocksX); + else { + dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX); + step(); + dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1); + } step(); stamp(1); for (unsigned stage = 0; stage < R2Network; ++stage) { diff --git a/packaging/build_engine_models.sh b/packaging/build_engine_models.sh index f5baa7b..a97c4e0 100755 --- a/packaging/build_engine_models.sh +++ b/packaging/build_engine_models.sh @@ -98,7 +98,7 @@ if (( DO_RR )); then done run ptx-f.log python3 "$KIT/kernels/tools/extract_dlss_ptx.py" "$DLL" "$WORK/ptx-f" run rr-f.log python3 "$KIT/engine/compile_rr2.py" --dll "$DLL" "$WORK/rr-f" --ptx-dir "$WORK/ptx-f" - place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv + place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv exposure_f2.spv fi if (( DO_SR )); then diff --git a/tools/d4r_nvngx_shim.cpp b/tools/d4r_nvngx_shim.cpp index d35a7ba..7fd5249 100644 --- a/tools/d4r_nvngx_shim.cpp +++ b/tools/d4r_nvngx_shim.cpp @@ -3385,6 +3385,9 @@ struct VulkanInterop // VK_EXT_shader_float8 (vkd3d-proton enables it when the GPU has it): M's and L's engine layers are FP8 // cooperative-matrix shaders, which RADV's compiler crashes on without it (RDNA3) bool float8 = false; + // shaderBufferInt64Atomics is enabled on vkd3d-proton's device (it enables it when the GPU has it): Ray + // Reconstruction preset F then measures the exposure in one pass, as NGX does, instead of two + bool int64Atomics = false; // The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects. bool native = false; PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr; @@ -9322,6 +9325,17 @@ static void init_engine_vulkan() } logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: " "M and L run on the CUDA backend"); + // the features vkd3d-proton created its device with + const VkPhysicalDeviceFeatures2* features = nullptr; + if (SUCCEEDED(g_vk.interop->GetDeviceFeatures(&features)) && features != nullptr) + for (auto* next = static_cast(features->pNext); next != nullptr; next = next->pNext) + { + if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES) + g_vk.int64Atomics = reinterpret_cast(next)->shaderBufferInt64Atomics == VK_TRUE; + if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES) + g_vk.int64Atomics = reinterpret_cast(next)->shaderBufferInt64Atomics == VK_TRUE; + } + logf("engine backend: 64-bit buffer atomics %s", g_vk.int64Atomics ? "enabled" : "not enabled: preset F's exposure in two passes"); } static void init_engine_backend() @@ -10316,7 +10330,7 @@ static void init_engine_backend_rr(int slot) const auto start = std::chrono::steady_clock::now(); try { - auto model = std::make_unique(g_vk.physical, g_vk.device, path); + auto model = std::make_unique(g_vk.physical, g_vk.device, path, VK_NULL_HANDLE, g_vk.int64Atomics); if (model->preset() != g_rrPreset[slot]) throw std::runtime_error("the directory holds a model of another preset"); g_engineR.models[slot] = std::move(model); From e94c4018e7f415d26922235df4d75b9f8eab12b6 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:22:03 +1300 Subject: [PATCH 14/19] engine: opt-in M_RESIDUAL_F32, the SR residual stream in f32 across the MLP With M_RELAXED, layer_m.comp's MLP adds every 32-channel chunk's products to the binary16 residual stream and rounds it again: one f32 -> f16 -> f32 round trip per chunk and element, 4 to 16 chunks a layer. M_RESIDUAL_F32 seeds an f32 accumulator from the residual before the MLP, adds every chunk to it with plain multiply-adds and rounds to binary16 once after the last chunk. It needs M_RELAXED (#error otherwise); compile_m.py builds it into every layer_m.comp stage with M_RESIDUAL_F32=1 and refuses it with M_EXACT=1. Off by default: the default build's shaders (M, L, HDR and LDR) and the exact build's are byte-identical to the previous ones. Against the default build, final RGB, peak 1, 1280x720 -> 1440p: six smooth temporal frames M 43.3 - 45.9 dB (L 44.6 - 49.2), at most 1.5 off; four frames of the noise stress input M 17 - 18 dB (L 19 - 22). The exact build is as far from the default (M 43.2 - 44.6 dB, L 44.5 - 47.0; noise M 16.7 - 17.9, L 19 - 20) and as far from this one (M 43.4 - 44.8, L 44.6 - 47.0): no further from NVIDIA's arithmetic than the default. RX 9070 XT, d4r-m 720p -> 1440p, interleaved: M 1521 -> 1410 us (network 953 -> 841), L 2195 -> 2077 us (network 944 -> 829). Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/compile_m.py | 13 +++++++++---- engine/layer_m.comp | 22 +++++++++++++++++++++- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/engine/compile_m.py b/engine/compile_m.py index 66b4ac7..a5293bf 100644 --- a/engine/compile_m.py +++ b/engine/compile_m.py @@ -34,6 +34,11 @@ FAST = [] if EXACT or '-DM_RELAXED' in os.environ.get('M_DEFS', '').split() else FAST_WPW = [] if EXACT or os.environ.get('M_WPW') or 'WPW=' in os.environ.get('M_DEFS', '') else ['-DWPW=2'] # preset L's expansion stage (dec0l) two windows per wave as well; its input stage (enc0l) is slower that way L_WPW = [] if EXACT or 'WPW=' in os.environ.get('M_DEFS', '') + os.environ.get('M_DEFS_DEC0L', '') else ['-DWPW=2'] +# M_RESIDUAL_F32=1 (opt-in, not with M_EXACT=1): every layer_m.comp stage keeps the residual stream f32 across the MLP's +# 32-channel chunks, one binary16 rounding after them instead of one per chunk (engine/README.md) +RESF32 = ['-DM_RESIDUAL_F32'] if os.environ.get('M_RESIDUAL_F32') == '1' else [] +if RESF32 and EXACT: + raise SystemExit('M_RESIDUAL_F32=1 is part of the relaxed arithmetic: not with M_EXACT=1') SHADERS = {'enc1': 'm_enc1', 'enc2': 'm_enc2', 'enc3_tube': 'm_tube', 'dec2': 'm_dec2', 'dec1': 'm_dec1'} @@ -107,7 +112,7 @@ def main(): # the network in the arithmetic of ZLUDA's lowering of NVIDIA's PTX (TEXL): byte-identical to the CUDA backend # with NVIDIA's translated kernels in the harness (2026-10-11), where the native accuracy kernels' K32_MIX order # drifted to 36-41 dB through the FP8 tube - for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW) + for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW + RESF32) for s in ('enc0_m', 'dec0_m', 'post_m', 'down_m'): glsl(s + '.comp', s + '.spv', ['-DLDR=1'] if a.ldr and s != 'dec0_m' else []) glsl('down_m.comp', 'down_m_rgb10.spv', ['-DOUTPUT_RGB10=1'] + (['-DLDR=1'] if a.ldr else [])) glsl('post_m.comp', 'post_m_q.spv', ['-DTILE_W=15', '-DTILE_H=15'] + (['-DLDR=1'] if a.ldr else [])) @@ -158,10 +163,10 @@ def compile_l(a, one, R): subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', *flags, str(HERE / src), '-o', str(a.output / out)], check=True, stdout=subprocess.DEVNULL) ldr = ['-DLDR=1'] if a.ldr else [] # the network in the arithmetic of ZLUDA's lowering of NVIDIA's PTX (TEXL), as for preset M (L_NOTES.md) - for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW) + for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW + RESF32) for planes, suffix in ((0, ''), (1, '_planes')): - glsl('layer_m.comp', f'enc0_l{suffix}.spv', defines('enc0l') + ldr + [f'-DPLANES={planes}'] + FAST) - glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST + L_WPW) + glsl('layer_m.comp', f'enc0_l{suffix}.spv', defines('enc0l') + ldr + [f'-DPLANES={planes}'] + FAST + RESF32) + glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST + L_WPW + RESF32) # the reconstruction one render texel wider per tier than M's (4x4 taps): 11 (reconstruction/render >= 3), # 16 (the whole post_3_2 range), 12 (the compact tier: render/reconstruction <= 0.45, jitter within half a texel) for out, tile in (('post_l', 11), ('post_l_q', 16), ('post_l_c', 12)): diff --git a/engine/layer_m.comp b/engine/layer_m.comp index 84e673c..2d85faa 100644 --- a/engine/layer_m.comp +++ b/engine/layer_m.comp @@ -725,6 +725,20 @@ void main() EACH_T(N2) } // ------------------------------------------------------------ the MLP, 32 hidden channels at a time + // M_RESIDUAL_F32 (opt-in, part of M_RELAXED): the residual stays f32 across the MLP's chunks and is rounded to + // binary16 once after them, instead of after every chunk +#ifdef M_RESIDUAL_F32 +#ifndef M_RELAXED +#error layer_m.comp: M_RESIDUAL_F32 needs M_RELAXED +#endif + ACC RF_[WPW][NT]; +#define RF_IN(N) RF_[w][N] = ACC(res[N]); + FORW { EACH_T(RF_IN) } +#undef RF_IN +#define K32R(N, a0, b0, a1, b1) { RF_[w][N] = coopMatMulAdd((a1), (b1), coopMatMulAdd((a0), (b0), RF_[w][N])); } +#else +#define K32R(N, a0, b0, a1, b1) K32H(res[N], a0, b0, a1, b1) +#endif for (uint c = 0u; c < UC / 8u; ++c) { const uint ow1 = p.o[20u + 3u * c], ob1 = p.o[21u + 3u * c], ow2 = p.o[22u + 3u * c]; @@ -759,9 +773,15 @@ void main() A8 a0, a1; \ LDA8(a0, ow2, 0u, UC, 16u * (N)); \ LDA8(a1, ow2, 1u, UC, 16u * (N)); \ - FORW { K32H(res[N], a0, G0_[w], a1, G1_[w]) } } + FORW { K32R(N, a0, G0_[w], a1, G1_[w]) } } EACH_T(MLP2_N) } +#undef K32R +#ifdef M_RESIDUAL_F32 +#define RF_OUT(N) res[N] = HAC(RF_[w][N]); + FORW { EACH_T(RF_OUT) } +#undef RF_OUT +#endif #if KIND == 4 // ------------------------------------------------------------ L's expansion head: per-pixel reconstruction inputs SYNC // every wave is done with the V tiles in HO From 44433c320afcf4654563175baf9aed7da6c5f308 Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:28:40 +1300 Subject: [PATCH 15/19] engine/rr: opt-in D4R_RR2_UNGUARDED, the translated code without its exactness guards The parts of preset F that are NVIDIA's PTX translated by ptx2rr.py (encoder 0's and decoder 0's per-pixel code around the native blocks, the output passes) keep NVIDIA's arithmetic bit for bit with three kinds of guard: every register precise, an opaque zero OR-ed into binary16 stores and conversions (SUST_Z), and the NoContraction pass spv_exact_convert.py on their modules. At level 3 the native stages already run this GPU's own arithmetic, so D4R_RR2_UNGUARDED=1 drops the guards there too: ptx2rr.py --unguarded (no precise, SUST_Z 0) and no spv_exact_convert.py on those modules. compile_rr2.py refuses it below level 3. Off by default: the default model's files are byte-identical to the previous ones. Presented image against the default, RGB, peak 1: 1280x720 -> 1440p 56.0 dB on the reset frame, then 43.0 - 45.6 dB; 1114x626 -> 1080p 48.5, then 43.4 - 44.1; the moderate stress input 52.7, then 25 - 30. The exact build (level 0) is as far from the default (43.0 - 44.4; 43.2 - 44.3; 23.8 - 24.3) as from this build (43.0 - 44.4; 43.4 - 44.3; 23.8 - 24.3). Builds with D4R_RR2_SPLIT=1 too. RX 9070 XT, d4r-rr 720p -> 1440p, interleaved: frame 3.526 -> 3.501 ms (output 649 -> 624 us, dec0_1 446 -> 442). Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/compile_rr2.py | 21 +++++++++++++++------ engine/rr/ptx2rr.py | 9 ++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 455ea45..7fcfef7 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -84,6 +84,13 @@ SPLIT = os.environ.get('D4R_RR2_SPLIT', '0' if RELAXED >= 3 and 'enc0' in ONE_WA # differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13) POOLED = os.environ.get('D4R_RR2_POOLED', 'rgba32f') SURFACES = {'dec0_1': {176: 'rgba8_snorm', 184: 'rgba8', 192: 'rgba8'}, 'post': {120: POOLED}, 'pooled': {8: POOLED}} +# D4R_RR2_UNGUARDED=1 (opt-in, D4R_RR_RELAXED=3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, +# the output passes) without its exactness guards: no `precise` registers, no opaque zero OR-ed into binary16 stores and +# conversions (ptx2rr.py --unguarded) and no NoContraction pass (spv_exact_convert.py) on those modules, so the driver may +# contract, fold and fuse there as it does in the relaxed native stages (engine/rr/README.md) +UNGUARDED = os.environ.get('D4R_RR2_UNGUARDED') == '1' +if UNGUARDED and RELAXED < 3: + raise SystemExit('D4R_RR2_UNGUARDED=1 is for D4R_RR_RELAXED=3') def find_module(ptx_dir, kernel): @@ -170,8 +177,8 @@ def main(): cmd += [f'--surface={off}={fmt}' for off, fmt in SURFACES.get(stage, {}).items()] # the hybrid stages' and the output passes' modules go through spv_exact_convert.py, so their conversions need # only the lighter guard (ptx2rr.py --exact-convert) - exact = (bool(a.native) and stage in HYBRID) or not network - cmd += ['--exact-convert'] if exact else [] + exact = ((bool(a.native) and stage in HYBRID) or not network) and not UNGUARDED + cmd += ['--exact-convert'] if exact else ['--unguarded'] if UNGUARDED else [] subprocess.run(cmd, check=True, stderr=subprocess.DEVNULL) info = json.loads(generated.with_suffix('.json').read_text()) weights = '-' @@ -195,8 +202,9 @@ def main(): subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', *defs, f'-I{work}', f'-I{HERE / "rr"}', str(work / 'enc0_g.comp'), '-o', str(out / 'enc0g.spv')], check=True, stdout=subprocess.DEVNULL) - subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / 'enc0g.spv')], - check=True, stdout=subprocess.DEVNULL) + if not UNGUARDED: + subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / 'enc0g.spv')], + check=True, stdout=subprocess.DEVNULL) record = int((work / 'enc0_g.record').read_text()) if RELAXED >= 3 and stage in ONE_WAVE and (split or GROUP == 1): how = how + ['--one-wave'] @@ -206,8 +214,9 @@ def main(): subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', *defs, f'-I{work}', f'-I{HERE / "rr"}', str(work / f'{stage}_m.comp'), '-o', str(out / f'{stage}.spv')], check=True, stdout=subprocess.DEVNULL) - subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / f'{stage}.spv')], - check=True, stdout=subprocess.DEVNULL) + if not UNGUARDED: + subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / f'{stage}.spv')], + check=True, stdout=subprocess.DEVNULL) else: subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', str(generated), '-o', str(out / f'{stage}.spv')], check=True, stdout=subprocess.DEVNULL) diff --git a/engine/rr/ptx2rr.py b/engine/rr/ptx2rr.py index 515982d..a279e36 100755 --- a/engine/rr/ptx2rr.py +++ b/engine/rr/ptx2rr.py @@ -950,6 +950,9 @@ class RRT(base.T): w(f'shared uint mms[{384 * (self.bx * self.by * self.bz // 32)}]; // mma_e4m3_f16 scratch, per subgroup') def runtime(self): + if getattr(self, 'unguarded', False): # --unguarded: no opaque zero + assert '#define SUST_Z (gl_NumWorkGroups.z - 1u)' in RR_RUNTIME + return RR_RUNTIME.replace('#define SUST_Z (gl_NumWorkGroups.z - 1u)', '#define SUST_Z 0u') return RR_RUNTIME @@ -976,6 +979,9 @@ def main(): ap.add_argument('--oob-report', action='store_true', help='debugging: count out-of-range buffer accesses') ap.add_argument('--exact-convert', action='store_true', help='the module goes through spv_exact_convert.py: lighter guards on binary16 conversions (CVT_GUARD)') + ap.add_argument('--unguarded', action='store_true', + help="none of the exactness guards: no 'precise' registers and no opaque zero OR-ed into binary16 " + "stores and conversions, so the driver may contract, fold and fuse (compile_rr2.py D4R_RR2_UNGUARDED)") ns = ap.parse_args() block = tuple(int(x) for x in ns.block.split(',')) if ns.block else default_block(ns.kernel) formats = {int(k): v for k, v in (x.split('=') for x in ns.surface)} @@ -991,8 +997,9 @@ def main(): t.probes = probes t.probe_block = tuple(int(x) for x in ns.probe_block.split(',')) t.oob = ns.oob_report + t.unguarded = ns.unguarded return t - t = base.build(make, False, False, False) + t = base.build(make, False, ns.unguarded, False) for off, fmt in formats.items(): if off in t.surfaces: t.surfaces[off] = fmt From b42faa992823afab40f78e6306da2049b27bfefa Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:03:21 +1300 Subject: [PATCH 16/19] engine: README: the byte-identical speed-ups for presets M, L and F Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/README.md | 17 +++++++++++++++++ engine/rr/README.md | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/engine/README.md b/engine/README.md index e0454d6..adf16b1 100644 --- a/engine/README.md +++ b/engine/README.md @@ -1093,6 +1093,23 @@ on the Vulkan shaders unless `[Engine] Network = hip`: A model built with no options was compared shader by shader with the explicitly optioned build of each row, and the exact switches still reproduce the captures (`build/rr/f/nv/defaults.sh`). +### Byte-identical speed-ups (2026-10-19) + +RX 9070 XT, RADV 26.2.2, `d4r-m` 1280x720 -> 2560x1440 with `D4R_ENGINE_M_LAYER_STAMPS=1`. Every output frame, and +everything `D4R_M_DUMP` writes, is byte-identical to the previous build (M and L, HDR and LDR, the three +reconstruction tile tiers and the stress inputs). + +| change | stage | before -> after (us) | +|---|---|---| +| `layer_m.comp`: hardware FP8 saturation where a value cannot be infinite, and the V tiles in a buffer that is dead during the attention (more waves per SIMD) | frame | M 1583 -> 1566, L 2349 -> 2293 | +| `post_m.comp`: the second history candidate reuses the first one's sample when both have the same vector | reconstruction, uniform motion | 325 -> 288 | +| `post_m.comp`: both rows of a pixel pair take their Gaussian taps in one pass | reconstruction | 287 -> 265 (L 355 -> 320) | +| `down_m.comp`: the horizontal pass reads its texels into registers, no shared source tile | downsample | L 102 -> 91, M 117 -> 113 | +| `layer_m.comp`, `compile_m.py`: L's expansion stage at two windows per wave (`M_DEFS_DEC0L=-DWPW=1` for one) | expansion | 383 -> 366 | + +The network layers are separated by an execution-only barrier (`emptyStage`). A memory barrier there gives the same +output and costs about 60 us (M 1520 -> 1592), so it stays. + ## Preset L (runtime and shim path exist; experimental) Preset L (render preset 12, NVIDIA's model for Ultra Performance) is M's pipeline with its own input, expansion and diff --git a/engine/rr/README.md b/engine/rr/README.md index f7efb44..e0ac464 100644 --- a/engine/rr/README.md +++ b/engine/rr/README.md @@ -615,3 +615,17 @@ was measured and not built for D / E (`build/rr/f/nv/dstage.sh`, `dnl.sh`, level loads between a window's waves) decoder 0 takes the same time (803-818 us), encoder 1 the same, decoder 1 10% less. These windows also have 96 key tokens (3 KiB each for the tokens, the keys and the values at C = 32), which does not fit the 4 KiB a single wave can hold at full rate. + +### Byte-identical speed-ups (2026-10-19) + +RX 9070 XT, RADV 26.2.2, 1280x720 -> 2560x1440 with `D4R_ENGINE_RR_STAGE_STAMPS=1`. Every image of every frame is +byte-identical to the previous model on the replay and stress inputs, at levels 0 to 3. + +- `ptx2rr.py`: OCML's tanh (decoder 0's second kernel) in f32 where f64 was not needed. Exp's error term becomes + Dekker's exact product error; the polynomial below 0.625 stays in f64, because the driver folds away the f32 + rounding between its f64 steps. Identical to the previous function on all 2^32 inputs. dec0_1 475 -> 446 us, frame + 3.568 -> 3.537 ms. +- `exposure_f.comp` PASS 2: the exposure in one pass as NGX does it, each block adding its fixed-point sum to a 64-bit + total with an atomic add and the last block finishing. It needs `shaderBufferInt64Atomics`, which vkd3d-proton + enables; the shim reads the device's features through `GetDeviceFeatures`. Without it, or with a model that has no + `exposure_f2.spv`, the two passes run (`D4R_RR_TEST_NO_INT64_ATOMICS=1` forces them). Exposure 25.2 -> 14.9 us. From fe5999dfbff619ae5422cfad28430fd1588b75be Mon Sep 17 00:00:00 2001 From: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:03:34 +1300 Subject: [PATCH 17/19] engine: README: the opt-in M_RESIDUAL_F32, D4R_RR2_UNGUARDED and D4R_RR2_POOLED builds measured Signed-off-by: itchyfeetleech <169226071+itchyfeetleech@users.noreply.github.com> --- engine/README.md | 9 +++++++++ engine/compile_rr2.py | 5 +++-- engine/rr/README.md | 13 +++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/engine/README.md b/engine/README.md index adf16b1..6ce22c0 100644 --- a/engine/README.md +++ b/engine/README.md @@ -1110,6 +1110,15 @@ reconstruction tile tiers and the stress inputs). The network layers are separated by an execution-only barrier (`emptyStage`). A memory barrier there gives the same output and costs about 60 us (M 1520 -> 1592), so it stays. +### `M_RESIDUAL_F32` (opt-in, 2026-10-19) + +`M_RESIDUAL_F32=1 python3 engine/compile_m.py ...` keeps the residual stream in f32 across the MLP's 32-channel chunks +and rounds it to binary16 once after them, instead of after every chunk. It needs the relaxed build (not +`M_EXACT=1`) and is off by default. 1280x720 -> 1440p: M 1521 -> 1410 us, L 2195 -> 2077. Against the default build, +final RGB, peak 1: 43 - 46 dB (L 45 - 49) on smooth frames, 17 - 18 dB on noise. The exact build is as far from the +default, so this moves no further from NVIDIA's arithmetic than `M_RELAXED` already does. Not checked against an +NVIDIA card. + ## Preset L (runtime and shim path exist; experimental) Preset L (render preset 12, NVIDIA's model for Ultra Performance) is M's pipeline with its own input, expansion and diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 7fcfef7..0d8fed8 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -80,8 +80,9 @@ GROUP = int(os.environ.get('D4R_RR2_GROUP', '1')) # D4R_RR2_SPLIT=1 / 0 forces the gather pass / the single shader. SPLIT = os.environ.get('D4R_RR2_SPLIT', '0' if RELAXED >= 3 and 'enc0' in ONE_WAVE else '1') != '0' # The pooled histories as RGBA32F holding q / 1023, as ZLUDA backs them (16 bytes a pixel). D4R_RR2_POOLED=rgb10_a2 -# makes them real UNORM 10:10:10:2 images (0.1 ms faster at 4K), but the hardware's bilinear filter then rounds -# differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13) +# makes them real UNORM 10:10:10:2 images (0.1 ms faster at 4K, 70 MB less), but the hardware's bilinear filter then +# rounds differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13); on +# random-motion noise also at 2x (29 dB on the history frames, engine/rr/README.md) POOLED = os.environ.get('D4R_RR2_POOLED', 'rgba32f') SURFACES = {'dec0_1': {176: 'rgba8_snorm', 184: 'rgba8', 192: 'rgba8'}, 'post': {120: POOLED}, 'pooled': {8: POOLED}} # D4R_RR2_UNGUARDED=1 (opt-in, D4R_RR_RELAXED=3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, diff --git a/engine/rr/README.md b/engine/rr/README.md index e0ac464..3e1f6a9 100644 --- a/engine/rr/README.md +++ b/engine/rr/README.md @@ -629,3 +629,16 @@ byte-identical to the previous model on the replay and stress inputs, at levels total with an atomic add and the last block finishing. It needs `shaderBufferInt64Atomics`, which vkd3d-proton enables; the shim reads the device's features through `GetDeviceFeatures`. Without it, or with a model that has no `exposure_f2.spv`, the two passes run (`D4R_RR_TEST_NO_INT64_ATOMICS=1` forces them). Exposure 25.2 -> 14.9 us. + +### Opt-in builds for preset F (2026-10-19) + +Against the default build, presented image, RGB, peak 1, 1280x720 -> 1440p. For scale, the exact build +(`D4R_RR_RELAXED=0`) is 43.0 - 44.4 dB from the default. + +- `D4R_RR2_UNGUARDED=1` (level 3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, the output + passes) without its exactness guards: no `precise`, no opaque zero on binary16 stores, no `spv_exact_convert.py`. + 56.0 dB on the reset frame, then 43.0 - 45.6 dB, as far from the exact build as the default is. Frame 3.526 -> 3.501 + ms. +- `D4R_RR2_POOLED=rgb10_a2` (since 2026-10-13): the pooled histories as UNORM 10:10:10:2 images, 31 MB less at 1440p. + Identical at 1280x720 -> 1440p and on reset frames; 43.7 - 47.2 dB on the history frames at 1114x626 -> 1080p and + 28.8 dB with random motion. Frame 3.526 -> 3.484 ms, 3.456 with `D4R_RR2_UNGUARDED=1` as well. From ca93d0fbf7eaba31012c8290b9dcc3ff452a04bc Mon Sep 17 00:00:00 2001 From: woutermans Date: Sun, 18 Oct 2026 17:26:19 +0200 Subject: [PATCH 18/19] shim: retry application-ID init with d4r's project identity NGX 617.14 rejects application-ID initialization on this path (Cyberpunk 2077: 0xBAD0000C, or SR unavailable with 0xBAD00004). Streamline's RR plugin reads capabilities right after that first Init, so a failure there hides RR for the session even though the game's later project-ID Init works. When the app-ID NVSDK_NGX_CUDA_Init fails and the caller gave no project identity, retry once with d4r's own project identity (the diagnostics' default) so the first Init succeeds; log the retry as D4R_NGX_IDENTITY_FALLBACK. Port of the windows-branch fix in tools/windows/ngx_runtime_initialization.h. Co-authored-by: omp --- tools/d4r_nvngx_shim.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tools/d4r_nvngx_shim.cpp b/tools/d4r_nvngx_shim.cpp index 7fd5249..29511f1 100644 --- a/tools/d4r_nvngx_shim.cpp +++ b/tools/d4r_nvngx_shim.cpp @@ -2046,6 +2046,22 @@ static NgxResult initialize(unsigned long long applicationId, const wchar_t* dat result = g.worker.call([&] { return g.ngx.init(id, g.dataPath.c_str(), &g.featureInfo, apiVersion); }); logf("NVSDK_NGX_CUDA_Init(app=%llu, sdk=0x%x) -> 0x%08x", id, version, result); } + const bool hasProject = project != nullptr && !project->id.empty(); + if (result != NGX_SUCCESS && !hasProject && g.ngx.initProjectId != nullptr) + { + // NGX 617.14 rejects application-ID initialization on this path (Cyberpunk 2077: + // 0xBAD0000C, or SR unavailable with 0xBAD00004). Streamline's RR plugin reads + // capabilities right after that first Init, so a failure there hides RR for the + // session even though the game's later project-ID Init works. Retry with d4r's + // own project identity (the diagnostics' default) so the first Init succeeds. + logf("D4R_NGX_IDENTITY_FALLBACK application=%llu: 0x%08x", id, result); + result = g.worker.call([&] { + return g.ngx.initProjectId("24480451-f00d-face-1304-0308dabad187", 0, "1.0", g.dataPath.c_str(), + apiVersion, &g.featureInfo); + }); + logf("NVSDK_NGX_CUDA_Init_ProjectID(d4r fallback project=24480451-f00d-face-1304-0308dabad187, engine=0 1.0, sdk=0x%x) -> 0x%08x", + version, result); + } using LoadErrorFn = const char*(WINAPI*)(); const auto loadError = reinterpret_cast(reinterpret_cast(GetProcAddress(g.cuda, "d4rLoadError"))); if (loadError != nullptr && loadError()[0] != '\0') From 2558fcbd3992525c72b06169caba373f1e048212 Mon Sep 17 00:00:00 2001 From: ayo <93222141+countervolts@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:36:40 -0400 Subject: [PATCH 19/19] Improve GPU compatibility, RDNA2 kernels and startup diagnostics - Match render/compute adapters and gate RR models by GPU capabilities - Add RDNA2 native texture kernels and fix FP16 rounding - Harden model builds, cache validation and packaging - Add startup tracing, timeouts, documentation and regression tests --- docs/architecture.md | 2 + docs/linux-startup-diagnostics.md | 65 ++++++++++ docs/native-kernels.md | 6 +- install.sh | 7 +- kernels/build.sh | 33 +++-- kernels/k/pwin_common.h | 11 ++ kernels/native/build_native.sh | 4 +- kernels/native/output_tile.h | 5 +- kernels/native/ptx2hip.py | 4 +- kernels/native/ptx_rt.h | 10 ++ packaging/D4R_README.txt | 4 + packaging/build_engine_models.sh | 51 ++++++-- packaging/model_tools.py | 164 +++++++++++++++++++++++ scripts/build_d4r_nvngx_shim.sh | 3 +- scripts/build_release_glibc241.sh | 10 +- scripts/build_wine_nvcuda_bridge.sh | 2 + scripts/d4r_config.py | 6 + scripts/package_release.sh | 24 +++- scripts/runtime_build_id.sh | 7 + scripts/stage_engine_kit.sh | 1 + tests/test_gpu_compatibility.py | 149 +++++++++++++++++++++ tests/test_model_builder.py | 96 ++++++++++++++ tests/test_rdna2_native_output.py | 193 ++++++++++++++++++++++++++++ tests/test_shim_config.py | 14 ++ tools/d3d12_dlss_harness.cpp | 26 +++- tools/d4r_build.h | 4 + tools/d4r_engine_compat.h | 35 +++++ tools/d4r_fsr_rr_identity.h | 8 +- tools/d4r_gpu_identity.h | 26 ++++ tools/d4r_nvngx_shim.cpp | 62 ++++++++- tools/d4r_vulkan_frontend.inl | 7 + tools/wine_nvcuda_bridge.c | 173 +++++++++++++++++++------ 32 files changed, 1131 insertions(+), 81 deletions(-) create mode 100644 docs/linux-startup-diagnostics.md create mode 100644 packaging/model_tools.py create mode 100644 scripts/runtime_build_id.sh create mode 100644 tests/test_gpu_compatibility.py create mode 100644 tests/test_model_builder.py create mode 100644 tests/test_rdna2_native_output.py create mode 100644 tools/d4r_build.h create mode 100644 tools/d4r_engine_compat.h create mode 100644 tools/d4r_gpu_identity.h diff --git a/docs/architecture.md b/docs/architecture.md index ca078ed..00ce2e0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -112,6 +112,8 @@ OptiScaler 0.9.4 checks a relative `NvngxPath` against the process working direc ## Logs +For startup stalls, GPU resets and model-build failures, see [Linux startup diagnostics](linux-startup-diagnostics.md). + The launcher writes each run's logs to `~/.cache/d4r-dlss-captures/game--