Merge remote-tracking branch 'origin/linux' into linux

# Conflicts:
#	engine/rr/ptx2rr.py
#	install.sh
#	packaging/build_engine_models.sh

engine/rr/README.md

@@ -615,3 +615,30 @@ was measured and not built for D / E (`build/rr/f/nv/dstage.sh`, `dnl.sh`, level
   loads between a window's waves) decoder 0 takes the same time (803-818 us), encoder 1 the same, decoder 1 10% less.
   These windows also have 96 key tokens (3 KiB each for the tokens, the keys and the values at C = 32), which does
   not fit the 4 KiB a single wave can hold at full rate.
+
+### Byte-identical speed-ups (2026-10-19)
+
+RX 9070 XT, RADV 26.2.2, 1280x720 -> 2560x1440 with `D4R_ENGINE_RR_STAGE_STAMPS=1`. Every image of every frame is
+byte-identical to the previous model on the replay and stress inputs, at levels 0 to 3.
+
+- `ptx2rr.py`: OCML's tanh (decoder 0's second kernel) in f32 where f64 was not needed. Exp's error term becomes
+  Dekker's exact product error; the polynomial below 0.625 stays in f64, because the driver folds away the f32
+  rounding between its f64 steps. Identical to the previous function on all 2^32 inputs. dec0_1 475 -> 446 us, frame
+  3.568 -> 3.537 ms.
+- `exposure_f.comp` PASS 2: the exposure in one pass as NGX does it, each block adding its fixed-point sum to a 64-bit
+  total with an atomic add and the last block finishing. It needs `shaderBufferInt64Atomics`, which vkd3d-proton
+  enables; the shim reads the device's features through `GetDeviceFeatures`. Without it, or with a model that has no
+  `exposure_f2.spv`, the two passes run (`D4R_RR_TEST_NO_INT64_ATOMICS=1` forces them). Exposure 25.2 -> 14.9 us.
+
+### Opt-in builds for preset F (2026-10-19)
+
+Against the default build, presented image, RGB, peak 1, 1280x720 -> 1440p. For scale, the exact build
+(`D4R_RR_RELAXED=0`) is 43.0 - 44.4 dB from the default.
+
+- `D4R_RR2_UNGUARDED=1` (level 3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, the output
+  passes) without its exactness guards: no `precise`, no opaque zero on binary16 stores, no `spv_exact_convert.py`.
+  56.0 dB on the reset frame, then 43.0 - 45.6 dB, as far from the exact build as the default is. Frame 3.526 -> 3.501
+  ms.
+- `D4R_RR2_POOLED=rgb10_a2` (since 2026-10-13): the pooled histories as UNORM 10:10:10:2 images, 31 MB less at 1440p.
+  Identical at 1280x720 -> 1440p and on reset frames; 43.7 - 47.2 dB on the history frames at 1114x626 -> 1080p and
+  28.8 dB with random motion. Frame 3.526 -> 3.484 ms, 3.456 with `D4R_RR2_UNGUARDED=1` as well.

engine/rr/exposure_f.comp

@@ -1,33 +1,39 @@
 #version 460
-// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel, in two passes.
+// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel.
 //
 // NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block
 // sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish
-// turns the total into the exposure. The integer total does not depend on the order of the blocks, so PASS 0 writes
-// every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last block does:
+// turns the total into the exposure:
 //   exposure = 0.23547088 / max(mean luma, 0.005)               -> the 1x1 exposure image of this frame
 //   values   = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 }   -> the 16-byte buffer encoder 0 and the output passes read
 // with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero).
+// PASS 2 is that kernel (64-bit atomics on a storage buffer: shaderBufferInt64Atomics). Without them the runtime runs
+// two passes: PASS 0 writes every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last
+// block does. The total is an integer: it does not depend on the order of the blocks, so both give the same values.
 #extension GL_EXT_shader_explicit_arithmetic_types : require
 #extension GL_EXT_control_flow_attributes : require
 #ifndef PASS
 #define PASS 0
 #endif
+#if PASS == 2
+#extension GL_EXT_shader_atomic_int64 : require
+#extension GL_KHR_memory_scope_semantics : require
+#endif
 layout(set = 0, binding = 0, std430) readonly buffer Params {
     ivec2 last;        // render size - 1
     uint blocksX, blocks;
     float invCount;    // 10000 / (2^32 * render pixels)
     uint reset;        // the kernel's argument 32: no log ratio when set
 } p;
-#if PASS == 0
+#if PASS != 1
 // One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128,
 // then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the
 // same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1).
 layout(local_size_x = 16) in;
 layout(set = 0, binding = 1) uniform sampler2D color;
-layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
 shared float col[16];
-void main()
+// the block's fixed-point sum, in thread 0
+uint64_t blockSum()
 {
     const uint tx = gl_LocalInvocationID.x;
     const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16;
@@ -49,33 +55,37 @@ void main()
     precise float c = w1 + w0;
     col[tx] = c;
     barrier();
-    if (tx == 0u) {
-        precise float d[8];
-        [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
-        precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
-        precise float f0 = e2 + e0, f1 = e3 + e1;
-        precise float g = f1 + f0;
-        sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = uint64_t(roundEven(g * 429496.71875));
-    }
+    if (tx != 0u) return 0ul;
+    precise float d[8];
+    [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
+    precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
+    precise float f0 = e2 + e0, f1 = e3 + e1;
+    precise float g = f1 + f0;
+    return uint64_t(roundEven(g * 429496.71875));
+}
+#endif
+#if PASS == 0
+layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
+void main()
+{
+    const uint64_t s = blockSum();
+    if (gl_LocalInvocationID.x == 0u) sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = s;
 }
 #else
+#if PASS == 1
 layout(local_size_x = 64) in;
 shared uint64_t part[64];
 layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums;
-layout(set = 0, binding = 2, std430) writeonly buffer Values { vec4 v; } values;
-layout(set = 0, binding = 3) uniform sampler2D previous;
-layout(set = 0, binding = 4, r16f) uniform writeonly image2D current;
-void main()
+#else
+// s[0]: the total, s[1]: the blocks done; the last block leaves both 0 for the next frame
+layout(set = 0, binding = 2, std430) buffer Sums { uint64_t s[]; } sums;
+#endif
+layout(set = 0, binding = PASS + 1, std430) writeonly buffer Values { vec4 v; } values;
+layout(set = 0, binding = PASS + 2) uniform sampler2D previous;
+layout(set = 0, binding = PASS + 3, r16f) uniform writeonly image2D current;
+// the last block's work, from the frame's total
+void finish(uint64_t total)
 {
-    // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
-    const uint tid = gl_LocalInvocationID.x;
-    uint64_t mine = 0ul;
-    for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
-    part[tid] = mine;
-    barrier();
-    if (tid != 0u) return;
-    uint64_t total = 0ul;
-    for (uint b = 0u; b < 64u; ++b) total += part[b];
     precise float mean = p.invCount * float(total);
     precise float e = 0.23547088 / max(mean, 0.005);
     imageStore(current, ivec2(0), vec4(e));
@@ -90,4 +100,33 @@ void main()
     if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667;
     values.v = vec4(cur, prev, 1.0 / cur, ratio);
 }
+#if PASS == 1
+void main()
+{
+    // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
+    const uint tid = gl_LocalInvocationID.x;
+    uint64_t mine = 0ul;
+    for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
+    part[tid] = mine;
+    barrier();
+    if (tid != 0u) return;
+    uint64_t total = 0ul;
+    for (uint b = 0u; b < 64u; ++b) total += part[b];
+    finish(total);
+}
+#else
+void main()
+{
+    const uint64_t s = blockSum();
+    if (gl_LocalInvocationID.x != 0u) return;
+    atomicAdd(sums.s[0], s, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    // the release orders this block's add before its count, and the last block's acquire every block's add before
+    // its read of the total
+    if (atomicAdd(sums.s[1], 1ul, gl_ScopeDevice, gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease) != uint64_t(p.blocks - 1u))
+        return;
+    const uint64_t total = atomicExchange(sums.s[0], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    atomicExchange(sums.s[1], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    finish(total);
+}
+#endif
 #endif

engine/rr/ptx2rr.py

@@ -139,31 +139,49 @@ void SA32(uint a, uint v) { CHECKS(a + 2u);
 // UNORM 10:10:10:2 surfaces (the pooled histories) are RGBA32F images holding the quantised values, like ZLUDA's.
 // tanh.approx.f32 as the reference computes it: ROCm's OCML tanh (device-libs ocml/src/tanhF.cl) with LLVM's AMDGPU
 // lowering of exp (lowerFEXP, fast-FMA path): the same fused operations, hardware exp2, ldexp and reciprocal.
-// (a fused f32 a * b + c through f64: the product is exact there and the sum is rounded to 53 bits before the f32
-// rounding. The driver's own f32 fma is a product and a sum whenever its product also exists on its own, as in
-// exp's ph / fma(x, c, -ph) pair.)
-float fmad(float a, float b, float c) { precise double s = double(a) * double(b) + double(c); return float(s); }
+// Up to 2026-10-19 every fused operation went through f64 (the product exact there, the sum rounded to 53 bits, then
+// to f32), which this GPU runs at a small fraction of the f32 rate. They are the driver's f32 fma now, with two
+// exceptions that keep every result bit for bit what it was, on all 2^32 arguments (RX 9070 XT, RADV; NaN payloads
+// included):
+// - the polynomial stays in f64: the driver drops the f32 rounding between nested f64 operations (it folds
+//   f2f64(f2f32(x)) to x; the conversions are not NoContraction), so its four nested steps always ran in f64 with one
+//   rounding at the end. In f32 (true fma), 24,836 arguments round one f32 step differently, 2 of them in binary16.
+// - exp's error term fma(w, c, -ph) next to ph = w * c: the driver makes that the product it already has and a sum,
+//   exactly 0, even with every operand precise or opaque. It is Dekker's exact product error: w split into two halves
+//   of 12 bits (w * 4097 = w * (2^12 + 1)), log2(e) into 1.4423828125 + 0.0003122091293334961 (11 and 12 bits), so
+//   every partial product is exact and so is each sum.
+// A NaN argument keeps its payload, quieted and positive, as the f64 conversion left it.
 float tanh_ocml(float x)
 {
     const float y = abs(x);
     precise float z;
     if (y < 0.625) {
         precise float y2 = y * y;
-        precise float p = fmad(y2, fmad(y2, fmad(y2, fmad(y2, -0.005700020585209131, 0.02063407190144062), -0.053737930953502655), 0.13331416249275208), -0.3333328068256378);
+        precise double y2d = double(y2);
+        precise double pd = (((y2d * -0.005700020585209131 + 0.02063407190144062) * y2d - 0.053737930953502655) * y2d +
+                             0.13331416249275208) * y2d - 0.3333328068256378;
+        precise float p = float(pd);
         precise float yp = y * p;
-        z = fmad(y2, yp, y);
+        z = fma(y2, yp, y);
     } else {
         const float w = 2.0 * y;
         precise float ph = w * 1.4426950216293335;
-        precise float f0 = fmad(w, 1.4426950216293335, -ph);
-        precise float pl = fmad(w, 1.925962855864327e-08, f0);
+        precise float wt = w * 4097.0;
+        precise float wh = wt - (wt - w);
+        precise float wl = w - wh;
+        precise float f1 = wh * 1.4423828125 - ph;
+        precise float f2 = f1 + wh * 0.0003122091293334961;
+        precise float f3 = f2 + wl * 1.4423828125;
+        precise float f0 = f3 + wl * 0.0003122091293334961;
+        precise float pl = fma(w, 1.925962855864327e-08, f0);
         const float e = roundEven(ph);
         precise float d = ph - e;
         precise float a = d + pl;
         float t = ldexp(exp2(a), int(e));
         if (w > 88.72283172607422) t = uintBitsToFloat(0x7f800000u);
         precise float s1 = t + 1.0;
-        z = fmad(-2.0, rcp_a(s1), 1.0);
+        z = fma(-2.0, rcp_a(s1), 1.0);
+        if (isnan(z)) z = uintBitsToFloat(floatBitsToUint(y) | 0x00400000u);
     }
     return x < 0.0 || floatBitsToUint(x) == 0x80000000u ? -z : z;
 }
@@ -1032,7 +1050,11 @@ class RRT(base.T):
         w(f'shared uint mms[{384 * (self.bx * self.by * self.bz // 32)}];   // mma_e4m3_f16 scratch, per subgroup')
 
     def runtime(self):
-        return (RR_RUNTIME + (ROOT / 'engine' / 'round_half.glsl').read_text() +
+        runtime = RR_RUNTIME
+        if getattr(self, 'unguarded', False):   # --unguarded: no opaque zero
+            assert '#define SUST_Z (gl_NumWorkGroups.z - 1u)' in RR_RUNTIME
+            runtime = RR_RUNTIME.replace('#define SUST_Z (gl_NumWorkGroups.z - 1u)', '#define SUST_Z 0u')
+        return (runtime + (ROOT / 'engine' / 'round_half.glsl').read_text() +
                 (ROOT / 'engine' / 'cnn_sr' / 'mma_f16.glsl').read_text())
 
 
@@ -1061,6 +1083,9 @@ def main():
     ap.add_argument('--oob-report', action='store_true', help='debugging: count out-of-range buffer accesses')
     ap.add_argument('--exact-convert', action='store_true',
                     help='the module goes through spv_exact_convert.py: lighter guards on binary16 conversions (CVT_GUARD)')
+    ap.add_argument('--unguarded', action='store_true',
+                    help="none of the exactness guards: no 'precise' registers and no opaque zero OR-ed into binary16 "
+                         "stores and conversions, so the driver may contract, fold and fuse (compile_rr2.py D4R_RR2_UNGUARDED)")
     ns = ap.parse_args()
     block = tuple(int(x) for x in ns.block.split(',')) if ns.block else default_block(ns.kernel)
     formats = {int(k): v for k, v in (x.split('=') for x in ns.surface)}
@@ -1077,8 +1102,9 @@ def main():
         t.probes = probes
         t.probe_block = tuple(int(x) for x in ns.probe_block.split(','))
         t.oob = ns.oob_report
+        t.unguarded = ns.unguarded
         return t
-    t = base.build(make, False, False, False)
+    t = base.build(make, False, ns.unguarded, False)
     for off, fmt in formats.items():
         if off in t.surfaces:
             t.surfaces[off] = fmt