engine/rr: preset F's exposure in one pass with 64-bit atomics

exposure_f.comp ran NGX's auto exposure in two passes: the 16x16 blocks wrote
their fixed-point luma sums, then a single workgroup added them after a
barrier and finished (11 us of a 25 us stage). PASS 2 does what NVIDIA's
kernel does: each block adds its sum to a 64-bit total with an atomic add
and counts itself in with an acquire-release add; the last block reads the
total, finishes and leaves both words zero for the next frame. The total is
an integer, the same in any order, so the exposure image and the values are
the two passes' bit for bit. compile_rr2.py builds exposure_f2.spv next to
the two passes' shaders (the block sum and the finish are shared functions
now; their arithmetic is unchanged).

64-bit buffer atomics are a device feature (shaderBufferInt64Atomics), so
RModel takes whether the device has it enabled (default no) and preset F
uses the one pass only then and only with a model that has exposure_f2.spv;
otherwise the two passes as before. vkd3d-proton creates its device with
every feature the GPU reports except a list it turns off, which does not
include this one (it backs D3D12's 64-bit atomics with it); the shim reads
the device's features through the interop interface (GetDeviceFeatures) and
logs the result. d4r-rr enables every feature the GPU has, and
D4R_RR_TEST_NO_INT64_ATOMICS=1 turns this one off to run the two passes.
Ray Reconstruction never runs on the native Vulkan frontend's device.

Every image of every frame byte-identical to the previous build (1280x720
-> 1440p, 1114x626 -> 1080p, 960x540 -> 1080p, the stress inputs, with
D4R_RR_TEST_ROBUST2=1, at levels 0 and 3, with D4R_RR2_SPLIT=1), and so is
the two-pass fallback of the new build; presets D and E unchanged. Controls:
the last block finishing with twice the total changes 64 of the 90 images of
the 1280x720 sequence (and nothing under the fallback), finishing without
its own block's sum 58 of 90.

RX 9070 XT, d4r-rr 720p -> 1440p, interleaved: exposure 25.2 -> 14.9 us,
frame 3.536 -> 3.526 ms (the fallback 25.1 us).

Signed-off-by: itchyfeetleech <[email protected]>

engine/compile_rr2.py

@@ -12,7 +12,7 @@ D4R_CUDA_REPLAY_DUMP_FILTER=rr2 D4R_CUDA_LAUNCH_STATS_FILTER=rr2 and preset hint
                        table of its PTX module; enc0-aux.bin is encoder 0's second table (argument 104)
   <stage>.args         the argument block of the capture's frame, pointers and handles cleared: the runtime keeps its
                        constants and fills what depends on the frame or the geometry (engine/runtime_rr2.inc)
-  exposure_f0.spv, exposure_f1.spv   engine/rr/exposure_f.comp
+  exposure_f0.spv, exposure_f1.spv   engine/rr/exposure_f.comp (exposure_f2.spv: both in one, 64-bit atomics)
   rr.txt               the manifest RModel reads ("D4RR0002")
 With --native, the stages engine/rr/rr2_m.comp implements (enc1 ... enc5, dec4 ... dec1, dec0's first kernel) are that
 shader with weights read from the DLL itself (engine/rr/rr2_pack.py) instead of the translation: bit-identical to
@@ -249,7 +249,7 @@ def main():
         if hybrid and split:   # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges
             lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3')
         print(f'{stage}: {kernel}, {len(args)} argument bytes' + (' (native block)' if hybrid else ''))
-    for p in (0, 1):
+    for p in (0, 1, 2):
         subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}',
                         str(HERE / 'rr' / 'exposure_f.comp'), '-o', str(out / f'exposure_f{p}.spv')],
                        check=True, stdout=subprocess.DEVNULL)

engine/replay_rr.cpp

@@ -158,6 +158,8 @@ int main(int argc, char** argv) try {
     if (std::getenv("D4R_RR_TEST_NO_ROBUST")) { f2.features.robustBufferAccess = VK_FALSE; v13.robustImageAccess = VK_FALSE; }
     // D4R_RR_TEST_NO_COOPMAT=1: a device like vkd3d-proton's, without the cooperative-matrix and float8 features
     if (std::getenv("D4R_RR_TEST_NO_COOPMAT")) { di.enabledExtensionCount = 0; v13.pNext = nullptr; }
+    // D4R_RR_TEST_NO_INT64_ATOMICS=1: no 64-bit buffer atomics (preset F's exposure then takes its two passes)
+    if (std::getenv("D4R_RR_TEST_NO_INT64_ATOMICS")) v12.shaderBufferInt64Atomics = v12.shaderSharedInt64Atomics = VK_FALSE;
     CK(vkCreateDevice(c.physical, &di, nullptr, &c.device));
     vkGetDeviceQueue(c.device, family, 0, &c.queue);
     VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pi.queueFamilyIndex = family;
@@ -169,7 +171,7 @@ int main(int argc, char** argv) try {
     std::string key;
     uint32_t rw, rh, ow, oh;
     frames >> key >> rw >> rh >> ow >> oh;
-    d4r::RModel model(c.physical, c.device, modelDir);
+    d4r::RModel model(c.physical, c.device, modelDir, VK_NULL_HANDLE, v12.shaderBufferInt64Atomics == VK_TRUE);
     d4r::REngine engine(model, rw, rh, ow, oh);
     const char* names[] = {"color", "motion", "depth", "normals", "roughness", "diffuse", "specular"};
     const VkFormat formats[] = {VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R32_SFLOAT,

engine/rr/exposure_f.comp

@@ -1,33 +1,39 @@
 #version 460
-// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel, in two passes.
+// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel.
 //
 // NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block
 // sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish
-// turns the total into the exposure. The integer total does not depend on the order of the blocks, so PASS 0 writes
-// every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last block does:
+// turns the total into the exposure:
 //   exposure = 0.23547088 / max(mean luma, 0.005)               -> the 1x1 exposure image of this frame
 //   values   = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 }   -> the 16-byte buffer encoder 0 and the output passes read
 // with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero).
+// PASS 2 is that kernel (64-bit atomics on a storage buffer: shaderBufferInt64Atomics). Without them the runtime runs
+// two passes: PASS 0 writes every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last
+// block does. The total is an integer: it does not depend on the order of the blocks, so both give the same values.
 #extension GL_EXT_shader_explicit_arithmetic_types : require
 #extension GL_EXT_control_flow_attributes : require
 #ifndef PASS
 #define PASS 0
 #endif
+#if PASS == 2
+#extension GL_EXT_shader_atomic_int64 : require
+#extension GL_KHR_memory_scope_semantics : require
+#endif
 layout(set = 0, binding = 0, std430) readonly buffer Params {
     ivec2 last;        // render size - 1
     uint blocksX, blocks;
     float invCount;    // 10000 / (2^32 * render pixels)
     uint reset;        // the kernel's argument 32: no log ratio when set
 } p;
-#if PASS == 0
+#if PASS != 1
 // One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128,
 // then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the
 // same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1).
 layout(local_size_x = 16) in;
 layout(set = 0, binding = 1) uniform sampler2D color;
-layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
 shared float col[16];
-void main()
+// the block's fixed-point sum, in thread 0
+uint64_t blockSum()
 {
     const uint tx = gl_LocalInvocationID.x;
     const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16;
@@ -49,33 +55,37 @@ void main()
     precise float c = w1 + w0;
     col[tx] = c;
     barrier();
-    if (tx == 0u) {
-        precise float d[8];
-        [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
-        precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
-        precise float f0 = e2 + e0, f1 = e3 + e1;
-        precise float g = f1 + f0;
-        sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = uint64_t(roundEven(g * 429496.71875));
-    }
+    if (tx != 0u) return 0ul;
+    precise float d[8];
+    [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
+    precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
+    precise float f0 = e2 + e0, f1 = e3 + e1;
+    precise float g = f1 + f0;
+    return uint64_t(roundEven(g * 429496.71875));
+}
+#endif
+#if PASS == 0
+layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
+void main()
+{
+    const uint64_t s = blockSum();
+    if (gl_LocalInvocationID.x == 0u) sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = s;
 }
 #else
+#if PASS == 1
 layout(local_size_x = 64) in;
 shared uint64_t part[64];
 layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums;
-layout(set = 0, binding = 2, std430) writeonly buffer Values { vec4 v; } values;
-layout(set = 0, binding = 3) uniform sampler2D previous;
-layout(set = 0, binding = 4, r16f) uniform writeonly image2D current;
-void main()
+#else
+// s[0]: the total, s[1]: the blocks done; the last block leaves both 0 for the next frame
+layout(set = 0, binding = 2, std430) buffer Sums { uint64_t s[]; } sums;
+#endif
+layout(set = 0, binding = PASS + 1, std430) writeonly buffer Values { vec4 v; } values;
+layout(set = 0, binding = PASS + 2) uniform sampler2D previous;
+layout(set = 0, binding = PASS + 3, r16f) uniform writeonly image2D current;
+// the last block's work, from the frame's total
+void finish(uint64_t total)
 {
-    // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
-    const uint tid = gl_LocalInvocationID.x;
-    uint64_t mine = 0ul;
-    for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
-    part[tid] = mine;
-    barrier();
-    if (tid != 0u) return;
-    uint64_t total = 0ul;
-    for (uint b = 0u; b < 64u; ++b) total += part[b];
     precise float mean = p.invCount * float(total);
     precise float e = 0.23547088 / max(mean, 0.005);
     imageStore(current, ivec2(0), vec4(e));
@@ -90,4 +100,33 @@ void main()
     if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667;
     values.v = vec4(cur, prev, 1.0 / cur, ratio);
 }
+#if PASS == 1
+void main()
+{
+    // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
+    const uint tid = gl_LocalInvocationID.x;
+    uint64_t mine = 0ul;
+    for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
+    part[tid] = mine;
+    barrier();
+    if (tid != 0u) return;
+    uint64_t total = 0ul;
+    for (uint b = 0u; b < 64u; ++b) total += part[b];
+    finish(total);
+}
+#else
+void main()
+{
+    const uint64_t s = blockSum();
+    if (gl_LocalInvocationID.x != 0u) return;
+    atomicAdd(sums.s[0], s, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    // the release orders this block's add before its count, and the last block's acquire every block's add before
+    // its read of the total
+    if (atomicAdd(sums.s[1], 1ul, gl_ScopeDevice, gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease) != uint64_t(p.blocks - 1u))
+        return;
+    const uint64_t total = atomicExchange(sums.s[0], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    atomicExchange(sums.s[1], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
+    finish(total);
+}
+#endif
 #endif

engine/runtime.h

@@ -273,7 +273,10 @@ struct RFrameSettings {
 };
 class RModel {
 public:
-    RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE);
+    // int64Atomics: the device has shaderBufferInt64Atomics enabled; preset F then measures the exposure in one pass
+    // (engine/rr/exposure_f.comp PASS 2) instead of two
+    RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE,
+           bool int64Atomics = false);
     ~RModel();
     RModel(const RModel&) = delete;
     RModel& operator=(const RModel&) = delete;

engine/runtime_rr.inc

@@ -90,7 +90,7 @@ struct RModel::Impl {
     bool uploaded = false;
     Impl(VkPhysicalDevice physical, VkDevice dev) : device(physical, dev) {}
 };
-RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache)
+RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache, bool int64Atomics)
     : impl(std::make_unique<Impl>(physical, device)) {
     auto& m = *impl;
     // rr.txt: "D4RR0001", "preset N", one "stage NAME SHADER.spv SUBGROUP WEIGHTS TEXTURES SURFACES" line per stage in
@@ -103,7 +103,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di
     if (line.rfind("D4RR0002", 0) == 0) {
         m.preset = 6;
         m.v2 = std::make_unique<R2Model>();
-        m.v2->load(m.device, directory, lines, cache);
+        m.v2->load(m.device, directory, lines, cache, int64Atomics);
         return;
     }
     if (line.rfind("D4RR0001", 0) != 0) throw std::runtime_error("not a Ray Reconstruction model (rr.txt D4RR0001)");

engine/runtime_rr2.inc

@@ -44,9 +44,10 @@ struct R2Model {
     Buffer aux;                   // encoder 0's second table (argument 104)
     std::vector<char> auxPending;
     std::array<Pipeline, 2> exposure;
+    Pipeline exposureOne;         // both exposure passes in one (exposure_f2.spv, 64-bit atomics); else null
     bool particles = false;       // the model has R2DownP
     bool gather = false;          // the model has R2Gather
-    void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache) {
+    void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache, bool int64Atomics) {
         const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
         auto split = [](const std::string& list) {
             std::vector<std::string> items;
@@ -108,6 +109,8 @@ struct R2Model {
         blob(aux, auxPending, read(directory + "/enc0-aux.bin"));
         exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache);
         exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache);
+        if (int64Atomics && exists(directory + "/exposure_f2.spv"))
+            exposureOne.create(d.handle, directory + "/exposure_f2.spv", {b, t, b, b, t, s}, 64, cache);
     }
     void recordUpload(VkCommandBuffer cb) {   // the weights into the command buffer (recordContents)
         for (auto& st : stages) recordContents(cb, st.weights, st.pending);
@@ -172,7 +175,7 @@ struct R2Engine {
         std::vector<VkDescriptorSet> down;
         std::vector<std::array<VkDescriptorSet, 2>> downP;
         VkDescriptorSet pooled = VK_NULL_HANDLE, exposure0 = VK_NULL_HANDLE;
-        std::array<VkDescriptorSet, 2> exposure1{};
+        std::array<VkDescriptorSet, 2> exposure1{}, exposureOne{};
     };
     std::vector<Bindings> bindings;
     const Bindings* bound = nullptr;
@@ -337,7 +340,7 @@ struct R2Engine {
         exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16);
         exposure.last[0] = static_cast<int32_t>(rw - 1); exposure.last[1] = static_cast<int32_t>(rh - 1);
         exposure.invCount = static_cast<float>(10000.0 / (4294967296.0 * rw * rh));
-        sums.create(d, exposure.blocks * 8, storage);
+        sums.create(d, std::max(exposure.blocks, 2u) * 8, storage);   // the one-pass exposure: the total and the count
         const size_t tiles = src.size();
         auto place = [&](size_t bytes) { const VkDeviceSize at = paramBytes; paramBytes += (bytes + 255) & ~VkDeviceSize(255); return at; };
         for (unsigned i = 0; i < R2Network; ++i) { blocks[i] = model.stages[i].args; paramAt[i] = place(blocks[i].size()); }
@@ -593,17 +596,29 @@ struct R2Engine {
                     b.downP.push_back({stageSet(b, R2DownP, downPAt[i], 0, output), stageSet(b, R2DownP, downPAt[i], 1, output)});
             }
             b.pooled = stageSet(b, R2Pooled, pooledAt, 0, output);
-            b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors);
-            bindRange(d.handle, b.exposure0, 0, paramAll, expAt);
-            bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout);
-            bindBuffer(d.handle, b.exposure0, 2, sums);
-            for (uint32_t par = 0; par < 2; ++par) {
-                VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors);
-                bindRange(d.handle, set, 0, paramAll, expAt);
-                bindBuffer(d.handle, set, 1, sums);
-                bindBuffer(d.handle, set, 2, values);
-                bindImage(d.handle, set, 3, e[par ^ 1].view, nearest);
-                bindImage(d.handle, set, 4, e[par].view);
+            if (model.exposureOne.handle)
+                for (uint32_t par = 0; par < 2; ++par) {
+                    VkDescriptorSet set = b.exposureOne[par] = descriptorSet(d.handle, b.pool, model.exposureOne.descriptors);
+                    bindRange(d.handle, set, 0, paramAll, expAt);
+                    bindImage(d.handle, set, 1, images.color, nearest, images.layout);
+                    bindBuffer(d.handle, set, 2, sums);
+                    bindBuffer(d.handle, set, 3, values);
+                    bindImage(d.handle, set, 4, e[par ^ 1].view, nearest);
+                    bindImage(d.handle, set, 5, e[par].view);
+                }
+            else {
+                b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors);
+                bindRange(d.handle, b.exposure0, 0, paramAll, expAt);
+                bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout);
+                bindBuffer(d.handle, b.exposure0, 2, sums);
+                for (uint32_t par = 0; par < 2; ++par) {
+                    VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors);
+                    bindRange(d.handle, set, 0, paramAll, expAt);
+                    bindBuffer(d.handle, set, 1, sums);
+                    bindBuffer(d.handle, set, 2, values);
+                    bindImage(d.handle, set, 3, e[par ^ 1].view, nearest);
+                    bindImage(d.handle, set, 4, e[par].view);
+                }
             }
             bindings.push_back(std::move(b));
             found = bindings.end() - 1;
@@ -675,9 +690,15 @@ struct R2Engine {
             vkCmdUpdateBuffer(cb, paramAll.handle, at, std::min<VkDeviceSize>(65536, paramBytes - at), all.data() + at);
         memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
                       VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
-        dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX);
-        step();
-        dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1);
+        // the exposure: one pass with 64-bit atomics (the last block finishes, as NGX's kernel), or the block sums and
+        // one workgroup that adds them
+        if (model.exposureOne.handle)
+            dispatch(cb, model.exposureOne, b.exposureOne[cur], exposure.blocksX, exposure.blocks / exposure.blocksX);
+        else {
+            dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX);
+            step();
+            dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1);
+        }
         step();
         stamp(1);
         for (unsigned stage = 0; stage < R2Network; ++stage) {