diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 90e07a5..455ea45 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -12,7 +12,7 @@ D4R_CUDA_REPLAY_DUMP_FILTER=rr2 D4R_CUDA_LAUNCH_STATS_FILTER=rr2 and preset hint table of its PTX module; enc0-aux.bin is encoder 0's second table (argument 104) .args the argument block of the capture's frame, pointers and handles cleared: the runtime keeps its constants and fills what depends on the frame or the geometry (engine/runtime_rr2.inc) - exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp + exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp (exposure_f2.spv: both in one, 64-bit atomics) rr.txt the manifest RModel reads ("D4RR0002") With --native, the stages engine/rr/rr2_m.comp implements (enc1 ... enc5, dec4 ... dec1, dec0's first kernel) are that shader with weights read from the DLL itself (engine/rr/rr2_pack.py) instead of the translation: bit-identical to @@ -249,7 +249,7 @@ def main(): if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3') print(f'{stage}: {kernel}, {len(args)} argument bytes' + (' (native block)' if hybrid else '')) - for p in (0, 1): + for p in (0, 1, 2): subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}', str(HERE / 'rr' / 'exposure_f.comp'), '-o', str(out / f'exposure_f{p}.spv')], check=True, stdout=subprocess.DEVNULL) diff --git a/engine/replay_rr.cpp b/engine/replay_rr.cpp index 4e3d605..c591d79 100644 --- a/engine/replay_rr.cpp +++ b/engine/replay_rr.cpp @@ -158,6 +158,8 @@ int main(int argc, char** argv) try { if (std::getenv("D4R_RR_TEST_NO_ROBUST")) { f2.features.robustBufferAccess = VK_FALSE; v13.robustImageAccess = VK_FALSE; } // D4R_RR_TEST_NO_COOPMAT=1: a device like vkd3d-proton's, without the cooperative-matrix and float8 features if (std::getenv("D4R_RR_TEST_NO_COOPMAT")) { di.enabledExtensionCount = 0; v13.pNext = nullptr; } + // D4R_RR_TEST_NO_INT64_ATOMICS=1: no 64-bit buffer atomics (preset F's exposure then takes its two passes) + if (std::getenv("D4R_RR_TEST_NO_INT64_ATOMICS")) v12.shaderBufferInt64Atomics = v12.shaderSharedInt64Atomics = VK_FALSE; CK(vkCreateDevice(c.physical, &di, nullptr, &c.device)); vkGetDeviceQueue(c.device, family, 0, &c.queue); VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pi.queueFamilyIndex = family; @@ -169,7 +171,7 @@ int main(int argc, char** argv) try { std::string key; uint32_t rw, rh, ow, oh; frames >> key >> rw >> rh >> ow >> oh; - d4r::RModel model(c.physical, c.device, modelDir); + d4r::RModel model(c.physical, c.device, modelDir, VK_NULL_HANDLE, v12.shaderBufferInt64Atomics == VK_TRUE); d4r::REngine engine(model, rw, rh, ow, oh); const char* names[] = {"color", "motion", "depth", "normals", "roughness", "diffuse", "specular"}; const VkFormat formats[] = {VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R32_SFLOAT, diff --git a/engine/rr/exposure_f.comp b/engine/rr/exposure_f.comp index fb4e308..fa456c6 100644 --- a/engine/rr/exposure_f.comp +++ b/engine/rr/exposure_f.comp @@ -1,33 +1,39 @@ #version 460 -// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel, in two passes. +// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel. // // NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block // sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish -// turns the total into the exposure. The integer total does not depend on the order of the blocks, so PASS 0 writes -// every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last block does: +// turns the total into the exposure: // exposure = 0.23547088 / max(mean luma, 0.005) -> the 1x1 exposure image of this frame // values = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 } -> the 16-byte buffer encoder 0 and the output passes read // with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero). +// PASS 2 is that kernel (64-bit atomics on a storage buffer: shaderBufferInt64Atomics). Without them the runtime runs +// two passes: PASS 0 writes every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last +// block does. The total is an integer: it does not depend on the order of the blocks, so both give the same values. #extension GL_EXT_shader_explicit_arithmetic_types : require #extension GL_EXT_control_flow_attributes : require #ifndef PASS #define PASS 0 #endif +#if PASS == 2 +#extension GL_EXT_shader_atomic_int64 : require +#extension GL_KHR_memory_scope_semantics : require +#endif layout(set = 0, binding = 0, std430) readonly buffer Params { ivec2 last; // render size - 1 uint blocksX, blocks; float invCount; // 10000 / (2^32 * render pixels) uint reset; // the kernel's argument 32: no log ratio when set } p; -#if PASS == 0 +#if PASS != 1 // One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128, // then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the // same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1). layout(local_size_x = 16) in; layout(set = 0, binding = 1) uniform sampler2D color; -layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums; shared float col[16]; -void main() +// the block's fixed-point sum, in thread 0 +uint64_t blockSum() { const uint tx = gl_LocalInvocationID.x; const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16; @@ -49,33 +55,37 @@ void main() precise float c = w1 + w0; col[tx] = c; barrier(); - if (tx == 0u) { - precise float d[8]; - [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k]; - precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3]; - precise float f0 = e2 + e0, f1 = e3 + e1; - precise float g = f1 + f0; - sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = uint64_t(roundEven(g * 429496.71875)); - } + if (tx != 0u) return 0ul; + precise float d[8]; + [[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k]; + precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3]; + precise float f0 = e2 + e0, f1 = e3 + e1; + precise float g = f1 + f0; + return uint64_t(roundEven(g * 429496.71875)); +} +#endif +#if PASS == 0 +layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums; +void main() +{ + const uint64_t s = blockSum(); + if (gl_LocalInvocationID.x == 0u) sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = s; } #else +#if PASS == 1 layout(local_size_x = 64) in; shared uint64_t part[64]; layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums; -layout(set = 0, binding = 2, std430) writeonly buffer Values { vec4 v; } values; -layout(set = 0, binding = 3) uniform sampler2D previous; -layout(set = 0, binding = 4, r16f) uniform writeonly image2D current; -void main() +#else +// s[0]: the total, s[1]: the blocks done; the last block leaves both 0 for the next frame +layout(set = 0, binding = 2, std430) buffer Sums { uint64_t s[]; } sums; +#endif +layout(set = 0, binding = PASS + 1, std430) writeonly buffer Values { vec4 v; } values; +layout(set = 0, binding = PASS + 2) uniform sampler2D previous; +layout(set = 0, binding = PASS + 3, r16f) uniform writeonly image2D current; +// the last block's work, from the frame's total +void finish(uint64_t total) { - // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes - const uint tid = gl_LocalInvocationID.x; - uint64_t mine = 0ul; - for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b]; - part[tid] = mine; - barrier(); - if (tid != 0u) return; - uint64_t total = 0ul; - for (uint b = 0u; b < 64u; ++b) total += part[b]; precise float mean = p.invCount * float(total); precise float e = 0.23547088 / max(mean, 0.005); imageStore(current, ivec2(0), vec4(e)); @@ -90,4 +100,33 @@ void main() if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667; values.v = vec4(cur, prev, 1.0 / cur, ratio); } +#if PASS == 1 +void main() +{ + // the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes + const uint tid = gl_LocalInvocationID.x; + uint64_t mine = 0ul; + for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b]; + part[tid] = mine; + barrier(); + if (tid != 0u) return; + uint64_t total = 0ul; + for (uint b = 0u; b < 64u; ++b) total += part[b]; + finish(total); +} +#else +void main() +{ + const uint64_t s = blockSum(); + if (gl_LocalInvocationID.x != 0u) return; + atomicAdd(sums.s[0], s, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + // the release orders this block's add before its count, and the last block's acquire every block's add before + // its read of the total + if (atomicAdd(sums.s[1], 1ul, gl_ScopeDevice, gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease) != uint64_t(p.blocks - 1u)) + return; + const uint64_t total = atomicExchange(sums.s[0], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + atomicExchange(sums.s[1], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed); + finish(total); +} +#endif #endif diff --git a/engine/runtime.h b/engine/runtime.h index 4ab8ed5..7c3d38e 100644 --- a/engine/runtime.h +++ b/engine/runtime.h @@ -273,7 +273,10 @@ struct RFrameSettings { }; class RModel { public: - RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE); + // int64Atomics: the device has shaderBufferInt64Atomics enabled; preset F then measures the exposure in one pass + // (engine/rr/exposure_f.comp PASS 2) instead of two + RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE, + bool int64Atomics = false); ~RModel(); RModel(const RModel&) = delete; RModel& operator=(const RModel&) = delete; diff --git a/engine/runtime_rr.inc b/engine/runtime_rr.inc index 825035e..04908c6 100644 --- a/engine/runtime_rr.inc +++ b/engine/runtime_rr.inc @@ -90,7 +90,7 @@ struct RModel::Impl { bool uploaded = false; Impl(VkPhysicalDevice physical, VkDevice dev) : device(physical, dev) {} }; -RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache) +RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache, bool int64Atomics) : impl(std::make_unique(physical, device)) { auto& m = *impl; // rr.txt: "D4RR0001", "preset N", one "stage NAME SHADER.spv SUBGROUP WEIGHTS TEXTURES SURFACES" line per stage in @@ -103,7 +103,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di if (line.rfind("D4RR0002", 0) == 0) { m.preset = 6; m.v2 = std::make_unique(); - m.v2->load(m.device, directory, lines, cache); + m.v2->load(m.device, directory, lines, cache, int64Atomics); return; } if (line.rfind("D4RR0001", 0) != 0) throw std::runtime_error("not a Ray Reconstruction model (rr.txt D4RR0001)"); diff --git a/engine/runtime_rr2.inc b/engine/runtime_rr2.inc index 85e8fcc..dfdf963 100644 --- a/engine/runtime_rr2.inc +++ b/engine/runtime_rr2.inc @@ -44,9 +44,10 @@ struct R2Model { Buffer aux; // encoder 0's second table (argument 104) std::vector auxPending; std::array exposure; + Pipeline exposureOne; // both exposure passes in one (exposure_f2.spv, 64-bit atomics); else null bool particles = false; // the model has R2DownP bool gather = false; // the model has R2Gather - void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache) { + void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache, bool int64Atomics) { const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; auto split = [](const std::string& list) { std::vector items; @@ -108,6 +109,8 @@ struct R2Model { blob(aux, auxPending, read(directory + "/enc0-aux.bin")); exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache); exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache); + if (int64Atomics && exists(directory + "/exposure_f2.spv")) + exposureOne.create(d.handle, directory + "/exposure_f2.spv", {b, t, b, b, t, s}, 64, cache); } void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents) for (auto& st : stages) recordContents(cb, st.weights, st.pending); @@ -172,7 +175,7 @@ struct R2Engine { std::vector down; std::vector> downP; VkDescriptorSet pooled = VK_NULL_HANDLE, exposure0 = VK_NULL_HANDLE; - std::array exposure1{}; + std::array exposure1{}, exposureOne{}; }; std::vector bindings; const Bindings* bound = nullptr; @@ -337,7 +340,7 @@ struct R2Engine { exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16); exposure.last[0] = static_cast(rw - 1); exposure.last[1] = static_cast(rh - 1); exposure.invCount = static_cast(10000.0 / (4294967296.0 * rw * rh)); - sums.create(d, exposure.blocks * 8, storage); + sums.create(d, std::max(exposure.blocks, 2u) * 8, storage); // the one-pass exposure: the total and the count const size_t tiles = src.size(); auto place = [&](size_t bytes) { const VkDeviceSize at = paramBytes; paramBytes += (bytes + 255) & ~VkDeviceSize(255); return at; }; for (unsigned i = 0; i < R2Network; ++i) { blocks[i] = model.stages[i].args; paramAt[i] = place(blocks[i].size()); } @@ -593,17 +596,29 @@ struct R2Engine { b.downP.push_back({stageSet(b, R2DownP, downPAt[i], 0, output), stageSet(b, R2DownP, downPAt[i], 1, output)}); } b.pooled = stageSet(b, R2Pooled, pooledAt, 0, output); - b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors); - bindRange(d.handle, b.exposure0, 0, paramAll, expAt); - bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout); - bindBuffer(d.handle, b.exposure0, 2, sums); - for (uint32_t par = 0; par < 2; ++par) { - VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors); - bindRange(d.handle, set, 0, paramAll, expAt); - bindBuffer(d.handle, set, 1, sums); - bindBuffer(d.handle, set, 2, values); - bindImage(d.handle, set, 3, e[par ^ 1].view, nearest); - bindImage(d.handle, set, 4, e[par].view); + if (model.exposureOne.handle) + for (uint32_t par = 0; par < 2; ++par) { + VkDescriptorSet set = b.exposureOne[par] = descriptorSet(d.handle, b.pool, model.exposureOne.descriptors); + bindRange(d.handle, set, 0, paramAll, expAt); + bindImage(d.handle, set, 1, images.color, nearest, images.layout); + bindBuffer(d.handle, set, 2, sums); + bindBuffer(d.handle, set, 3, values); + bindImage(d.handle, set, 4, e[par ^ 1].view, nearest); + bindImage(d.handle, set, 5, e[par].view); + } + else { + b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors); + bindRange(d.handle, b.exposure0, 0, paramAll, expAt); + bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout); + bindBuffer(d.handle, b.exposure0, 2, sums); + for (uint32_t par = 0; par < 2; ++par) { + VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors); + bindRange(d.handle, set, 0, paramAll, expAt); + bindBuffer(d.handle, set, 1, sums); + bindBuffer(d.handle, set, 2, values); + bindImage(d.handle, set, 3, e[par ^ 1].view, nearest); + bindImage(d.handle, set, 4, e[par].view); + } } bindings.push_back(std::move(b)); found = bindings.end() - 1; @@ -675,9 +690,15 @@ struct R2Engine { vkCmdUpdateBuffer(cb, paramAll.handle, at, std::min(65536, paramBytes - at), all.data() + at); memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX); - step(); - dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1); + // the exposure: one pass with 64-bit atomics (the last block finishes, as NGX's kernel), or the block sums and + // one workgroup that adds them + if (model.exposureOne.handle) + dispatch(cb, model.exposureOne, b.exposureOne[cur], exposure.blocksX, exposure.blocks / exposure.blocksX); + else { + dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX); + step(); + dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1); + } step(); stamp(1); for (unsigned stage = 0; stage < R2Network; ++stage) { diff --git a/packaging/build_engine_models.sh b/packaging/build_engine_models.sh index f5baa7b..a97c4e0 100755 --- a/packaging/build_engine_models.sh +++ b/packaging/build_engine_models.sh @@ -98,7 +98,7 @@ if (( DO_RR )); then done run ptx-f.log python3 "$KIT/kernels/tools/extract_dlss_ptx.py" "$DLL" "$WORK/ptx-f" run rr-f.log python3 "$KIT/engine/compile_rr2.py" --dll "$DLL" "$WORK/rr-f" --ptx-dir "$WORK/ptx-f" - place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv + place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv exposure_f2.spv fi if (( DO_SR )); then diff --git a/tools/d4r_nvngx_shim.cpp b/tools/d4r_nvngx_shim.cpp index d35a7ba..7fd5249 100644 --- a/tools/d4r_nvngx_shim.cpp +++ b/tools/d4r_nvngx_shim.cpp @@ -3385,6 +3385,9 @@ struct VulkanInterop // VK_EXT_shader_float8 (vkd3d-proton enables it when the GPU has it): M's and L's engine layers are FP8 // cooperative-matrix shaders, which RADV's compiler crashes on without it (RDNA3) bool float8 = false; + // shaderBufferInt64Atomics is enabled on vkd3d-proton's device (it enables it when the GPU has it): Ray + // Reconstruction preset F then measures the exposure in one pass, as NGX does, instead of two + bool int64Atomics = false; // The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects. bool native = false; PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr; @@ -9322,6 +9325,17 @@ static void init_engine_vulkan() } logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: " "M and L run on the CUDA backend"); + // the features vkd3d-proton created its device with + const VkPhysicalDeviceFeatures2* features = nullptr; + if (SUCCEEDED(g_vk.interop->GetDeviceFeatures(&features)) && features != nullptr) + for (auto* next = static_cast(features->pNext); next != nullptr; next = next->pNext) + { + if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES) + g_vk.int64Atomics = reinterpret_cast(next)->shaderBufferInt64Atomics == VK_TRUE; + if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES) + g_vk.int64Atomics = reinterpret_cast(next)->shaderBufferInt64Atomics == VK_TRUE; + } + logf("engine backend: 64-bit buffer atomics %s", g_vk.int64Atomics ? "enabled" : "not enabled: preset F's exposure in two passes"); } static void init_engine_backend() @@ -10316,7 +10330,7 @@ static void init_engine_backend_rr(int slot) const auto start = std::chrono::steady_clock::now(); try { - auto model = std::make_unique(g_vk.physical, g_vk.device, path); + auto model = std::make_unique(g_vk.physical, g_vk.device, path, VK_NULL_HANDLE, g_vk.int64Atomics); if (model->preset() != g_rrPreset[slot]) throw std::runtime_error("the directory holds a model of another preset"); g_engineR.models[slot] = std::move(model);