From af450a92b959d554cff0f495d529fb6a935d25bc Mon Sep 17 00:00:00 2001 From: woutermans Date: Wed, 14 Oct 2026 15:14:28 +0200 Subject: [PATCH] engine: cut the models' and the engines' memory - the weights go into the command buffer with vkCmdUpdateBuffer (recordContents); the host-visible staging copies are gone, and the host copies are dropped once recorded - an engine's own output image is created only for frames that do not write the caller's texture (K, M, RR D/E, RR F), cleared on first use - preset F: the tile slots share the arena with the network's regions, and the gather records' rows fill the gaps between encoder 0's regions (rr.txt "s3", six words on enc0's block; D4R_ENGINE_RR_ARENA keeps the old separate layout for arena dumps) - the shim reserves the render size the feature was created with for the M/L Vulkan network instead of the output size; a frame beyond it goes to a second, output-sized engine (1.1 GB less for M, 1.4 for L at 1280x720 -> 4K); replay_m gains D4R_M_CAPACITY Every reference capture still byte-identical; through the shim the harness process peaks 159 MiB lower. --- engine/README.md | 8 ++- engine/compile_rr2.py | 10 +-- engine/replay_m.cpp | 7 +- engine/rr/README.md | 29 +++++++- engine/rr/gen_rr2_enc0.py | 12 +++- engine/runtime.cpp | 40 ++++++++--- engine/runtime_m.inc | 23 ++++--- engine/runtime_rr.inc | 40 ++++++----- engine/runtime_rr2.inc | 139 ++++++++++++++++++++++++++++---------- tools/d4r_nvngx_shim.cpp | 22 ++++-- 10 files changed, 237 insertions(+), 93 deletions(-) diff --git a/engine/README.md b/engine/README.md index 1385d0e..947ffce 100644 --- a/engine/README.md +++ b/engine/README.md @@ -755,8 +755,12 @@ conversion inputs up to the output size. Use a K package with all sixteen recons upscaling ratios; older single-table packages cannot match NGX at every ratio. L and M update the Vulkan network's active token grid, exposure grid and reconstruction tier per frame. -Their input and history planes reserve the constructor's render capacity; the shim reserves output-sized -capacity for the Vulkan path. Bilinear reads clamp to the active rectangle. A render-size change resets +Their input and history planes reserve the constructor's render capacity. The shim reserves the render size the +feature was created with (or the first frame's, when larger), as NGX does; a later frame beyond that goes to a second, +output-sized Vulkan engine, and the first is kept until the feature is released. Reserving the output size for every +feature cost 0.85 GB (M) and 1.08 GB (L) at 1280x720 -> 4K; the images are byte-identical either way (harness, +2026-10-14: created 1280x720 with 854x480 frames, created 854x480 with 1280x720 frames, and the second engine +forced). Bilinear reads clamp to the active rectangle. A render-size change resets L/M temporal history. Input dimensions must be at least 64 on each axis and cannot exceed the output. Standalone callers set `GameFrameM::renderWidth/renderHeight` or `MFrameSettings::renderWidth/renderHeight`; zero keeps the constructor size. diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 4fcf3e9..9f74e0a 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -147,8 +147,8 @@ def main(): subprocess.run([sys.executable, str(HERE / 'rr' / 'rr2_pack.py'), stage, a.native, str(out / weights), str(work / 'rr2_stage.h')], check=True, stdout=subprocess.DEVNULL) defs = ['-DRR_EXACT_CONVERT'] + (['-DRR_MIX'] if os.environ.get('D4R_RR_MIX') == '1' else []) + FUSE_DEFS + RELAX_DEFS - # encoder 0 in two passes (gen_rr2_enc0.py --split): its gather as the stage enc0g, whose record's address - # is a word appended to the argument block + # encoder 0 in two passes (gen_rr2_enc0.py --split): its gather as the stage enc0g, whose records' place + # (three ranges) is six words appended to the argument block split = stage == 'enc0' and SPLIT and GROUP == 1 how = ['--split', 'block', '--xf-word', str((len(launch.args) + 7) // 8 * 2)] if split else ['--group', str(GROUP)] if split: @@ -186,7 +186,7 @@ def main(): (out / 'enc0-aux.bin').write_bytes(launch.pre(launch.pointer(104).alloc)) args = stage_params(launch, 24 if network else 9999)[:len(launch.args)] if hybrid and split: - args = args + bytes(-len(args) % 8) + bytes(8) # the gather record's address (the runtime's) + args = args + bytes(-len(args) % 8) + bytes(24) # the gather records' three ranges (the runtime's) (out / f'{stage}.args').write_bytes(args) # the stage's arena regions, in units of 1/3152 of NGX's arena (W0 * H0 / 32 bytes: the regions scale with it) arena = launch.arena_alloc @@ -203,8 +203,8 @@ def main(): wave = 32 if network else 64 lines.append(f'stage {stage} {stage}.spv {wave} {weights} {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"}' + (f' x{GROUP}' if hybrid and GROUP > 1 else '')) # windows per workgroup (engine/rr/gen_rr2_enc0.py) - if hybrid and split: # the same block, bindings and regions; "rN": N words per window record - lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record}') + if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges + lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3') print(f'{stage}: {kernel}, {len(launch.args)} argument bytes' + (' (native block)' if hybrid else '')) for p in (0, 1): subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}', diff --git a/engine/replay_m.cpp b/engine/replay_m.cpp index 3b71975..817e22b 100644 --- a/engine/replay_m.cpp +++ b/engine/replay_m.cpp @@ -127,7 +127,11 @@ int main(int argc, char** argv) try { network = std::make_unique(model, rw, rh, tokens.fd, result.fd, tokens.allocation, result.allocation); } #endif - d4r::MEngine engine(m, rw, rh, ow, oh, hip ? &external : nullptr); + // D4R_M_CAPACITY=WxH: the engine reserves a larger render size than the frames use (dynamic resolution, as a + // shim reserving more than the feature's size would) + uint32_t cw = rw, ch = rh; + if (const char* capacity = std::getenv("D4R_M_CAPACITY")) std::sscanf(capacity, "%ux%u", &cw, &ch); + d4r::MEngine engine(m, cw, ch, ow, oh, hip ? &external : nullptr); Image color = image(c, rw, rh, VK_FORMAT_R16G16B16A16_SFLOAT), motion = image(c, rw, rh, VK_FORMAT_R16G16_SFLOAT), depth = image(c, rw, rh, VK_FORMAT_R32_SFLOAT); const VkDeviceSize sizes[3] = {VkDeviceSize(rw) * rh * 8, VkDeviceSize(rw) * rh * 4, VkDeviceSize(rw) * rh * 4}; Staging in[3] = {staging(c, sizes[0]), staging(c, sizes[1]), staging(c, sizes[2])}, out = staging(c, VkDeviceSize(ow) * oh * 8); @@ -162,6 +166,7 @@ int main(int argc, char** argv) try { if (pass == 0 || frames > 1) for (int k = 0; k < 3; ++k) vkCmdCopyBufferToImage(cb, in[k].buffer, images[k]->image, VK_IMAGE_LAYOUT_GENERAL, 1, ®ion); d4r::MFrameSettings s; s.jitter[0] = std::atof(a[3]); s.jitter[1] = std::atof(a[4]); s.reset = (pass == 0 && f == 0) || resetAll; + s.renderWidth = rw; s.renderHeight = rh; const double t0 = now(); engine.setFrame({color.view, motion.view, depth.view}, s); vkCmdResetQueryPool(cb, qp, 0, 7); diff --git a/engine/rr/README.md b/engine/rr/README.md index 555232e..eb119e1 100644 --- a/engine/rr/README.md +++ b/engine/rr/README.md @@ -312,7 +312,7 @@ how the passes were arranged: - **Encoder 0 in two passes** (-0.2 ms). One shader ran NVIDIA's 32-lane gather on one of the block's four waves while the other three waited: 0.40 ms for the gather, 0.24 ms as a pass of its own in 32-thread groups. `gen_rr2_enc0.py --split gather | block` writes both; the gather pass (`stage enc0g`, the same bindings and - parameter block as enc0) stores each window's record in the arena after the tile slots: the first product's A + parameter block as enc0) stores each window's record in the arena buffer (see Memory below): the first product's A fragments (1024 words) and, per lane, the values the tail reads from the gather (six binary16 on the reference build; the generator finds them and recomputes the tail's index arithmetic). The record's address is a word compile_rr2.py appends to enc0's argument block. Cost: 76 bytes per render pixel of arena. @@ -354,6 +354,33 @@ scripts/run_rr_engine_compare.sh`, frames aligned by one): moving scene that showed as 40 - 50 dB on later frames in a synthetic scene whose affected feature is rounding noise around zero in both paths; no other configuration was affected. +### Memory (2026-10-14) + +At 1280x720 -> 4K the engine needs about 600 MB for preset F; the two 1.5x output histories (2 x 149 MB, NGX has +the same) are half of it. NVIDIA's DLL, its allocations traced through the CUDA path and replayed on an RTX PRO 4500 +Blackwell, takes 526 MiB of feature state, 106 MiB of input / output arrays and 2 MiB of code, plus 262 MiB of CUDA +context. Three cuts, every capture still byte-identical (`verify.sh`, `verify_x.sh`) and the shim's frames +byte-identical to the previous build, same speed: + +- **The tile slots share the arena with the network's regions.** Every arena region is written and read within one + frame; in the output stage only the network's output (dec0_1's argument 64) and something after it are read, so + the slots that fit before the network's output go there (all 8 at 720p -> 4K) and the rest after the arena. A + first try that put the slots right after the network's output broke 360p -> 1080p's output stage. +- **The gather records' rows of windows fill the gaps between encoder 0's regions**: the two largest (units + 1216 - 2240 and 2304 - 2560) and then from the end of its last region (unit 2816) on, instead of one piece after + the tile slots. The records (76 bytes a render pixel, about 2450 units) are larger than all the gaps, so 46 of 93 + rows at 720p still reach past the arena (36 MB). Models from compile_rr2.py mark this with "s3" on the enc0g line + and append six words to enc0's block (three ranges' addresses, the first two's rows; `gen_rr2_enc0.py`); older + models keep the one-piece layout. Same speed as one piece (four alternating runs at 720p -> 4K). +- **The engine's own output image is created only for frames that do not write the game's texture**, and the + weights go into the command buffer with `vkCmdUpdateBuffer` instead of through a host-visible staging copy that + stayed allocated. + +Through the shim (direct output) the harness process peaks 159 MiB lower (1100 -> 941 MiB). Not exact, so not +done: the pooled histories as RGB10A2 (`D4R_RR2_POOLED=rgb10_a2`, -70 MB, see above); they hold q / 1023, which +neither binary16 nor UNORM16 represents exactly. `D4R_ENGINE_RR_ARENA` +keeps the old separate layout, so that arena dumps show the network's regions as NGX leaves them. + ## Against a real NVIDIA card: `D4R_RR_FUSE` (2026-10-13) Every reference above is the CUDA path through ZLUDA. NVIDIA's own kernels (310.9.1's `rr2_*` PTX, replayed through diff --git a/engine/rr/gen_rr2_enc0.py b/engine/rr/gen_rr2_enc0.py index 08b7cce..6025983 100644 --- a/engine/rr/gen_rr2_enc0.py +++ b/engine/rr/gen_rr2_enc0.py @@ -117,8 +117,15 @@ def split_layout(xf_word, pre, tail, f16vars): if not (v in f16vars or re.fullmatch(r'r\d+', v)): raise SystemExit(f'the tail reads {v} from the gather: not a 16- or 32-bit register') record = 1024 + 32 * len(saved) + # the records' rows of windows lie in up to three ranges of the arena (the runtime's choice, where the network's + # regions are dead during encoder 0): words XF_WORD, +2, +4 their byte addresses, +1 and +3 the first two's rows where = (f' const uint rr2_gx = (param.w[0] + (param.w[20] & 0xffffu) + 7u) / 8u;\n' - f' const uint rr2_rec = param.w[{xf_word}u] / 4u + {record}u * (gl_WorkGroupID.y * (param.w[0] / 8u + 1u) + gl_WorkGroupID.x);\n') + f' uint rr2_y = gl_WorkGroupID.y, rr2_base = param.w[{xf_word}u];\n' + f' if (rr2_y >= param.w[{xf_word + 1}u]) {{\n' + f' rr2_y -= param.w[{xf_word + 1}u]; rr2_base = param.w[{xf_word + 2}u];\n' + f' if (rr2_y >= param.w[{xf_word + 3}u]) {{ rr2_y -= param.w[{xf_word + 3}u]; rr2_base = param.w[{xf_word + 4}u]; }}\n' + f' }}\n' + f' const uint rr2_rec = rr2_base / 4u + {record}u * (rr2_y * (param.w[0] / 8u + 1u) + gl_WorkGroupID.x);\n') return recomputed, saved, record, where @@ -128,7 +135,8 @@ def gather_pass(xf_word, head, decls, pre, tail, first, f16vars): One shader holding both runs the gather on one of the block's four waves while the other three wait, which costs more than the record's trip through memory (RX 9070 XT, 1280x720, 2026-10-13: gather 0.40 -> 0.24 ms). The record: 1024 words of A fragments ([token slot][64 FP8 feature slots], as the block's XF), then per lane the - gather's values the tail reads. Its address: parameter word XF_WORD (bytes) + 4 * RECORD * (y * (W / 8 + 1) + x).""" + gather's values the tail reads. Its address: the byte address of row y's range (parameter words XF_WORD to + XF_WORD + 4) + 4 * RECORD * (y' * (W / 8 + 1) + x), y' the row within its range.""" recomputed, saved, record, where = split_layout(xf_word, pre, tail, f16vars) var = lambda r: r.lstrip('%') store = [' if (rr2_valid) { // the first product\'s A fragments: lane 4g + t, register q -> row, 4 k', diff --git a/engine/runtime.cpp b/engine/runtime.cpp index 1b304f8..aa19e7e 100644 --- a/engine/runtime.cpp +++ b/engine/runtime.cpp @@ -79,6 +79,14 @@ struct Buffer { if (host) VKCHECK(vkMapMemory(d.handle, memory, 0, VK_WHOLE_SIZE, 0, &mapped)); } }; +// A buffer's contents written into the command buffer itself (vkCmdUpdateBuffer, at most 65536 bytes a call), so no +// host-visible staging copy stays allocated; the host copy (padded to the buffer's size) is dropped once recorded +void recordContents(VkCommandBuffer cb, const Buffer& buffer, std::vector& bytes) { + bytes.resize(buffer.size); + for (VkDeviceSize at = 0; at < bytes.size(); at += 65536) + vkCmdUpdateBuffer(cb, buffer.handle, at, std::min(65536, bytes.size() - at), bytes.data() + at); + std::vector().swap(bytes); +} struct Image { const Device* device = nullptr; VkImage handle = VK_NULL_HANDLE; @@ -125,6 +133,15 @@ void imageBarrier(VkCommandBuffer cb, VkImage image, VkImageLayout before, VkIma barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; vkCmdPipelineBarrier(cb, source, destination, 0, 0, nullptr, 0, nullptr, 1, &barrier); } +// An image created after recordInitialize (an output only some frames need): GENERAL and zero, as recordInitialize +// leaves its images; the caller's next barrier covers the clear +void clearNewImage(VkCommandBuffer cb, VkImage image) { + VkClearColorValue zero{}; + VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + imageBarrier(cb, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); + vkCmdClearColorImage(cb, image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range); +} void memoryBarrier(VkCommandBuffer cb, VkPipelineStageFlags source, VkPipelineStageFlags destination, VkAccessFlags src, VkAccessFlags dst) { VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; @@ -274,7 +291,8 @@ FrameParams makeKFrameParams(const Shape& shape, const FrameSettings& settings) struct Model::Impl { Device device; - Buffer weights, lut, weightUpload, lutUpload; + Buffer weights, lut; + std::vector weightBytes, lutBytes; // until recordUpload std::array, KLayers> offsets{}; std::array network; Pipeline input, output, outputRgb10; @@ -332,9 +350,7 @@ Model::Model(VkPhysicalDevice physical, VkDevice device, const std::string& dire throw std::runtime_error("model weights exceed the device's storage buffer range"); m.weights.create(m.device, weights.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); m.lut.create(m.device, lut.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); - m.weightUpload.create(m.device, weights.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); - m.lutUpload.create(m.device, lut.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); - std::memcpy(m.weightUpload.mapped, weights.data(), weights.size()); std::memcpy(m.lutUpload.mapped, lut.data(), lut.size()); + m.weightBytes = std::move(weights); m.lutBytes = std::move(lut); const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; for (unsigned i = 0; i < KLayers; ++i) m.network[i].create(device, directory + "/" + Names[i] + ".spv", {b,b,b,b,b,b}, 32, cache); m.input.create(device, directory + "/input_k.spv", {b,t,t,t,t,t,b}, 64, cache); @@ -359,8 +375,7 @@ bool Model::uploadRecorded() const { return impl->uploaded; } void Model::recordUpload(VkCommandBuffer cb) { auto& m = *impl; if (m.uploaded) throw std::runtime_error("model upload already recorded"); - VkBufferCopy copy{0, 0, m.weightUpload.size}; vkCmdCopyBuffer(cb, m.weightUpload.handle, m.weights.handle, 1, ©); - copy.size = m.lutUpload.size; vkCmdCopyBuffer(cb, m.lutUpload.handle, m.lut.handle, 1, ©); + recordContents(cb, m.weights, m.weightBytes); recordContents(cb, m.lut, m.lutBytes); memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); m.uploaded = true; } @@ -378,7 +393,8 @@ struct Engine::Impl { std::array skip; struct History { Image color, luma, feature; }; std::array history; - Image final; + Image final; // the engine's own output: only for frames that do not write the caller's image (created on first use) + bool finalNew = false; VkDescriptorPool pool = VK_NULL_HANDLE; std::array layerSets{}; std::array inputSets{}, outputSets{}; @@ -473,7 +489,6 @@ Engine::Engine(Model& model, const Shape& shape, const ExternalNetwork* external VKCHECK(vkCreateSampler(dev, &si, nullptr, &e.nearest)); si.magFilter = si.minFilter = VK_FILTER_LINEAR; VKCHECK(vkCreateSampler(dev, &si, nullptr, &e.linear)); - e.final.create(d, shape.outputWidth, shape.outputHeight); for (unsigned i = 0; i < 2; ++i) { auto& h = e.history[i]; h.color.create(d, shape.outputWidth, shape.outputHeight); @@ -504,7 +519,7 @@ Engine::Engine(Model& model, const Shape& shape, const ExternalNetwork* external bindBuffer(dev, out, 0, e.outParams); bindBuffer(dev, out, 10, e.main[1]); bindBuffer(dev, out, 11, e.model.lut); bindImage(dev, out, 2, h.color.sample, e.linear); bindImage(dev, out, 3, h.luma.sample, e.linear); bindImage(dev, out, 6, dest.color.storage); bindImage(dev, out, 7, dest.luma.storage); - bindImage(dev, out, 8, dest.feature.storage); bindImage(dev, out, 9, e.final.storage); + bindImage(dev, out, 8, dest.feature.storage); // binding 9, the output: setFrame } } Engine::~Engine() = default; @@ -519,7 +534,6 @@ void Engine::recordInitialize(VkCommandBuffer cb) { vkCmdClearColorImage(cb, image.handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range); }; for (auto& history : e.history) { init(history.color); init(history.luma); init(history.feature); } - init(e.final); if (e.model.hasExposure) { const float one[2] = {1, 1}; vkCmdUpdateBuffer(cb, e.exposureState.handle, 0, sizeof(one), one); @@ -585,6 +599,10 @@ void Engine::setFrame(const FrameImages& images, const InputParams& input, const images.colorLayout == b.colorLayout && images.motionLayout == b.motionLayout && images.depthLayout == b.depthLayout && images.exposure == b.exposure && images.exposureLayout == b.exposureLayout; }; + if (!finalOutput.view && !e.final.handle) { + e.final.create(e.model.device, e.shape.outputWidth, e.shape.outputHeight); + e.finalNew = true; + } auto found = std::find_if(e.bindings.begin(), e.bindings.end(), [&](const auto& b) { return sameImages(b.images) && finalOutput.view == b.finalOutput.view && finalOutput.format == b.finalOutput.format; }); @@ -664,6 +682,7 @@ void Engine::recordFrame(VkCommandBuffer cb, const FrameProfiler& profiler) { if (!e.initialized || !e.model.uploaded || !e.frameSet) throw std::runtime_error("initialize/upload/setFrame before recording inference"); auto mark = [&](uint32_t boundary) { if (profiler.mark) profiler.mark(cb, boundary, profiler.user); }; mark(0); + if (e.finalNew) { clearNewImage(cb, e.final.handle); e.finalNew = false; } // Includes reads-to-writes when reusing scratch and histories, and copies made by the caller after the last frame. memoryBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); @@ -716,6 +735,7 @@ void Engine::recordFront(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first) if (!e.external) throw std::runtime_error("this engine's network is internal: record the whole frame"); if (!e.initialized || !e.model.uploaded || !e.frameSet || e.frontDone) throw std::runtime_error("initialize/upload/setFrame before recording inference"); const auto compute = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + if (e.finalNew) { clearNewImage(cb, e.final.handle); e.finalNew = false; } memoryBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, compute | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); vkCmdUpdateBuffer(cb, e.inParams.handle, 0, sizeof(InputParams), &e.input); diff --git a/engine/runtime_m.inc b/engine/runtime_m.inc index 94aa475..803f18d 100644 --- a/engine/runtime_m.inc +++ b/engine/runtime_m.inc @@ -68,7 +68,8 @@ void bindRange(VkDevice device, VkDescriptorSet set, uint32_t binding, const Buf struct MModel::Impl { Device device; - Buffer w8, w16, up8, up16; + Buffer w8, w16; + std::vector bytes8, bytes16; // until recordUpload uint32_t enc0[3]{}, dec0[2]{}; std::array, MLayers> layers{}; std::array network; // one per layer (five distinct shaders) @@ -115,8 +116,7 @@ MModel::MModel(VkPhysicalDevice physical, VkDevice device, const std::string& di for (auto& l : m.layers) inside(l[0], l[1]); const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; m.w8.create(m.device, b8.size(), usage); m.w16.create(m.device, b16.size(), usage); - m.up8.create(m.device, b8.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); m.up16.create(m.device, b16.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); - std::memcpy(m.up8.mapped, b8.data(), b8.size()); std::memcpy(m.up16.mapped, b16.data(), b16.size()); + m.bytes8 = b8; m.bytes16 = b16; const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; for (unsigned i = 0; i < MLayers; ++i) m.network[i].create(device, directory + "/" + MShader[i] + ".spv", {b,b,b,b,b,b,b}, 32, cache); // layer_m.comp's L stages: bindings 0-6 of a layer, then the stage's own (layer_m.comp KIND 3 and 4) @@ -149,8 +149,7 @@ bool MModel::isPresetL() const { return impl->presetL; } void MModel::recordUpload(VkCommandBuffer cb) { auto& m = *impl; if (m.uploaded) throw std::runtime_error("model upload already recorded"); - VkBufferCopy copy{0, 0, m.up8.size}; vkCmdCopyBuffer(cb, m.up8.handle, m.w8.handle, 1, ©); - copy.size = m.up16.size; vkCmdCopyBuffer(cb, m.up16.handle, m.w16.handle, 1, ©); + recordContents(cb, m.w8, m.bytes8); recordContents(cb, m.w16, m.bytes16); memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); m.uploaded = true; } @@ -163,7 +162,9 @@ struct MEngine::Impl { Buffer tokens, skip1, merged1, skip2, merged2, tubeA, tubeB, dec2Out, dec1Out; Buffer skipL, enc0LP, dec0LP; // preset L: the input stage's pixel output and the two stages' layer blocks std::array mvA, feat, hist, half; - Tex mvB, mvC, gates, mix, cov, output; + Tex mvB, mvC, gates, mix, cov; + Tex output; // the engine's own output: only for frames that do not write the caller's image (created on first use) + bool outputNew = false; VkDescriptorPool pool = VK_NULL_HANDLE; std::array layerSets{}; std::array enc0Sets{}, dec0Sets{}, postSets{}, expSets{}; @@ -233,7 +234,6 @@ MEngine::MEngine(MModel& model, uint32_t renderWidth, uint32_t renderHeight, uin for (auto& t : e.hist) t.create(d, e.hw, e.hh, VK_FORMAT_R16G16B16A16_SFLOAT); // The native formatted RGB10 history has RGBA32F backing. UNORM filtering differs by up to three ulps. for (auto& t : e.half) t.create(d, e.hw / 2, e.hh / 2, VK_FORMAT_R32G32B32A32_SFLOAT); - e.output.create(d, outputWidth, outputHeight, VK_FORMAT_R16G16B16A16_SFLOAT); VkDescriptorPoolSize sizes[] = {{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 128}, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 48}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 32}}; VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; pi.maxSets = 24; pi.poolSizeCount = 3; pi.pPoolSizes = sizes; @@ -305,14 +305,14 @@ MEngine::MEngine(MModel& model, uint32_t renderWidth, uint32_t renderHeight, uin bindBuffer(dev, e.expSets[1], 1, e.expSums); bindBuffer(dev, e.expSets[1], 2, e.expState); bindBuffer(dev, e.expSets[1], 3, e.enc0P); bindBuffer(dev, e.expSets[1], 4, e.postP); bindBuffer(dev, e.expSets[1], 5, e.downP); e.downSet = descriptorSet(dev, e.pool, m.down.descriptors); - bindBuffer(dev, e.downSet, 0, e.downP); bindImage(dev, e.downSet, 2, e.output.view); bindBuffer(dev, e.downSet, 3, e.packed); + bindBuffer(dev, e.downSet, 0, e.downP); bindBuffer(dev, e.downSet, 3, e.packed); // binding 2, the output: setFrame } MEngine::~MEngine() = default; void MEngine::recordInitialize(VkCommandBuffer cb) { auto& e = *impl; if (e.initialized) throw std::runtime_error("engine initialization already recorded"); VkClearColorValue zero{}; VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - for (Tex* t : {&e.mvA[0], &e.mvA[1], &e.mvB, &e.mvC, &e.feat[0], &e.feat[1], &e.gates, &e.mix, &e.cov, &e.hist[0], &e.hist[1], &e.half[0], &e.half[1], &e.output}) { + for (Tex* t : {&e.mvA[0], &e.mvA[1], &e.mvB, &e.mvC, &e.feat[0], &e.feat[1], &e.gates, &e.mix, &e.cov, &e.hist[0], &e.hist[1], &e.half[0], &e.half[1]}) { imageBarrier(cb, t->handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); vkCmdClearColorImage(cb, t->handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range); } @@ -358,6 +358,10 @@ void MEngine::setFrame(const FrameImages& images, const MFrameSettings& s, Frame e.prevJitter[0] = s.jitter[0]; e.prevJitter[1] = s.jitter[1]; auto dev = e.model.device.handle; auto& m = e.model; + if (!finalOutput.view && !e.output.handle) { + e.output.create(e.model.device, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT); + e.outputNew = true; + } auto found = std::find_if(e.bindings.begin(), e.bindings.end(), [&](const auto& b) { return b.images.color == images.color && b.images.motion == images.motion && b.images.depth == images.depth && b.images.colorLayout == images.colorLayout && b.images.motionLayout == images.motionLayout && @@ -446,6 +450,7 @@ void closeStages(VkCommandBuffer cb) { memoryBarrier(cb, compute, VK_PIPELINE_ST void MEngine::recordFront(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first) { auto& e = *impl; auto& m = e.model; if (!e.initialized || !m.uploaded || !e.frameSet || e.frontDone) throw std::runtime_error("initialize/upload/setFrame before recording inference"); + if (e.outputNew) { clearNewImage(cb, e.output.handle); e.outputNew = false; } openStages(cb); vkCmdUpdateBuffer(cb, e.enc0P.handle, 0, sizeof(e.enc0), &e.enc0); vkCmdUpdateBuffer(cb, e.dec0P.handle, 0, sizeof(e.dec0), &e.dec0); vkCmdUpdateBuffer(cb, e.postP.handle, 0, sizeof(e.post), &e.post); vkCmdUpdateBuffer(cb, e.downP.handle, 0, sizeof(e.down), &e.down); diff --git a/engine/runtime_rr.inc b/engine/runtime_rr.inc index 117bc34..ba3d767 100644 --- a/engine/runtime_rr.inc +++ b/engine/runtime_rr.inc @@ -81,7 +81,8 @@ struct RModel::Impl { // surface argument offsets with their GLSL image format, both ascending std::array, RStages> textures; std::array>, RStages> surfaces; - std::array weights, uploads; + std::array weights; + std::array, RStages> weightBytes; // until recordUpload std::array stages; std::array exposure; std::vector enc0Template, hkpnTemplate; @@ -140,8 +141,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di auto blob = read(directory + "/" + weightFile); const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; m.weights[stage].create(m.device, blob.size(), usage); - m.uploads[stage].create(m.device, blob.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); - std::memcpy(m.uploads[stage].mapped, blob.data(), blob.size()); + m.weightBytes[stage] = std::move(blob); ++stage; } else if (key == "template") { std::string which, file; f >> which >> file; @@ -169,8 +169,7 @@ void RModel::recordUpload(VkCommandBuffer cb) { return; } for (unsigned i = 0; i < RStages; ++i) { - VkBufferCopy copy{0, 0, m.uploads[i].size}; - vkCmdCopyBuffer(cb, m.uploads[i].handle, m.weights[i].handle, 1, ©); + recordContents(cb, m.weights[i], m.weightBytes[i]); } memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); m.uploaded = true; @@ -188,7 +187,8 @@ struct REngine::Impl { std::array d112, d120; // the decoders' surfaces (texture 88 of the next decoder; dec0's 112: output stage) std::array q, r, h, e; // ping-pong: enc0's 304 / 224, dec0's 144 / enc0's 488, the output-size history, // and e: this frame's and the previous frame's exposure (1x1) - Tex out; // the presented image (the output stage's surface 240) + Tex out; // the presented image (the output stage's surface 240) when the caller passes none + bool outNew = false; // created by setFrame (first frame without the caller's output) // the output stage's second history (surface 384, read back as texture 392), kept only for games that pass // DLSSD.ColorBeforeParticles; created on first use Tex pb[2]; @@ -316,7 +316,6 @@ REngine::REngine(RModel& model, uint32_t renderWidth, uint32_t renderHeight, uin e.h[i].create(d, e.ow, e.oh, half4); e.e[i].create(d, 1, 1, VK_FORMAT_R16_SFLOAT); } - e.out.create(d, e.ow, e.oh, half4); VkSamplerCreateInfo si{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; si.addressModeU = si.addressModeV = si.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; si.maxLod = 0; VKCHECK(vkCreateSampler(d.handle, &si, nullptr, &e.nearest)); @@ -367,7 +366,7 @@ std::vector REngine::debugImages() const { list.push_back({"h" + std::to_string(i), e.h[i].handle, half4, e.ow, e.oh}); list.push_back({"e" + std::to_string(i), e.e[i].handle, VK_FORMAT_R16_SFLOAT, 1, 1}); } - list.push_back({"out", e.out.handle, half4, e.ow, e.oh}); + if (e.out.handle) list.push_back({"out", e.out.handle, half4, e.ow, e.oh}); return list; } @@ -378,7 +377,6 @@ void REngine::recordInitialize(VkCommandBuffer cb) { for (auto* group : {&e.s112, &e.s120}) for (auto& t : *group) images.push_back(t.handle); for (auto* group : {&e.s128, &e.d112, &e.d120}) for (auto& t : *group) images.push_back(t.handle); for (auto* group : {&e.q, &e.r, &e.h, &e.e}) for (auto& t : *group) images.push_back(t.handle); - images.push_back(e.out.handle); images.push_back(e.zero.handle); for (auto& [format, t] : e.spare) images.push_back(t.handle); VkClearColorValue zero{}; @@ -481,14 +479,19 @@ void REngine::setFrame(const RFrameImages& images, const RFrameSettings& setting for (auto& t : e.pb) t.create(d, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT); e.pbNew = true; } + if (!finalOutput.view && !e.out.handle) { + e.out.create(d, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT); + e.outNew = true; + } + const VkImageView frameOutput = finalOutput.view ? finalOutput.view : e.out.view; // texture and surface handles of the two stages with optional inputs: 1 where something is bound, 0 where NGX // leaves the argument null. The kernels branch on that (enc0 takes its specular hit distance path only when // argument 248's handle is not null; capDX, 2026-10-12). for (unsigned stage : {static_cast(REnc0), static_cast(RHkpn)}) { for (uint32_t offset : e.model.textures[stage]) - put64(e.blocks[stage], offset, e.resolve(stage, false, offset, 0, images, e.out.view).view ? 1u : 0u); + put64(e.blocks[stage], offset, e.resolve(stage, false, offset, 0, images, frameOutput).view ? 1u : 0u); for (const auto& [offset, format] : e.model.surfaces[stage]) - put64(e.blocks[stage], offset, e.resolve(stage, true, offset, 0, images, e.out.view).view ? 1u : 0u); + put64(e.blocks[stage], offset, e.resolve(stage, true, offset, 0, images, frameOutput).view ? 1u : 0u); } auto same = [&](const Impl::Bindings& b) { return !std::memcmp(&b.images, &images, sizeof(images)) && b.finalOutput.view == finalOutput.view; @@ -509,7 +512,7 @@ void REngine::setFrame(const RFrameImages& images, const RFrameSettings& setting VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; pi.maxSets = 2 * RStages + 1; pi.poolSizeCount = 3; pi.pPoolSizes = sizes; VKCHECK(vkCreateDescriptorPool(d.handle, &pi, nullptr, &b.pool)); - const VkImageView output = finalOutput.view ? finalOutput.view : e.out.view; + const VkImageView output = frameOutput; for (unsigned stage = 0; stage < RStages; ++stage) { for (uint32_t par = 0; par < 2; ++par) { VkDescriptorSet set = b.stages[stage][par] = descriptorSet(d.handle, b.pool, e.model.stages[stage].descriptors); @@ -558,17 +561,12 @@ void REngine::recordFrame(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); }; - if (e.pbNew) { // the particle history starts from zero, like the rest of the temporal state - VkClearColorValue zero{}; - VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - for (auto& t : e.pb) { - imageBarrier(cb, t.handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); - vkCmdClearColorImage(cb, t.handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range); - } + if (e.pbNew || e.outNew) { // images created by setFrame start from zero, like the rest of the temporal state + if (e.pbNew) for (auto& t : e.pb) clearNewImage(cb, t.handle); + if (e.outNew) clearNewImage(cb, e.out.handle); memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - e.pbNew = false; + e.pbNew = e.outNew = false; } stamp(0); // the frame's parameter blocks, then (on a reset) a clean temporal state diff --git a/engine/runtime_rr2.inc b/engine/runtime_rr2.inc index f3579ff..8b163c6 100644 --- a/engine/runtime_rr2.inc +++ b/engine/runtime_rr2.inc @@ -29,17 +29,20 @@ uint32_t r2Shift(unsigned stage, uint32_t frame) { } struct R2Stage { Pipeline pipeline; - Buffer weights, upload; + Buffer weights; + std::vector pending; // the weights until recordUpload puts them in the command buffer std::vector args; std::vector textures, buffers; std::vector> surfaces; std::vector> pointers; // argument offset, arena offset in units uint32_t group = 1; // "xN": a workgroup holds N windows along x (the stages generated around translated code) uint32_t record = 0; // R2Gather, "rN": 32-bit words per window in the arena + bool ranges = false; // R2Gather, "s3": the records' rows lie in three ranges (six words end enc0's block; else one address) }; struct R2Model { std::array stages; - Buffer aux, auxUpload; // encoder 0's second table (argument 104) + Buffer aux; // encoder 0's second table (argument 104) + std::vector auxPending; std::array exposure; bool particles = false; // the model has R2DownP bool gather = false; // the model has R2Gather @@ -53,10 +56,9 @@ struct R2Model { return items; }; const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; - auto blob = [&](Buffer& buffer, Buffer& upload, const std::vector& bytes) { + auto blob = [&](Buffer& buffer, std::vector& pending, std::vector bytes) { buffer.create(d, bytes.size(), usage); - upload.create(d, bytes.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); - std::memcpy(upload.mapped, bytes.data(), bytes.size()); + pending = std::move(bytes); }; std::array seen{}; std::string line; @@ -67,15 +69,18 @@ struct R2Model { // "stage NAME SHADER.spv SUBGROUP WEIGHTS|- ARGS TEXTURES SURFACES BUFFERS POINTERS": the lists are // comma-separated argument offsets ("-": none), surfaces OFFSET:FORMAT, pointers OFFSET:UNITS; an optional "xN" std::string name, shader, weightFile, argsFile, textureList, surfaceList, bufferList, pointerList, option; uint32_t subgroup = 32; - f >> name >> shader >> subgroup >> weightFile >> argsFile >> textureList >> surfaceList >> bufferList >> pointerList >> option; + f >> name >> shader >> subgroup >> weightFile >> argsFile >> textureList >> surfaceList >> bufferList >> pointerList; unsigned stage = 0; while (stage < R2Stages && name != R2Names[stage]) ++stage; if (stage >= R2Stages || seen[stage] || pointerList.empty()) throw std::runtime_error("rr.txt: unknown, repeated or invalid stage: " + line); seen[stage] = true; auto& st = stages[stage]; - if (option.size() == 2 && option[0] == 'x' && option[1] >= '1' && option[1] <= '4') st.group = static_cast(option[1] - '0'); - if (option.size() > 1 && option[0] == 'r') st.record = static_cast(std::stoul(option.substr(1))); + while (f >> option) { + if (option.size() == 2 && option[0] == 'x' && option[1] >= '1' && option[1] <= '4') st.group = static_cast(option[1] - '0'); + if (option.size() > 1 && option[0] == 'r') st.record = static_cast(std::stoul(option.substr(1))); + if (option == "s3") st.ranges = true; + } for (const auto& item : split(textureList)) st.textures.push_back(static_cast(std::stoul(item))); for (const auto& item : split(bufferList)) st.buffers.push_back(static_cast(std::stoul(item))); for (const auto& item : split(surfaceList)) { @@ -92,22 +97,21 @@ struct R2Model { types.insert(types.end(), st.buffers.size(), b); st.pipeline.create(d.handle, directory + "/" + shader, types, subgroup, cache); { const auto bytes = read(directory + "/" + argsFile); st.args.assign(bytes.begin(), bytes.end()); } - blob(st.weights, st.upload, weightFile == "-" ? std::vector(16) : read(directory + "/" + weightFile)); + blob(st.weights, st.pending, weightFile == "-" ? std::vector(16) : read(directory + "/" + weightFile)); } for (unsigned stage = 0; stage < R2DownP; ++stage) if (!seen[stage]) throw std::runtime_error("rr.txt: incomplete Ray Reconstruction preset F model"); particles = seen[R2DownP]; gather = seen[R2Gather] && stages[R2Gather].record != 0 && stages[R2Enc0].args.size() >= 8; for (unsigned stage : {R2DownP, R2Gather}) - if (!seen[stage]) blob(stages[stage].weights, stages[stage].upload, std::vector(16)); - blob(aux, auxUpload, read(directory + "/enc0-aux.bin")); + if (!seen[stage]) blob(stages[stage].weights, stages[stage].pending, std::vector(16)); + blob(aux, auxPending, read(directory + "/enc0-aux.bin")); exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache); exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache); } - void recordUpload(VkCommandBuffer cb) { - for (auto& st : stages) { VkBufferCopy copy{0, 0, st.upload.size}; vkCmdCopyBuffer(cb, st.upload.handle, st.weights.handle, 1, ©); } - VkBufferCopy copy{0, 0, auxUpload.size}; - vkCmdCopyBuffer(cb, auxUpload.handle, aux.handle, 1, ©); + void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents) + for (auto& st : stages) recordContents(cb, st.weights, st.pending); + recordContents(cb, aux, auxPending); } }; struct R2ExposureParams { int32_t last[2]; uint32_t blocksX, blocks; float invCount; uint32_t reset; }; @@ -118,15 +122,17 @@ struct R2Engine { uint32_t rw, rh, ow, oh, w0, h0; uint32_t iw, ih, cw, ch, qw, qh; // the intermediate (1.5x output), its half and quarter uint32_t nx, ny, maxW, maxH; // output tiles and the largest tile rectangle (with its overlap) - uint64_t unit, tileA, tileB; // arena unit; the first tile's two regions; a tile slot's bytes - uint64_t tileBytes = 0; + uint64_t unit, tileB; // arena unit; where a tile slot's second region starts in it + uint64_t tileBytes = 0; // a tile slot's bytes + std::vector slotAt; // each tile slot's place in the arena buffer // Tile slots (a pair of regions each) and tiles per group. The post passes of a group have a slot each and run // without waiting for one another, as do the down passes that read them; a group's down passes also overlap the // next group's post passes, which write the other half of the slots (NGX overlaps one post with one down the same // way, on two slots). Serialising all 32 passes of a 4K frame cost 0.17 ms in the gaps (RX 9070 XT, 2026-10-13). static constexpr uint32_t R2TileSlots = 8; uint32_t tileGroup = 1; - uint64_t gatherAt = 0; // the gather pass's records (R2Gather): after the tile regions + // the gather pass's records (R2Gather): byte address of range 0, its rows of windows, range 1, its rows, range 2 + std::array gatherWords{}; struct Rect { uint32_t x0, y0, x1, y1; }; std::vector src, dst; // per tile: intermediate rectangle (with overlap), output rectangle std::vector downPAt; @@ -134,6 +140,8 @@ struct R2Engine { // the particle layer's output-size history (R2DownP's surface 264, read back as texture 256): created on first use Tex pb[2]; bool pbNew = false; + // the engine's own output image: only for frames that do not write the caller's texture (created on first use) + bool outNew = false; Buffer arena, values, sums, dummy; // every parameter block of the frame in one buffer (one upload per frame): the network stages', per tile the two // output passes', the pooling pass's and the exposure's, each at a 256-byte boundary @@ -256,7 +264,6 @@ struct R2Engine { for (const auto& r : src) { maxW = std::max(maxW, r.x1 - r.x0); maxH = std::max(maxH, r.y1 - r.y0); } unit = static_cast(w0) * h0 / 32; const uint64_t tilePixels = static_cast(maxW + 8) * (maxH + 8); - tileA = unit * R2ArenaUnits; tileB = tileA + tilePixels * 4; tileBytes = tilePixels * 6; const uint32_t tileCount = nx * ny; const uint32_t slots = std::min(tileCount, R2TileSlots); @@ -266,11 +273,61 @@ struct R2Engine { // debugging: D4R_ENGINE_RR_ARENA=DIR keeps the arena and the exposure values host-visible and writes them at the // next setFrame (DIR/F-arena.bin, F-values.bin), when the caller has waited for frame F const bool debug = std::getenv("D4R_ENGINE_RR_ARENA") != nullptr; + // Every arena region is written and read within one frame, so the tile slots (live in the output stage only) and + // the gather records (live from the gather pass to the end of encoder 0) can share the buffer with the network's + // regions where those are dead: the tile slots that fit before the network's output (dec0_1's argument 64; the + // output stage also reads the arena after it), the records' rows of windows in the two largest gaps between + // encoder 0's regions ("s3" models) and from the end of its last region on (a region ends at the next region's + // start). Saves 105 MB at 720p -> 4K. D4R_ENGINE_RR_ARENA keeps the regions apart, so that dumps show the + // network's regions as NGX leaves them. + std::vector starts{R2ArenaUnits}; + for (unsigned stage = 0; stage < R2Network; ++stage) + for (const auto& [offset, units] : model.stages[stage].pointers) starts.push_back(static_cast(std::lround(units))); + std::sort(starts.begin(), starts.end()); + auto regionEnd = [&](uint32_t start) { return *std::upper_bound(starts.begin(), starts.end(), start); }; + uint32_t netOutStart = 0, enc0End = 0; + for (const auto& [offset, units] : model.stages[R2Dec0B].pointers) + if (offset == 64) netOutStart = static_cast(std::lround(units)); + for (unsigned stage : {R2Enc0, R2Gather}) + for (const auto& [offset, units] : model.stages[stage].pointers) enc0End = std::max(enc0End, regionEnd(static_cast(std::lround(units)))); + const uint64_t arenaBytes = unit * R2ArenaUnits; + const uint32_t before = debug ? 0 : static_cast(std::min(slots, unit * netOutStart / tileBytes)); + for (uint32_t k = 0; k < slots; ++k) slotAt.push_back(k < before ? tileBytes * k : arenaBytes + tileBytes * (k - before)); + tileB = tilePixels * 4; + const uint64_t tileEnd = arenaBytes + tileBytes * (slots - before); + const uint64_t tail = debug ? (tileEnd + 255) & ~uint64_t(255) : (unit * enc0End + 255) & ~uint64_t(255); // the gather pass's records: one per window of level 0, (W0 / 8 + 1) to a row (a shifted level has one more) - gatherAt = (tileA + tileBytes * slots + 255) & ~uint64_t(255); - const uint64_t gatherBytes = model.gather ? uint64_t(w0 / 8 + 1) * (h0 / 8 + 1) * model.stages[R2Gather].record * 4 : 0; - if (gatherAt + gatherBytes > 0xffffffffull) throw std::runtime_error("Ray Reconstruction: the render size is too large for the arena"); - arena.create(d, gatherAt + gatherBytes, storage, debug); + const uint32_t rows = h0 / 8 + 1; + const uint64_t rowBytes = model.gather ? uint64_t(w0 / 8 + 1) * model.stages[R2Gather].record * 4 : 1; + std::vector> ranges; // (byte address, rows) of the gaps used + if (!debug && model.gather && model.stages[R2Gather].ranges) { + std::vector> used; + for (unsigned stage : {R2Enc0, R2Gather}) + for (const auto& [offset, units] : model.stages[stage].pointers) { + const uint32_t start = static_cast(std::lround(units)); + used.push_back({start, regionEnd(start)}); + } + std::sort(used.begin(), used.end()); + uint32_t at = 0; + for (const auto& [start, end] : used) { + if (start > at) { + const uint64_t from = (unit * at + 255) & ~uint64_t(255), to = unit * start; + if (to > from && (to - from) / rowBytes > 0) ranges.push_back({from, static_cast((to - from) / rowBytes)}); + } + at = std::max(at, end); + } + std::sort(ranges.begin(), ranges.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); + if (ranges.size() > 2) ranges.resize(2); + } + uint32_t placed = 0; + for (const auto& r : ranges) placed += r.second; + ranges.resize(2, {tail, 0}); + gatherWords = {static_cast(ranges[0].first), ranges[0].second, static_cast(ranges[1].first), + ranges[1].second, static_cast(tail)}; + const uint64_t gatherTail = placed >= rows ? 0 : (rows - placed) * rowBytes; + const uint64_t arenaTotal = std::max({tileEnd, model.gather ? tail + gatherTail : 0}); + if (arenaTotal > 0xffffffffull) throw std::runtime_error("Ray Reconstruction: the render size is too large for the arena"); + arena.create(d, arenaTotal, storage, debug); values.create(d, 16, storage, debug); dummy.create(d, 4096, storage); exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16); @@ -298,7 +355,6 @@ struct R2Engine { rec.create(d, rw, rh, VK_FORMAT_R8G8B8A8_SNORM); ka.create(d, rw, rh, VK_FORMAT_R8G8B8A8_UNORM); kb.create(d, rw, rh, VK_FORMAT_R8G8B8A8_UNORM); kc.create(d, rw, rh, half4); half.create(d, cw, ch, pooledFormat()); quarter.create(d, qw, qh, pooledFormat()); - out.create(d, ow, oh, half4); zero.create(d, 1, 1, half4); for (const auto& st : model.stages) for (const auto& [offset, format] : st.surfaces) @@ -320,7 +376,10 @@ struct R2Engine { const float fr = static_cast(rw), fh = static_cast(rh), fiw = static_cast(iw), fih = static_cast(ih); const float fow = static_cast(ow), foh = static_cast(oh); auto& p = blocks[R2Enc0]; - if (model.gather) put64(p, p.size() - 8, gatherAt); // the word compile_rr2.py appends to the block + if (model.gather && model.stages[R2Gather].ranges) // the six words compile_rr2.py appends to the block + for (unsigned i = 0; i < 5; ++i) put32(p, p.size() - 24 + 4 * i, gatherWords[i]); + else if (model.gather) // an older model: one address, the records in one piece + put64(p, p.size() - 8, gatherWords[4]); put32(p, 128, rw); put32(p, 132, rh); put32(p, 136, w0); put32(p, 140, h0); putF(p, 408, fr / fiw); putF(p, 412, fh / fih); putF(p, 416, fiw / fr); putF(p, 420, fih / fh); putF(p, 424, 1.0f / fr); putF(p, 428, 1.0f / fh); putF(p, 432, 1.0f / fiw); putF(p, 436, 1.0f / fih); @@ -336,7 +395,7 @@ struct R2Engine { for (size_t i = 0; i < tiles; ++i) { auto& o = postBlocks[i]; put64(o, 0, netOut); put32(o, 8, w0); // the row stride of the network's output: the padded width - put64(o, 104, tileA + tileBytes * tileSlot(i)); put64(o, 112, tileB + tileBytes * tileSlot(i)); + put64(o, 104, slotAt[tileSlot(i)]); put64(o, 112, slotAt[tileSlot(i)] + tileB); putF(o, 132, fr / fiw); putF(o, 136, fh / fih); put32(o, 148, iw); put32(o, 152, ih); putF(o, 160, 1.0f / fiw); putF(o, 164, 1.0f / fih); put32(o, 168, rw); put32(o, 172, rh); put32(o, 176, src[i].x0); put32(o, 180, src[i].y0); put32(o, 184, src[i].x1); put32(o, 188, src[i].y1); @@ -347,7 +406,7 @@ struct R2Engine { if (!wb) continue; auto& w = *wb; put32(w, 0, iw); put32(w, 4, ih); put32(w, 8, ow); put32(w, 12, oh); put32(w, 16, ow); put32(w, 20, oh); - put64(w, 24, tileA + tileBytes * tileSlot(i)); put64(w, 32, tileB + tileBytes * tileSlot(i)); + put64(w, 24, slotAt[tileSlot(i)]); put64(w, 32, slotAt[tileSlot(i)] + tileB); put32(w, 96, rw - 1); put32(w, 100, rh - 1); putF(w, 104, fr / fow); putF(w, 108, fh / foh); putF(w, 112, fow / fiw); putF(w, 116, foh / fih); put32(w, 128, dst[i].x0); put32(w, 132, dst[i].y0); put32(w, 136, dst[i].x1); put32(w, 140, dst[i].y1); @@ -379,13 +438,13 @@ struct R2Engine { list.push_back({"kc", kc.handle, half4, rw, rh}); list.push_back({"half", half.handle, pooledFormat(), cw, ch}); list.push_back({"quarter", quarter.handle, pooledFormat(), qw, qh}); - list.push_back({"out", out.handle, half4, ow, oh}); + if (out.handle) list.push_back({"out", out.handle, half4, ow, oh}); return list; } void recordInitialize(VkCommandBuffer cb) { std::vector images; for (auto* group : {&feat, &hist, &e, &mv}) for (auto& t : *group) images.push_back(t.handle); - for (auto* t : {&rec, &ka, &kb, &kc, &half, &quarter, &out, &zero}) images.push_back(t->handle); + for (auto* t : {&rec, &ka, &kb, &kc, &half, &quarter, &zero}) images.push_back(t->handle); for (auto& [format, t] : spare) images.push_back(t.handle); VkClearColorValue clear{}; VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; @@ -483,6 +542,10 @@ struct R2Engine { pbNew = true; } } + if (!finalOutput.view && !out.handle) { + out.create(d, ow, oh, VK_FORMAT_R16G16B16A16_SFLOAT); + outNew = true; + } exposure.reset = rst ? 1u : 0u; reset = rst; auto same = [&](const Bindings& b) { @@ -540,10 +603,11 @@ struct R2Engine { bound = &*found; // texture and surface handles: 1 where something is bound, 0 where NGX leaves the argument null (the kernels // test optional inputs' handles); buffer pointers other than the arena's are zero + const VkImageView output = finalOutput.view ? finalOutput.view : out.view; auto handles = [&](unsigned stage, std::vector& block) { const auto& st = model.stages[stage]; - for (uint32_t offset : st.textures) put64(block, offset, resolve(stage, false, offset, 0, images, out.view).view ? 1u : 0u); - for (const auto& [offset, format] : st.surfaces) put64(block, offset, resolve(stage, true, offset, 0, images, out.view).view ? 1u : 0u); + for (uint32_t offset : st.textures) put64(block, offset, resolve(stage, false, offset, 0, images, output).view ? 1u : 0u); + for (const auto& [offset, format] : st.surfaces) put64(block, offset, resolve(stage, true, offset, 0, images, output).view ? 1u : 0u); for (uint32_t offset : st.buffers) put64(block, offset, 0); }; for (unsigned stage = 0; stage < R2Network; ++stage) handles(stage, blocks[stage]); @@ -570,17 +634,20 @@ struct R2Engine { memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); }; - if (pbNew) { // the particle history starts from zero, like the rest of the temporal state + if (pbNew || outNew) { // images created on first use start from zero, like the rest of the temporal state VkClearColorValue clear{}; VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - for (auto& t : pb) { - imageBarrier(cb, t.handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + std::vector fresh; + if (pbNew) for (auto& t : pb) fresh.push_back(t.handle); + if (outNew) fresh.push_back(out.handle); + for (VkImage image : fresh) { + imageBarrier(cb, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); - vkCmdClearColorImage(cb, t.handle, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range); + vkCmdClearColorImage(cb, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range); } memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - pbNew = false; + pbNew = outNew = false; } stamp(0); memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); diff --git a/tools/d4r_nvngx_shim.cpp b/tools/d4r_nvngx_shim.cpp index f94b01a..79ac72d 100644 --- a/tools/d4r_nvngx_shim.cpp +++ b/tools/d4r_nvngx_shim.cpp @@ -8736,12 +8736,17 @@ static bool engine_evaluate_m(Feature& feature, ID3D12GraphicsCommandList* list, model->hasExternalNetworkStages(), p.renderWidth, p.renderHeight, bytes, bytes); const d4r::ExternalNetwork external = {feature.engineTokens.buffer, feature.engineHead.buffer}; + // The Vulkan network of a dynamic-resolution model reserves the render size the feature was created with + // (NGX's maximum; at least this frame's), not the output size: at 1280x720 -> 4K that is 1.1 GB less for + // M and 1.4 GB for L. A larger frame goes to the output-sized engine below (engineMDynamic). + const bool dynamic = !hip && model->hasDynamicResolution(); + const uint32_t capacityWidth = dynamic ? std::min(feature.outWidth, std::max(feature.width, p.renderWidth)) : p.renderWidth; + const uint32_t capacityHeight = dynamic ? std::min(feature.outHeight, std::max(feature.height, p.renderHeight)) : p.renderHeight; feature.engineM = std::make_unique(*model, g_vk.physical, g_vk.device, feature.outWidth, - feature.outHeight, (hip || !model->hasDynamicResolution()) ? p.renderWidth : feature.outWidth, - (hip || !model->hasDynamicResolution()) ? p.renderHeight : feature.outHeight, + feature.outHeight, capacityWidth, capacityHeight, hip ? &external : nullptr); - feature.engineMRenderWidth = p.renderWidth; - feature.engineMRenderHeight = p.renderHeight; + feature.engineMRenderWidth = capacityWidth; + feature.engineMRenderHeight = capacityHeight; logf("engine backend (%s): feature %u runs preset %s in the game's command list (%ux%u -> %ux%u), network on %s, RGB10A2 direct output %d", name, feature.handle.Id, name, p.renderWidth, p.renderHeight, feature.outWidth, feature.outHeight, hip ? "the native HIP layers (list split around it)" : "Vulkan", @@ -8758,9 +8763,12 @@ static bool engine_evaluate_m(Feature& feature, ID3D12GraphicsCommandList* list, return refuse("the engine could not be created"); } d4r::GameUpscalerM* adapter = feature.engineM.get(); + // the HIP network's grid is fixed; the Vulkan engine takes render sizes up to the one it reserved const bool resizedHip = feature.engineNet != nullptr && (p.renderWidth != feature.engineMRenderWidth || p.renderHeight != feature.engineMRenderHeight); - if (resizedHip) + const bool beyondCapacity = feature.engineNet == nullptr && + (p.renderWidth > feature.engineMRenderWidth || p.renderHeight > feature.engineMRenderHeight); + if (resizedHip || beyondCapacity) { try { @@ -8768,7 +8776,9 @@ static bool engine_evaluate_m(Feature& feature, ID3D12GraphicsCommandList* list, { feature.engineMDynamic = std::make_unique(*model, g_vk.physical, g_vk.device, feature.outWidth, feature.outHeight, feature.outWidth, feature.outHeight); - logf("engine backend (%s): dynamic render sizes use the Vulkan network; original HIP resources retained", name); + logf("engine backend (%s): render size %ux%u beyond %ux%u: an output-sized Vulkan engine takes such frames; " + "the original engine is retained", name, p.renderWidth, p.renderHeight, feature.engineMRenderWidth, + feature.engineMRenderHeight); } adapter = feature.engineMDynamic.get(); }