engine: cut the models' and the engines' memory
- the weights go into the command buffer with vkCmdUpdateBuffer (recordContents); the host-visible staging copies are gone, and the host copies are dropped once recorded - an engine's own output image is created only for frames that do not write the caller's texture (K, M, RR D/E, RR F), cleared on first use - preset F: the tile slots share the arena with the network's regions, and the gather records' rows fill the gaps between encoder 0's regions (rr.txt "s3", six words on enc0's block; D4R_ENGINE_RR_ARENA keeps the old separate layout for arena dumps) - the shim reserves the render size the feature was created with for the M/L Vulkan network instead of the output size; a frame beyond it goes to a second, output-sized engine (1.1 GB less for M, 1.4 for L at 1280x720 -> 4K); replay_m gains D4R_M_CAPACITY Every reference capture still byte-identical; through the shim the harness process peaks 159 MiB lower.
This commit is contained in:
woutermans committed
1 parent
162bbd77c8
commit
af450a92b9
10 files changed
+237
-93
No files matched your search
+6
-2
@@ -755,8 +755,12 @@ conversion inputs up to the output size. Use a K package with all sixteen recons
|
||||
upscaling ratios; older single-table packages cannot match NGX at every ratio.
|
||||
|
||||
L and M update the Vulkan network's active token grid, exposure grid and reconstruction tier per frame.
|
||||
Their input and history planes reserve the constructor's render capacity; the shim reserves output-sized
|
||||
capacity for the Vulkan path. Bilinear reads clamp to the active rectangle. A render-size change resets
|
||||
Their input and history planes reserve the constructor's render capacity. The shim reserves the render size the
|
||||
feature was created with (or the first frame's, when larger), as NGX does; a later frame beyond that goes to a second,
|
||||
output-sized Vulkan engine, and the first is kept until the feature is released. Reserving the output size for every
|
||||
feature cost 0.85 GB (M) and 1.08 GB (L) at 1280x720 -> 4K; the images are byte-identical either way (harness,
|
||||
2026-10-14: created 1280x720 with 854x480 frames, created 854x480 with 1280x720 frames, and the second engine
|
||||
forced). Bilinear reads clamp to the active rectangle. A render-size change resets
|
||||
L/M temporal history. Input dimensions must be at least 64 on each axis and cannot exceed the output.
|
||||
Standalone callers set `GameFrameM::renderWidth/renderHeight` or `MFrameSettings::renderWidth/renderHeight`;
|
||||
zero keeps the constructor size.
|
||||
|
||||
@@ -147,8 +147,8 @@ def main():
|
||||
subprocess.run([sys.executable, str(HERE / 'rr' / 'rr2_pack.py'), stage, a.native, str(out / weights),
|
||||
str(work / 'rr2_stage.h')], check=True, stdout=subprocess.DEVNULL)
|
||||
defs = ['-DRR_EXACT_CONVERT'] + (['-DRR_MIX'] if os.environ.get('D4R_RR_MIX') == '1' else []) + FUSE_DEFS + RELAX_DEFS
|
||||
# encoder 0 in two passes (gen_rr2_enc0.py --split): its gather as the stage enc0g, whose record's address
|
||||
# is a word appended to the argument block
|
||||
# encoder 0 in two passes (gen_rr2_enc0.py --split): its gather as the stage enc0g, whose records' place
|
||||
# (three ranges) is six words appended to the argument block
|
||||
split = stage == 'enc0' and SPLIT and GROUP == 1
|
||||
how = ['--split', 'block', '--xf-word', str((len(launch.args) + 7) // 8 * 2)] if split else ['--group', str(GROUP)]
|
||||
if split:
|
||||
@@ -186,7 +186,7 @@ def main():
|
||||
(out / 'enc0-aux.bin').write_bytes(launch.pre(launch.pointer(104).alloc))
|
||||
args = stage_params(launch, 24 if network else 9999)[:len(launch.args)]
|
||||
if hybrid and split:
|
||||
args = args + bytes(-len(args) % 8) + bytes(8) # the gather record's address (the runtime's)
|
||||
args = args + bytes(-len(args) % 8) + bytes(24) # the gather records' three ranges (the runtime's)
|
||||
(out / f'{stage}.args').write_bytes(args)
|
||||
# the stage's arena regions, in units of 1/3152 of NGX's arena (W0 * H0 / 32 bytes: the regions scale with it)
|
||||
arena = launch.arena_alloc
|
||||
@@ -203,8 +203,8 @@ def main():
|
||||
wave = 32 if network else 64
|
||||
lines.append(f'stage {stage} {stage}.spv {wave} {weights} {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"}' +
|
||||
(f' x{GROUP}' if hybrid and GROUP > 1 else '')) # windows per workgroup (engine/rr/gen_rr2_enc0.py)
|
||||
if hybrid and split: # the same block, bindings and regions; "rN": N words per window record
|
||||
lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record}')
|
||||
if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges
|
||||
lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3')
|
||||
print(f'{stage}: {kernel}, {len(launch.args)} argument bytes' + (' (native block)' if hybrid else ''))
|
||||
for p in (0, 1):
|
||||
subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}',
|
||||
|
||||
+6
-1
@@ -127,7 +127,11 @@ int main(int argc, char** argv) try {
|
||||
network = std::make_unique<d4r::HipNetwork>(model, rw, rh, tokens.fd, result.fd, tokens.allocation, result.allocation);
|
||||
}
|
||||
#endif
|
||||
d4r::MEngine engine(m, rw, rh, ow, oh, hip ? &external : nullptr);
|
||||
// D4R_M_CAPACITY=WxH: the engine reserves a larger render size than the frames use (dynamic resolution, as a
|
||||
// shim reserving more than the feature's size would)
|
||||
uint32_t cw = rw, ch = rh;
|
||||
if (const char* capacity = std::getenv("D4R_M_CAPACITY")) std::sscanf(capacity, "%ux%u", &cw, &ch);
|
||||
d4r::MEngine engine(m, cw, ch, ow, oh, hip ? &external : nullptr);
|
||||
Image color = image(c, rw, rh, VK_FORMAT_R16G16B16A16_SFLOAT), motion = image(c, rw, rh, VK_FORMAT_R16G16_SFLOAT), depth = image(c, rw, rh, VK_FORMAT_R32_SFLOAT);
|
||||
const VkDeviceSize sizes[3] = {VkDeviceSize(rw) * rh * 8, VkDeviceSize(rw) * rh * 4, VkDeviceSize(rw) * rh * 4};
|
||||
Staging in[3] = {staging(c, sizes[0]), staging(c, sizes[1]), staging(c, sizes[2])}, out = staging(c, VkDeviceSize(ow) * oh * 8);
|
||||
@@ -162,6 +166,7 @@ int main(int argc, char** argv) try {
|
||||
if (pass == 0 || frames > 1)
|
||||
for (int k = 0; k < 3; ++k) vkCmdCopyBufferToImage(cb, in[k].buffer, images[k]->image, VK_IMAGE_LAYOUT_GENERAL, 1, ®ion);
|
||||
d4r::MFrameSettings s; s.jitter[0] = std::atof(a[3]); s.jitter[1] = std::atof(a[4]); s.reset = (pass == 0 && f == 0) || resetAll;
|
||||
s.renderWidth = rw; s.renderHeight = rh;
|
||||
const double t0 = now();
|
||||
engine.setFrame({color.view, motion.view, depth.view}, s);
|
||||
vkCmdResetQueryPool(cb, qp, 0, 7);
|
||||
|
||||
+28
-1
@@ -312,7 +312,7 @@ how the passes were arranged:
|
||||
- **Encoder 0 in two passes** (-0.2 ms). One shader ran NVIDIA's 32-lane gather on one of the block's four waves
|
||||
while the other three waited: 0.40 ms for the gather, 0.24 ms as a pass of its own in 32-thread groups.
|
||||
`gen_rr2_enc0.py --split gather | block` writes both; the gather pass (`stage enc0g`, the same bindings and
|
||||
parameter block as enc0) stores each window's record in the arena after the tile slots: the first product's A
|
||||
parameter block as enc0) stores each window's record in the arena buffer (see Memory below): the first product's A
|
||||
fragments (1024 words) and, per lane, the values the tail reads from the gather (six binary16 on the reference
|
||||
build; the generator finds them and recomputes the tail's index arithmetic). The record's address is a word
|
||||
compile_rr2.py appends to enc0's argument block. Cost: 76 bytes per render pixel of arena.
|
||||
@@ -354,6 +354,33 @@ scripts/run_rr_engine_compare.sh`, frames aligned by one):
|
||||
moving scene that showed as 40 - 50 dB on later frames in a synthetic scene whose affected feature is rounding
|
||||
noise around zero in both paths; no other configuration was affected.
|
||||
|
||||
### Memory (2026-10-14)
|
||||
|
||||
At 1280x720 -> 4K the engine needs about 600 MB for preset F; the two 1.5x output histories (2 x 149 MB, NGX has
|
||||
the same) are half of it. NVIDIA's DLL, its allocations traced through the CUDA path and replayed on an RTX PRO 4500
|
||||
Blackwell, takes 526 MiB of feature state, 106 MiB of input / output arrays and 2 MiB of code, plus 262 MiB of CUDA
|
||||
context. Three cuts, every capture still byte-identical (`verify.sh`, `verify_x.sh`) and the shim's frames
|
||||
byte-identical to the previous build, same speed:
|
||||
|
||||
- **The tile slots share the arena with the network's regions.** Every arena region is written and read within one
|
||||
frame; in the output stage only the network's output (dec0_1's argument 64) and something after it are read, so
|
||||
the slots that fit before the network's output go there (all 8 at 720p -> 4K) and the rest after the arena. A
|
||||
first try that put the slots right after the network's output broke 360p -> 1080p's output stage.
|
||||
- **The gather records' rows of windows fill the gaps between encoder 0's regions**: the two largest (units
|
||||
1216 - 2240 and 2304 - 2560) and then from the end of its last region (unit 2816) on, instead of one piece after
|
||||
the tile slots. The records (76 bytes a render pixel, about 2450 units) are larger than all the gaps, so 46 of 93
|
||||
rows at 720p still reach past the arena (36 MB). Models from compile_rr2.py mark this with "s3" on the enc0g line
|
||||
and append six words to enc0's block (three ranges' addresses, the first two's rows; `gen_rr2_enc0.py`); older
|
||||
models keep the one-piece layout. Same speed as one piece (four alternating runs at 720p -> 4K).
|
||||
- **The engine's own output image is created only for frames that do not write the game's texture**, and the
|
||||
weights go into the command buffer with `vkCmdUpdateBuffer` instead of through a host-visible staging copy that
|
||||
stayed allocated.
|
||||
|
||||
Through the shim (direct output) the harness process peaks 159 MiB lower (1100 -> 941 MiB). Not exact, so not
|
||||
done: the pooled histories as RGB10A2 (`D4R_RR2_POOLED=rgb10_a2`, -70 MB, see above); they hold q / 1023, which
|
||||
neither binary16 nor UNORM16 represents exactly. `D4R_ENGINE_RR_ARENA`
|
||||
keeps the old separate layout, so that arena dumps show the network's regions as NGX leaves them.
|
||||
|
||||
## Against a real NVIDIA card: `D4R_RR_FUSE` (2026-10-13)
|
||||
|
||||
Every reference above is the CUDA path through ZLUDA. NVIDIA's own kernels (310.9.1's `rr2_*` PTX, replayed through
|
||||
|
||||
@@ -117,8 +117,15 @@ def split_layout(xf_word, pre, tail, f16vars):
|
||||
if not (v in f16vars or re.fullmatch(r'r\d+', v)):
|
||||
raise SystemExit(f'the tail reads {v} from the gather: not a 16- or 32-bit register')
|
||||
record = 1024 + 32 * len(saved)
|
||||
# the records' rows of windows lie in up to three ranges of the arena (the runtime's choice, where the network's
|
||||
# regions are dead during encoder 0): words XF_WORD, +2, +4 their byte addresses, +1 and +3 the first two's rows
|
||||
where = (f' const uint rr2_gx = (param.w[0] + (param.w[20] & 0xffffu) + 7u) / 8u;\n'
|
||||
f' const uint rr2_rec = param.w[{xf_word}u] / 4u + {record}u * (gl_WorkGroupID.y * (param.w[0] / 8u + 1u) + gl_WorkGroupID.x);\n')
|
||||
f' uint rr2_y = gl_WorkGroupID.y, rr2_base = param.w[{xf_word}u];\n'
|
||||
f' if (rr2_y >= param.w[{xf_word + 1}u]) {{\n'
|
||||
f' rr2_y -= param.w[{xf_word + 1}u]; rr2_base = param.w[{xf_word + 2}u];\n'
|
||||
f' if (rr2_y >= param.w[{xf_word + 3}u]) {{ rr2_y -= param.w[{xf_word + 3}u]; rr2_base = param.w[{xf_word + 4}u]; }}\n'
|
||||
f' }}\n'
|
||||
f' const uint rr2_rec = rr2_base / 4u + {record}u * (rr2_y * (param.w[0] / 8u + 1u) + gl_WorkGroupID.x);\n')
|
||||
return recomputed, saved, record, where
|
||||
|
||||
|
||||
@@ -128,7 +135,8 @@ def gather_pass(xf_word, head, decls, pre, tail, first, f16vars):
|
||||
One shader holding both runs the gather on one of the block's four waves while the other three wait, which
|
||||
costs more than the record's trip through memory (RX 9070 XT, 1280x720, 2026-10-13: gather 0.40 -> 0.24 ms).
|
||||
The record: 1024 words of A fragments ([token slot][64 FP8 feature slots], as the block's XF), then per lane the
|
||||
gather's values the tail reads. Its address: parameter word XF_WORD (bytes) + 4 * RECORD * (y * (W / 8 + 1) + x)."""
|
||||
gather's values the tail reads. Its address: the byte address of row y's range (parameter words XF_WORD to
|
||||
XF_WORD + 4) + 4 * RECORD * (y' * (W / 8 + 1) + x), y' the row within its range."""
|
||||
recomputed, saved, record, where = split_layout(xf_word, pre, tail, f16vars)
|
||||
var = lambda r: r.lstrip('%')
|
||||
store = [' if (rr2_valid) { // the first product\'s A fragments: lane 4g + t, register q -> row, 4 k',
|
||||
|
||||
+30
-10
@@ -79,6 +79,14 @@ struct Buffer {
|
||||
if (host) VKCHECK(vkMapMemory(d.handle, memory, 0, VK_WHOLE_SIZE, 0, &mapped));
|
||||
}
|
||||
};
|
||||
// A buffer's contents written into the command buffer itself (vkCmdUpdateBuffer, at most 65536 bytes a call), so no
|
||||
// host-visible staging copy stays allocated; the host copy (padded to the buffer's size) is dropped once recorded
|
||||
void recordContents(VkCommandBuffer cb, const Buffer& buffer, std::vector<char>& bytes) {
|
||||
bytes.resize(buffer.size);
|
||||
for (VkDeviceSize at = 0; at < bytes.size(); at += 65536)
|
||||
vkCmdUpdateBuffer(cb, buffer.handle, at, std::min<VkDeviceSize>(65536, bytes.size() - at), bytes.data() + at);
|
||||
std::vector<char>().swap(bytes);
|
||||
}
|
||||
struct Image {
|
||||
const Device* device = nullptr;
|
||||
VkImage handle = VK_NULL_HANDLE;
|
||||
@@ -125,6 +133,15 @@ void imageBarrier(VkCommandBuffer cb, VkImage image, VkImageLayout before, VkIma
|
||||
barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
vkCmdPipelineBarrier(cb, source, destination, 0, 0, nullptr, 0, nullptr, 1, &barrier);
|
||||
}
|
||||
// An image created after recordInitialize (an output only some frames need): GENERAL and zero, as recordInitialize
|
||||
// leaves its images; the caller's next barrier covers the clear
|
||||
void clearNewImage(VkCommandBuffer cb, VkImage image) {
|
||||
VkClearColorValue zero{};
|
||||
VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
imageBarrier(cb, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
vkCmdClearColorImage(cb, image, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range);
|
||||
}
|
||||
void memoryBarrier(VkCommandBuffer cb, VkPipelineStageFlags source, VkPipelineStageFlags destination,
|
||||
VkAccessFlags src, VkAccessFlags dst) {
|
||||
VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
||||
@@ -274,7 +291,8 @@ FrameParams makeKFrameParams(const Shape& shape, const FrameSettings& settings)
|
||||
|
||||
struct Model::Impl {
|
||||
Device device;
|
||||
Buffer weights, lut, weightUpload, lutUpload;
|
||||
Buffer weights, lut;
|
||||
std::vector<char> weightBytes, lutBytes; // until recordUpload
|
||||
std::array<std::array<uint32_t, 108>, KLayers> offsets{};
|
||||
std::array<Pipeline, KLayers> network;
|
||||
Pipeline input, output, outputRgb10;
|
||||
@@ -332,9 +350,7 @@ Model::Model(VkPhysicalDevice physical, VkDevice device, const std::string& dire
|
||||
throw std::runtime_error("model weights exceed the device's storage buffer range");
|
||||
m.weights.create(m.device, weights.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
m.lut.create(m.device, lut.size(), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
|
||||
m.weightUpload.create(m.device, weights.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true);
|
||||
m.lutUpload.create(m.device, lut.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true);
|
||||
std::memcpy(m.weightUpload.mapped, weights.data(), weights.size()); std::memcpy(m.lutUpload.mapped, lut.data(), lut.size());
|
||||
m.weightBytes = std::move(weights); m.lutBytes = std::move(lut);
|
||||
const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||
for (unsigned i = 0; i < KLayers; ++i) m.network[i].create(device, directory + "/" + Names[i] + ".spv", {b,b,b,b,b,b}, 32, cache);
|
||||
m.input.create(device, directory + "/input_k.spv", {b,t,t,t,t,t,b}, 64, cache);
|
||||
@@ -359,8 +375,7 @@ bool Model::uploadRecorded() const { return impl->uploaded; }
|
||||
void Model::recordUpload(VkCommandBuffer cb) {
|
||||
auto& m = *impl;
|
||||
if (m.uploaded) throw std::runtime_error("model upload already recorded");
|
||||
VkBufferCopy copy{0, 0, m.weightUpload.size}; vkCmdCopyBuffer(cb, m.weightUpload.handle, m.weights.handle, 1, ©);
|
||||
copy.size = m.lutUpload.size; vkCmdCopyBuffer(cb, m.lutUpload.handle, m.lut.handle, 1, ©);
|
||||
recordContents(cb, m.weights, m.weightBytes); recordContents(cb, m.lut, m.lutBytes);
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
m.uploaded = true;
|
||||
}
|
||||
@@ -378,7 +393,8 @@ struct Engine::Impl {
|
||||
std::array<Buffer, 5> skip;
|
||||
struct History { Image color, luma, feature; };
|
||||
std::array<History, 2> history;
|
||||
Image final;
|
||||
Image final; // the engine's own output: only for frames that do not write the caller's image (created on first use)
|
||||
bool finalNew = false;
|
||||
VkDescriptorPool pool = VK_NULL_HANDLE;
|
||||
std::array<VkDescriptorSet, KLayers> layerSets{};
|
||||
std::array<VkDescriptorSet, 2> inputSets{}, outputSets{};
|
||||
@@ -473,7 +489,6 @@ Engine::Engine(Model& model, const Shape& shape, const ExternalNetwork* external
|
||||
VKCHECK(vkCreateSampler(dev, &si, nullptr, &e.nearest));
|
||||
si.magFilter = si.minFilter = VK_FILTER_LINEAR;
|
||||
VKCHECK(vkCreateSampler(dev, &si, nullptr, &e.linear));
|
||||
e.final.create(d, shape.outputWidth, shape.outputHeight);
|
||||
for (unsigned i = 0; i < 2; ++i) {
|
||||
auto& h = e.history[i];
|
||||
h.color.create(d, shape.outputWidth, shape.outputHeight);
|
||||
@@ -504,7 +519,7 @@ Engine::Engine(Model& model, const Shape& shape, const ExternalNetwork* external
|
||||
bindBuffer(dev, out, 0, e.outParams); bindBuffer(dev, out, 10, e.main[1]); bindBuffer(dev, out, 11, e.model.lut);
|
||||
bindImage(dev, out, 2, h.color.sample, e.linear); bindImage(dev, out, 3, h.luma.sample, e.linear);
|
||||
bindImage(dev, out, 6, dest.color.storage); bindImage(dev, out, 7, dest.luma.storage);
|
||||
bindImage(dev, out, 8, dest.feature.storage); bindImage(dev, out, 9, e.final.storage);
|
||||
bindImage(dev, out, 8, dest.feature.storage); // binding 9, the output: setFrame
|
||||
}
|
||||
}
|
||||
Engine::~Engine() = default;
|
||||
@@ -519,7 +534,6 @@ void Engine::recordInitialize(VkCommandBuffer cb) {
|
||||
vkCmdClearColorImage(cb, image.handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range);
|
||||
};
|
||||
for (auto& history : e.history) { init(history.color); init(history.luma); init(history.feature); }
|
||||
init(e.final);
|
||||
if (e.model.hasExposure) {
|
||||
const float one[2] = {1, 1};
|
||||
vkCmdUpdateBuffer(cb, e.exposureState.handle, 0, sizeof(one), one);
|
||||
@@ -585,6 +599,10 @@ void Engine::setFrame(const FrameImages& images, const InputParams& input, const
|
||||
images.colorLayout == b.colorLayout && images.motionLayout == b.motionLayout && images.depthLayout == b.depthLayout &&
|
||||
images.exposure == b.exposure && images.exposureLayout == b.exposureLayout;
|
||||
};
|
||||
if (!finalOutput.view && !e.final.handle) {
|
||||
e.final.create(e.model.device, e.shape.outputWidth, e.shape.outputHeight);
|
||||
e.finalNew = true;
|
||||
}
|
||||
auto found = std::find_if(e.bindings.begin(), e.bindings.end(), [&](const auto& b) {
|
||||
return sameImages(b.images) && finalOutput.view == b.finalOutput.view && finalOutput.format == b.finalOutput.format;
|
||||
});
|
||||
@@ -664,6 +682,7 @@ void Engine::recordFrame(VkCommandBuffer cb, const FrameProfiler& profiler) {
|
||||
if (!e.initialized || !e.model.uploaded || !e.frameSet) throw std::runtime_error("initialize/upload/setFrame before recording inference");
|
||||
auto mark = [&](uint32_t boundary) { if (profiler.mark) profiler.mark(cb, boundary, profiler.user); };
|
||||
mark(0);
|
||||
if (e.finalNew) { clearNewImage(cb, e.final.handle); e.finalNew = false; }
|
||||
// Includes reads-to-writes when reusing scratch and histories, and copies made by the caller after the last frame.
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
@@ -716,6 +735,7 @@ void Engine::recordFront(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first)
|
||||
if (!e.external) throw std::runtime_error("this engine's network is internal: record the whole frame");
|
||||
if (!e.initialized || !e.model.uploaded || !e.frameSet || e.frontDone) throw std::runtime_error("initialize/upload/setFrame before recording inference");
|
||||
const auto compute = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
|
||||
if (e.finalNew) { clearNewImage(cb, e.final.handle); e.finalNew = false; }
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, compute | VK_PIPELINE_STAGE_TRANSFER_BIT,
|
||||
VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
vkCmdUpdateBuffer(cb, e.inParams.handle, 0, sizeof(InputParams), &e.input);
|
||||
|
||||
+14
-9
@@ -68,7 +68,8 @@ void bindRange(VkDevice device, VkDescriptorSet set, uint32_t binding, const Buf
|
||||
|
||||
struct MModel::Impl {
|
||||
Device device;
|
||||
Buffer w8, w16, up8, up16;
|
||||
Buffer w8, w16;
|
||||
std::vector<char> bytes8, bytes16; // until recordUpload
|
||||
uint32_t enc0[3]{}, dec0[2]{};
|
||||
std::array<std::array<uint32_t, 74>, MLayers> layers{};
|
||||
std::array<Pipeline, MLayers> network; // one per layer (five distinct shaders)
|
||||
@@ -115,8 +116,7 @@ MModel::MModel(VkPhysicalDevice physical, VkDevice device, const std::string& di
|
||||
for (auto& l : m.layers) inside(l[0], l[1]);
|
||||
const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
m.w8.create(m.device, b8.size(), usage); m.w16.create(m.device, b16.size(), usage);
|
||||
m.up8.create(m.device, b8.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true); m.up16.create(m.device, b16.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true);
|
||||
std::memcpy(m.up8.mapped, b8.data(), b8.size()); std::memcpy(m.up16.mapped, b16.data(), b16.size());
|
||||
m.bytes8 = b8; m.bytes16 = b16;
|
||||
const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||
for (unsigned i = 0; i < MLayers; ++i) m.network[i].create(device, directory + "/" + MShader[i] + ".spv", {b,b,b,b,b,b,b}, 32, cache);
|
||||
// layer_m.comp's L stages: bindings 0-6 of a layer, then the stage's own (layer_m.comp KIND 3 and 4)
|
||||
@@ -149,8 +149,7 @@ bool MModel::isPresetL() const { return impl->presetL; }
|
||||
void MModel::recordUpload(VkCommandBuffer cb) {
|
||||
auto& m = *impl;
|
||||
if (m.uploaded) throw std::runtime_error("model upload already recorded");
|
||||
VkBufferCopy copy{0, 0, m.up8.size}; vkCmdCopyBuffer(cb, m.up8.handle, m.w8.handle, 1, ©);
|
||||
copy.size = m.up16.size; vkCmdCopyBuffer(cb, m.up16.handle, m.w16.handle, 1, ©);
|
||||
recordContents(cb, m.w8, m.bytes8); recordContents(cb, m.w16, m.bytes16);
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
m.uploaded = true;
|
||||
}
|
||||
@@ -163,7 +162,9 @@ struct MEngine::Impl {
|
||||
Buffer tokens, skip1, merged1, skip2, merged2, tubeA, tubeB, dec2Out, dec1Out;
|
||||
Buffer skipL, enc0LP, dec0LP; // preset L: the input stage's pixel output and the two stages' layer blocks
|
||||
std::array<Tex, 2> mvA, feat, hist, half;
|
||||
Tex mvB, mvC, gates, mix, cov, output;
|
||||
Tex mvB, mvC, gates, mix, cov;
|
||||
Tex output; // the engine's own output: only for frames that do not write the caller's image (created on first use)
|
||||
bool outputNew = false;
|
||||
VkDescriptorPool pool = VK_NULL_HANDLE;
|
||||
std::array<VkDescriptorSet, MLayers> layerSets{};
|
||||
std::array<VkDescriptorSet, 2> enc0Sets{}, dec0Sets{}, postSets{}, expSets{};
|
||||
@@ -233,7 +234,6 @@ MEngine::MEngine(MModel& model, uint32_t renderWidth, uint32_t renderHeight, uin
|
||||
for (auto& t : e.hist) t.create(d, e.hw, e.hh, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
// The native formatted RGB10 history has RGBA32F backing. UNORM filtering differs by up to three ulps.
|
||||
for (auto& t : e.half) t.create(d, e.hw / 2, e.hh / 2, VK_FORMAT_R32G32B32A32_SFLOAT);
|
||||
e.output.create(d, outputWidth, outputHeight, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
VkDescriptorPoolSize sizes[] = {{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 128}, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 48}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 32}};
|
||||
VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
|
||||
pi.maxSets = 24; pi.poolSizeCount = 3; pi.pPoolSizes = sizes;
|
||||
@@ -305,14 +305,14 @@ MEngine::MEngine(MModel& model, uint32_t renderWidth, uint32_t renderHeight, uin
|
||||
bindBuffer(dev, e.expSets[1], 1, e.expSums); bindBuffer(dev, e.expSets[1], 2, e.expState);
|
||||
bindBuffer(dev, e.expSets[1], 3, e.enc0P); bindBuffer(dev, e.expSets[1], 4, e.postP); bindBuffer(dev, e.expSets[1], 5, e.downP);
|
||||
e.downSet = descriptorSet(dev, e.pool, m.down.descriptors);
|
||||
bindBuffer(dev, e.downSet, 0, e.downP); bindImage(dev, e.downSet, 2, e.output.view); bindBuffer(dev, e.downSet, 3, e.packed);
|
||||
bindBuffer(dev, e.downSet, 0, e.downP); bindBuffer(dev, e.downSet, 3, e.packed); // binding 2, the output: setFrame
|
||||
}
|
||||
MEngine::~MEngine() = default;
|
||||
void MEngine::recordInitialize(VkCommandBuffer cb) {
|
||||
auto& e = *impl;
|
||||
if (e.initialized) throw std::runtime_error("engine initialization already recorded");
|
||||
VkClearColorValue zero{}; VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
for (Tex* t : {&e.mvA[0], &e.mvA[1], &e.mvB, &e.mvC, &e.feat[0], &e.feat[1], &e.gates, &e.mix, &e.cov, &e.hist[0], &e.hist[1], &e.half[0], &e.half[1], &e.output}) {
|
||||
for (Tex* t : {&e.mvA[0], &e.mvA[1], &e.mvB, &e.mvC, &e.feat[0], &e.feat[1], &e.gates, &e.mix, &e.cov, &e.hist[0], &e.hist[1], &e.half[0], &e.half[1]}) {
|
||||
imageBarrier(cb, t->handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
vkCmdClearColorImage(cb, t->handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range);
|
||||
}
|
||||
@@ -358,6 +358,10 @@ void MEngine::setFrame(const FrameImages& images, const MFrameSettings& s, Frame
|
||||
e.prevJitter[0] = s.jitter[0]; e.prevJitter[1] = s.jitter[1];
|
||||
auto dev = e.model.device.handle;
|
||||
auto& m = e.model;
|
||||
if (!finalOutput.view && !e.output.handle) {
|
||||
e.output.create(e.model.device, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
e.outputNew = true;
|
||||
}
|
||||
auto found = std::find_if(e.bindings.begin(), e.bindings.end(), [&](const auto& b) {
|
||||
return b.images.color == images.color && b.images.motion == images.motion && b.images.depth == images.depth &&
|
||||
b.images.colorLayout == images.colorLayout && b.images.motionLayout == images.motionLayout &&
|
||||
@@ -446,6 +450,7 @@ void closeStages(VkCommandBuffer cb) { memoryBarrier(cb, compute, VK_PIPELINE_ST
|
||||
void MEngine::recordFront(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first) {
|
||||
auto& e = *impl; auto& m = e.model;
|
||||
if (!e.initialized || !m.uploaded || !e.frameSet || e.frontDone) throw std::runtime_error("initialize/upload/setFrame before recording inference");
|
||||
if (e.outputNew) { clearNewImage(cb, e.output.handle); e.outputNew = false; }
|
||||
openStages(cb);
|
||||
vkCmdUpdateBuffer(cb, e.enc0P.handle, 0, sizeof(e.enc0), &e.enc0); vkCmdUpdateBuffer(cb, e.dec0P.handle, 0, sizeof(e.dec0), &e.dec0);
|
||||
vkCmdUpdateBuffer(cb, e.postP.handle, 0, sizeof(e.post), &e.post); vkCmdUpdateBuffer(cb, e.downP.handle, 0, sizeof(e.down), &e.down);
|
||||
|
||||
+19
-21
@@ -81,7 +81,8 @@ struct RModel::Impl {
|
||||
// surface argument offsets with their GLSL image format, both ascending
|
||||
std::array<std::vector<uint32_t>, RStages> textures;
|
||||
std::array<std::vector<std::pair<uint32_t, std::string>>, RStages> surfaces;
|
||||
std::array<Buffer, RStages> weights, uploads;
|
||||
std::array<Buffer, RStages> weights;
|
||||
std::array<std::vector<char>, RStages> weightBytes; // until recordUpload
|
||||
std::array<Pipeline, RStages> stages;
|
||||
std::array<Pipeline, 2> exposure;
|
||||
std::vector<uint8_t> enc0Template, hkpnTemplate;
|
||||
@@ -140,8 +141,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di
|
||||
auto blob = read(directory + "/" + weightFile);
|
||||
const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
m.weights[stage].create(m.device, blob.size(), usage);
|
||||
m.uploads[stage].create(m.device, blob.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true);
|
||||
std::memcpy(m.uploads[stage].mapped, blob.data(), blob.size());
|
||||
m.weightBytes[stage] = std::move(blob);
|
||||
++stage;
|
||||
} else if (key == "template") {
|
||||
std::string which, file; f >> which >> file;
|
||||
@@ -169,8 +169,7 @@ void RModel::recordUpload(VkCommandBuffer cb) {
|
||||
return;
|
||||
}
|
||||
for (unsigned i = 0; i < RStages; ++i) {
|
||||
VkBufferCopy copy{0, 0, m.uploads[i].size};
|
||||
vkCmdCopyBuffer(cb, m.uploads[i].handle, m.weights[i].handle, 1, ©);
|
||||
recordContents(cb, m.weights[i], m.weightBytes[i]);
|
||||
}
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
|
||||
m.uploaded = true;
|
||||
@@ -188,7 +187,8 @@ struct REngine::Impl {
|
||||
std::array<Tex, 5> d112, d120; // the decoders' surfaces (texture 88 of the next decoder; dec0's 112: output stage)
|
||||
std::array<Tex, 2> q, r, h, e; // ping-pong: enc0's 304 / 224, dec0's 144 / enc0's 488, the output-size history,
|
||||
// and e: this frame's and the previous frame's exposure (1x1)
|
||||
Tex out; // the presented image (the output stage's surface 240)
|
||||
Tex out; // the presented image (the output stage's surface 240) when the caller passes none
|
||||
bool outNew = false; // created by setFrame (first frame without the caller's output)
|
||||
// the output stage's second history (surface 384, read back as texture 392), kept only for games that pass
|
||||
// DLSSD.ColorBeforeParticles; created on first use
|
||||
Tex pb[2];
|
||||
@@ -316,7 +316,6 @@ REngine::REngine(RModel& model, uint32_t renderWidth, uint32_t renderHeight, uin
|
||||
e.h[i].create(d, e.ow, e.oh, half4);
|
||||
e.e[i].create(d, 1, 1, VK_FORMAT_R16_SFLOAT);
|
||||
}
|
||||
e.out.create(d, e.ow, e.oh, half4);
|
||||
VkSamplerCreateInfo si{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
|
||||
si.addressModeU = si.addressModeV = si.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; si.maxLod = 0;
|
||||
VKCHECK(vkCreateSampler(d.handle, &si, nullptr, &e.nearest));
|
||||
@@ -367,7 +366,7 @@ std::vector<REngine::DebugImage> REngine::debugImages() const {
|
||||
list.push_back({"h" + std::to_string(i), e.h[i].handle, half4, e.ow, e.oh});
|
||||
list.push_back({"e" + std::to_string(i), e.e[i].handle, VK_FORMAT_R16_SFLOAT, 1, 1});
|
||||
}
|
||||
list.push_back({"out", e.out.handle, half4, e.ow, e.oh});
|
||||
if (e.out.handle) list.push_back({"out", e.out.handle, half4, e.ow, e.oh});
|
||||
return list;
|
||||
}
|
||||
|
||||
@@ -378,7 +377,6 @@ void REngine::recordInitialize(VkCommandBuffer cb) {
|
||||
for (auto* group : {&e.s112, &e.s120}) for (auto& t : *group) images.push_back(t.handle);
|
||||
for (auto* group : {&e.s128, &e.d112, &e.d120}) for (auto& t : *group) images.push_back(t.handle);
|
||||
for (auto* group : {&e.q, &e.r, &e.h, &e.e}) for (auto& t : *group) images.push_back(t.handle);
|
||||
images.push_back(e.out.handle);
|
||||
images.push_back(e.zero.handle);
|
||||
for (auto& [format, t] : e.spare) images.push_back(t.handle);
|
||||
VkClearColorValue zero{};
|
||||
@@ -481,14 +479,19 @@ void REngine::setFrame(const RFrameImages& images, const RFrameSettings& setting
|
||||
for (auto& t : e.pb) t.create(d, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
e.pbNew = true;
|
||||
}
|
||||
if (!finalOutput.view && !e.out.handle) {
|
||||
e.out.create(d, e.ow, e.oh, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
e.outNew = true;
|
||||
}
|
||||
const VkImageView frameOutput = finalOutput.view ? finalOutput.view : e.out.view;
|
||||
// texture and surface handles of the two stages with optional inputs: 1 where something is bound, 0 where NGX
|
||||
// leaves the argument null. The kernels branch on that (enc0 takes its specular hit distance path only when
|
||||
// argument 248's handle is not null; capDX, 2026-10-12).
|
||||
for (unsigned stage : {static_cast<unsigned>(REnc0), static_cast<unsigned>(RHkpn)}) {
|
||||
for (uint32_t offset : e.model.textures[stage])
|
||||
put64(e.blocks[stage], offset, e.resolve(stage, false, offset, 0, images, e.out.view).view ? 1u : 0u);
|
||||
put64(e.blocks[stage], offset, e.resolve(stage, false, offset, 0, images, frameOutput).view ? 1u : 0u);
|
||||
for (const auto& [offset, format] : e.model.surfaces[stage])
|
||||
put64(e.blocks[stage], offset, e.resolve(stage, true, offset, 0, images, e.out.view).view ? 1u : 0u);
|
||||
put64(e.blocks[stage], offset, e.resolve(stage, true, offset, 0, images, frameOutput).view ? 1u : 0u);
|
||||
}
|
||||
auto same = [&](const Impl::Bindings& b) {
|
||||
return !std::memcmp(&b.images, &images, sizeof(images)) && b.finalOutput.view == finalOutput.view;
|
||||
@@ -509,7 +512,7 @@ void REngine::setFrame(const RFrameImages& images, const RFrameSettings& setting
|
||||
VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
|
||||
pi.maxSets = 2 * RStages + 1; pi.poolSizeCount = 3; pi.pPoolSizes = sizes;
|
||||
VKCHECK(vkCreateDescriptorPool(d.handle, &pi, nullptr, &b.pool));
|
||||
const VkImageView output = finalOutput.view ? finalOutput.view : e.out.view;
|
||||
const VkImageView output = frameOutput;
|
||||
for (unsigned stage = 0; stage < RStages; ++stage) {
|
||||
for (uint32_t par = 0; par < 2; ++par) {
|
||||
VkDescriptorSet set = b.stages[stage][par] = descriptorSet(d.handle, b.pool, e.model.stages[stage].descriptors);
|
||||
@@ -558,17 +561,12 @@ void REngine::recordFrame(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
};
|
||||
if (e.pbNew) { // the particle history starts from zero, like the rest of the temporal state
|
||||
VkClearColorValue zero{};
|
||||
VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
for (auto& t : e.pb) {
|
||||
imageBarrier(cb, t.handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
vkCmdClearColorImage(cb, t.handle, VK_IMAGE_LAYOUT_GENERAL, &zero, 1, &range);
|
||||
}
|
||||
if (e.pbNew || e.outNew) { // images created by setFrame start from zero, like the rest of the temporal state
|
||||
if (e.pbNew) for (auto& t : e.pb) clearNewImage(cb, t.handle);
|
||||
if (e.outNew) clearNewImage(cb, e.out.handle);
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||
VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
e.pbNew = false;
|
||||
e.pbNew = e.outNew = false;
|
||||
}
|
||||
stamp(0);
|
||||
// the frame's parameter blocks, then (on a reset) a clean temporal state
|
||||
|
||||
+103
-36
@@ -29,17 +29,20 @@ uint32_t r2Shift(unsigned stage, uint32_t frame) {
|
||||
}
|
||||
struct R2Stage {
|
||||
Pipeline pipeline;
|
||||
Buffer weights, upload;
|
||||
Buffer weights;
|
||||
std::vector<char> pending; // the weights until recordUpload puts them in the command buffer
|
||||
std::vector<uint8_t> args;
|
||||
std::vector<uint32_t> textures, buffers;
|
||||
std::vector<std::pair<uint32_t, std::string>> surfaces;
|
||||
std::vector<std::pair<uint32_t, double>> pointers; // argument offset, arena offset in units
|
||||
uint32_t group = 1; // "xN": a workgroup holds N windows along x (the stages generated around translated code)
|
||||
uint32_t record = 0; // R2Gather, "rN": 32-bit words per window in the arena
|
||||
bool ranges = false; // R2Gather, "s3": the records' rows lie in three ranges (six words end enc0's block; else one address)
|
||||
};
|
||||
struct R2Model {
|
||||
std::array<R2Stage, R2Stages> stages;
|
||||
Buffer aux, auxUpload; // encoder 0's second table (argument 104)
|
||||
Buffer aux; // encoder 0's second table (argument 104)
|
||||
std::vector<char> auxPending;
|
||||
std::array<Pipeline, 2> exposure;
|
||||
bool particles = false; // the model has R2DownP
|
||||
bool gather = false; // the model has R2Gather
|
||||
@@ -53,10 +56,9 @@ struct R2Model {
|
||||
return items;
|
||||
};
|
||||
const VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
|
||||
auto blob = [&](Buffer& buffer, Buffer& upload, const std::vector<char>& bytes) {
|
||||
auto blob = [&](Buffer& buffer, std::vector<char>& pending, std::vector<char> bytes) {
|
||||
buffer.create(d, bytes.size(), usage);
|
||||
upload.create(d, bytes.size(), VK_BUFFER_USAGE_TRANSFER_SRC_BIT, true);
|
||||
std::memcpy(upload.mapped, bytes.data(), bytes.size());
|
||||
pending = std::move(bytes);
|
||||
};
|
||||
std::array<bool, R2Stages> seen{};
|
||||
std::string line;
|
||||
@@ -67,15 +69,18 @@ struct R2Model {
|
||||
// "stage NAME SHADER.spv SUBGROUP WEIGHTS|- ARGS TEXTURES SURFACES BUFFERS POINTERS": the lists are
|
||||
// comma-separated argument offsets ("-": none), surfaces OFFSET:FORMAT, pointers OFFSET:UNITS; an optional "xN"
|
||||
std::string name, shader, weightFile, argsFile, textureList, surfaceList, bufferList, pointerList, option; uint32_t subgroup = 32;
|
||||
f >> name >> shader >> subgroup >> weightFile >> argsFile >> textureList >> surfaceList >> bufferList >> pointerList >> option;
|
||||
f >> name >> shader >> subgroup >> weightFile >> argsFile >> textureList >> surfaceList >> bufferList >> pointerList;
|
||||
unsigned stage = 0;
|
||||
while (stage < R2Stages && name != R2Names[stage]) ++stage;
|
||||
if (stage >= R2Stages || seen[stage] || pointerList.empty())
|
||||
throw std::runtime_error("rr.txt: unknown, repeated or invalid stage: " + line);
|
||||
seen[stage] = true;
|
||||
auto& st = stages[stage];
|
||||
if (option.size() == 2 && option[0] == 'x' && option[1] >= '1' && option[1] <= '4') st.group = static_cast<uint32_t>(option[1] - '0');
|
||||
if (option.size() > 1 && option[0] == 'r') st.record = static_cast<uint32_t>(std::stoul(option.substr(1)));
|
||||
while (f >> option) {
|
||||
if (option.size() == 2 && option[0] == 'x' && option[1] >= '1' && option[1] <= '4') st.group = static_cast<uint32_t>(option[1] - '0');
|
||||
if (option.size() > 1 && option[0] == 'r') st.record = static_cast<uint32_t>(std::stoul(option.substr(1)));
|
||||
if (option == "s3") st.ranges = true;
|
||||
}
|
||||
for (const auto& item : split(textureList)) st.textures.push_back(static_cast<uint32_t>(std::stoul(item)));
|
||||
for (const auto& item : split(bufferList)) st.buffers.push_back(static_cast<uint32_t>(std::stoul(item)));
|
||||
for (const auto& item : split(surfaceList)) {
|
||||
@@ -92,22 +97,21 @@ struct R2Model {
|
||||
types.insert(types.end(), st.buffers.size(), b);
|
||||
st.pipeline.create(d.handle, directory + "/" + shader, types, subgroup, cache);
|
||||
{ const auto bytes = read(directory + "/" + argsFile); st.args.assign(bytes.begin(), bytes.end()); }
|
||||
blob(st.weights, st.upload, weightFile == "-" ? std::vector<char>(16) : read(directory + "/" + weightFile));
|
||||
blob(st.weights, st.pending, weightFile == "-" ? std::vector<char>(16) : read(directory + "/" + weightFile));
|
||||
}
|
||||
for (unsigned stage = 0; stage < R2DownP; ++stage)
|
||||
if (!seen[stage]) throw std::runtime_error("rr.txt: incomplete Ray Reconstruction preset F model");
|
||||
particles = seen[R2DownP];
|
||||
gather = seen[R2Gather] && stages[R2Gather].record != 0 && stages[R2Enc0].args.size() >= 8;
|
||||
for (unsigned stage : {R2DownP, R2Gather})
|
||||
if (!seen[stage]) blob(stages[stage].weights, stages[stage].upload, std::vector<char>(16));
|
||||
blob(aux, auxUpload, read(directory + "/enc0-aux.bin"));
|
||||
if (!seen[stage]) blob(stages[stage].weights, stages[stage].pending, std::vector<char>(16));
|
||||
blob(aux, auxPending, read(directory + "/enc0-aux.bin"));
|
||||
exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache);
|
||||
exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache);
|
||||
}
|
||||
void recordUpload(VkCommandBuffer cb) {
|
||||
for (auto& st : stages) { VkBufferCopy copy{0, 0, st.upload.size}; vkCmdCopyBuffer(cb, st.upload.handle, st.weights.handle, 1, ©); }
|
||||
VkBufferCopy copy{0, 0, auxUpload.size};
|
||||
vkCmdCopyBuffer(cb, auxUpload.handle, aux.handle, 1, ©);
|
||||
void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents)
|
||||
for (auto& st : stages) recordContents(cb, st.weights, st.pending);
|
||||
recordContents(cb, aux, auxPending);
|
||||
}
|
||||
};
|
||||
struct R2ExposureParams { int32_t last[2]; uint32_t blocksX, blocks; float invCount; uint32_t reset; };
|
||||
@@ -118,15 +122,17 @@ struct R2Engine {
|
||||
uint32_t rw, rh, ow, oh, w0, h0;
|
||||
uint32_t iw, ih, cw, ch, qw, qh; // the intermediate (1.5x output), its half and quarter
|
||||
uint32_t nx, ny, maxW, maxH; // output tiles and the largest tile rectangle (with its overlap)
|
||||
uint64_t unit, tileA, tileB; // arena unit; the first tile's two regions; a tile slot's bytes
|
||||
uint64_t tileBytes = 0;
|
||||
uint64_t unit, tileB; // arena unit; where a tile slot's second region starts in it
|
||||
uint64_t tileBytes = 0; // a tile slot's bytes
|
||||
std::vector<uint64_t> slotAt; // each tile slot's place in the arena buffer
|
||||
// Tile slots (a pair of regions each) and tiles per group. The post passes of a group have a slot each and run
|
||||
// without waiting for one another, as do the down passes that read them; a group's down passes also overlap the
|
||||
// next group's post passes, which write the other half of the slots (NGX overlaps one post with one down the same
|
||||
// way, on two slots). Serialising all 32 passes of a 4K frame cost 0.17 ms in the gaps (RX 9070 XT, 2026-10-13).
|
||||
static constexpr uint32_t R2TileSlots = 8;
|
||||
uint32_t tileGroup = 1;
|
||||
uint64_t gatherAt = 0; // the gather pass's records (R2Gather): after the tile regions
|
||||
// the gather pass's records (R2Gather): byte address of range 0, its rows of windows, range 1, its rows, range 2
|
||||
std::array<uint32_t, 5> gatherWords{};
|
||||
struct Rect { uint32_t x0, y0, x1, y1; };
|
||||
std::vector<Rect> src, dst; // per tile: intermediate rectangle (with overlap), output rectangle
|
||||
std::vector<VkDeviceSize> downPAt;
|
||||
@@ -134,6 +140,8 @@ struct R2Engine {
|
||||
// the particle layer's output-size history (R2DownP's surface 264, read back as texture 256): created on first use
|
||||
Tex pb[2];
|
||||
bool pbNew = false;
|
||||
// the engine's own output image: only for frames that do not write the caller's texture (created on first use)
|
||||
bool outNew = false;
|
||||
Buffer arena, values, sums, dummy;
|
||||
// every parameter block of the frame in one buffer (one upload per frame): the network stages', per tile the two
|
||||
// output passes', the pooling pass's and the exposure's, each at a 256-byte boundary
|
||||
@@ -256,7 +264,6 @@ struct R2Engine {
|
||||
for (const auto& r : src) { maxW = std::max(maxW, r.x1 - r.x0); maxH = std::max(maxH, r.y1 - r.y0); }
|
||||
unit = static_cast<uint64_t>(w0) * h0 / 32;
|
||||
const uint64_t tilePixels = static_cast<uint64_t>(maxW + 8) * (maxH + 8);
|
||||
tileA = unit * R2ArenaUnits; tileB = tileA + tilePixels * 4;
|
||||
tileBytes = tilePixels * 6;
|
||||
const uint32_t tileCount = nx * ny;
|
||||
const uint32_t slots = std::min(tileCount, R2TileSlots);
|
||||
@@ -266,11 +273,61 @@ struct R2Engine {
|
||||
// debugging: D4R_ENGINE_RR_ARENA=DIR keeps the arena and the exposure values host-visible and writes them at the
|
||||
// next setFrame (DIR/F-arena.bin, F-values.bin), when the caller has waited for frame F
|
||||
const bool debug = std::getenv("D4R_ENGINE_RR_ARENA") != nullptr;
|
||||
// Every arena region is written and read within one frame, so the tile slots (live in the output stage only) and
|
||||
// the gather records (live from the gather pass to the end of encoder 0) can share the buffer with the network's
|
||||
// regions where those are dead: the tile slots that fit before the network's output (dec0_1's argument 64; the
|
||||
// output stage also reads the arena after it), the records' rows of windows in the two largest gaps between
|
||||
// encoder 0's regions ("s3" models) and from the end of its last region on (a region ends at the next region's
|
||||
// start). Saves 105 MB at 720p -> 4K. D4R_ENGINE_RR_ARENA keeps the regions apart, so that dumps show the
|
||||
// network's regions as NGX leaves them.
|
||||
std::vector<uint32_t> starts{R2ArenaUnits};
|
||||
for (unsigned stage = 0; stage < R2Network; ++stage)
|
||||
for (const auto& [offset, units] : model.stages[stage].pointers) starts.push_back(static_cast<uint32_t>(std::lround(units)));
|
||||
std::sort(starts.begin(), starts.end());
|
||||
auto regionEnd = [&](uint32_t start) { return *std::upper_bound(starts.begin(), starts.end(), start); };
|
||||
uint32_t netOutStart = 0, enc0End = 0;
|
||||
for (const auto& [offset, units] : model.stages[R2Dec0B].pointers)
|
||||
if (offset == 64) netOutStart = static_cast<uint32_t>(std::lround(units));
|
||||
for (unsigned stage : {R2Enc0, R2Gather})
|
||||
for (const auto& [offset, units] : model.stages[stage].pointers) enc0End = std::max(enc0End, regionEnd(static_cast<uint32_t>(std::lround(units))));
|
||||
const uint64_t arenaBytes = unit * R2ArenaUnits;
|
||||
const uint32_t before = debug ? 0 : static_cast<uint32_t>(std::min<uint64_t>(slots, unit * netOutStart / tileBytes));
|
||||
for (uint32_t k = 0; k < slots; ++k) slotAt.push_back(k < before ? tileBytes * k : arenaBytes + tileBytes * (k - before));
|
||||
tileB = tilePixels * 4;
|
||||
const uint64_t tileEnd = arenaBytes + tileBytes * (slots - before);
|
||||
const uint64_t tail = debug ? (tileEnd + 255) & ~uint64_t(255) : (unit * enc0End + 255) & ~uint64_t(255);
|
||||
// the gather pass's records: one per window of level 0, (W0 / 8 + 1) to a row (a shifted level has one more)
|
||||
gatherAt = (tileA + tileBytes * slots + 255) & ~uint64_t(255);
|
||||
const uint64_t gatherBytes = model.gather ? uint64_t(w0 / 8 + 1) * (h0 / 8 + 1) * model.stages[R2Gather].record * 4 : 0;
|
||||
if (gatherAt + gatherBytes > 0xffffffffull) throw std::runtime_error("Ray Reconstruction: the render size is too large for the arena");
|
||||
arena.create(d, gatherAt + gatherBytes, storage, debug);
|
||||
const uint32_t rows = h0 / 8 + 1;
|
||||
const uint64_t rowBytes = model.gather ? uint64_t(w0 / 8 + 1) * model.stages[R2Gather].record * 4 : 1;
|
||||
std::vector<std::pair<uint64_t, uint32_t>> ranges; // (byte address, rows) of the gaps used
|
||||
if (!debug && model.gather && model.stages[R2Gather].ranges) {
|
||||
std::vector<std::pair<uint32_t, uint32_t>> used;
|
||||
for (unsigned stage : {R2Enc0, R2Gather})
|
||||
for (const auto& [offset, units] : model.stages[stage].pointers) {
|
||||
const uint32_t start = static_cast<uint32_t>(std::lround(units));
|
||||
used.push_back({start, regionEnd(start)});
|
||||
}
|
||||
std::sort(used.begin(), used.end());
|
||||
uint32_t at = 0;
|
||||
for (const auto& [start, end] : used) {
|
||||
if (start > at) {
|
||||
const uint64_t from = (unit * at + 255) & ~uint64_t(255), to = unit * start;
|
||||
if (to > from && (to - from) / rowBytes > 0) ranges.push_back({from, static_cast<uint32_t>((to - from) / rowBytes)});
|
||||
}
|
||||
at = std::max(at, end);
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end(), [](const auto& a, const auto& b) { return a.second > b.second; });
|
||||
if (ranges.size() > 2) ranges.resize(2);
|
||||
}
|
||||
uint32_t placed = 0;
|
||||
for (const auto& r : ranges) placed += r.second;
|
||||
ranges.resize(2, {tail, 0});
|
||||
gatherWords = {static_cast<uint32_t>(ranges[0].first), ranges[0].second, static_cast<uint32_t>(ranges[1].first),
|
||||
ranges[1].second, static_cast<uint32_t>(tail)};
|
||||
const uint64_t gatherTail = placed >= rows ? 0 : (rows - placed) * rowBytes;
|
||||
const uint64_t arenaTotal = std::max({tileEnd, model.gather ? tail + gatherTail : 0});
|
||||
if (arenaTotal > 0xffffffffull) throw std::runtime_error("Ray Reconstruction: the render size is too large for the arena");
|
||||
arena.create(d, arenaTotal, storage, debug);
|
||||
values.create(d, 16, storage, debug);
|
||||
dummy.create(d, 4096, storage);
|
||||
exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16);
|
||||
@@ -298,7 +355,6 @@ struct R2Engine {
|
||||
rec.create(d, rw, rh, VK_FORMAT_R8G8B8A8_SNORM);
|
||||
ka.create(d, rw, rh, VK_FORMAT_R8G8B8A8_UNORM); kb.create(d, rw, rh, VK_FORMAT_R8G8B8A8_UNORM); kc.create(d, rw, rh, half4);
|
||||
half.create(d, cw, ch, pooledFormat()); quarter.create(d, qw, qh, pooledFormat());
|
||||
out.create(d, ow, oh, half4);
|
||||
zero.create(d, 1, 1, half4);
|
||||
for (const auto& st : model.stages)
|
||||
for (const auto& [offset, format] : st.surfaces)
|
||||
@@ -320,7 +376,10 @@ struct R2Engine {
|
||||
const float fr = static_cast<float>(rw), fh = static_cast<float>(rh), fiw = static_cast<float>(iw), fih = static_cast<float>(ih);
|
||||
const float fow = static_cast<float>(ow), foh = static_cast<float>(oh);
|
||||
auto& p = blocks[R2Enc0];
|
||||
if (model.gather) put64(p, p.size() - 8, gatherAt); // the word compile_rr2.py appends to the block
|
||||
if (model.gather && model.stages[R2Gather].ranges) // the six words compile_rr2.py appends to the block
|
||||
for (unsigned i = 0; i < 5; ++i) put32(p, p.size() - 24 + 4 * i, gatherWords[i]);
|
||||
else if (model.gather) // an older model: one address, the records in one piece
|
||||
put64(p, p.size() - 8, gatherWords[4]);
|
||||
put32(p, 128, rw); put32(p, 132, rh); put32(p, 136, w0); put32(p, 140, h0);
|
||||
putF(p, 408, fr / fiw); putF(p, 412, fh / fih); putF(p, 416, fiw / fr); putF(p, 420, fih / fh);
|
||||
putF(p, 424, 1.0f / fr); putF(p, 428, 1.0f / fh); putF(p, 432, 1.0f / fiw); putF(p, 436, 1.0f / fih);
|
||||
@@ -336,7 +395,7 @@ struct R2Engine {
|
||||
for (size_t i = 0; i < tiles; ++i) {
|
||||
auto& o = postBlocks[i];
|
||||
put64(o, 0, netOut); put32(o, 8, w0); // the row stride of the network's output: the padded width
|
||||
put64(o, 104, tileA + tileBytes * tileSlot(i)); put64(o, 112, tileB + tileBytes * tileSlot(i));
|
||||
put64(o, 104, slotAt[tileSlot(i)]); put64(o, 112, slotAt[tileSlot(i)] + tileB);
|
||||
putF(o, 132, fr / fiw); putF(o, 136, fh / fih); put32(o, 148, iw); put32(o, 152, ih);
|
||||
putF(o, 160, 1.0f / fiw); putF(o, 164, 1.0f / fih); put32(o, 168, rw); put32(o, 172, rh);
|
||||
put32(o, 176, src[i].x0); put32(o, 180, src[i].y0); put32(o, 184, src[i].x1); put32(o, 188, src[i].y1);
|
||||
@@ -347,7 +406,7 @@ struct R2Engine {
|
||||
if (!wb) continue;
|
||||
auto& w = *wb;
|
||||
put32(w, 0, iw); put32(w, 4, ih); put32(w, 8, ow); put32(w, 12, oh); put32(w, 16, ow); put32(w, 20, oh);
|
||||
put64(w, 24, tileA + tileBytes * tileSlot(i)); put64(w, 32, tileB + tileBytes * tileSlot(i));
|
||||
put64(w, 24, slotAt[tileSlot(i)]); put64(w, 32, slotAt[tileSlot(i)] + tileB);
|
||||
put32(w, 96, rw - 1); put32(w, 100, rh - 1);
|
||||
putF(w, 104, fr / fow); putF(w, 108, fh / foh); putF(w, 112, fow / fiw); putF(w, 116, foh / fih);
|
||||
put32(w, 128, dst[i].x0); put32(w, 132, dst[i].y0); put32(w, 136, dst[i].x1); put32(w, 140, dst[i].y1);
|
||||
@@ -379,13 +438,13 @@ struct R2Engine {
|
||||
list.push_back({"kc", kc.handle, half4, rw, rh});
|
||||
list.push_back({"half", half.handle, pooledFormat(), cw, ch});
|
||||
list.push_back({"quarter", quarter.handle, pooledFormat(), qw, qh});
|
||||
list.push_back({"out", out.handle, half4, ow, oh});
|
||||
if (out.handle) list.push_back({"out", out.handle, half4, ow, oh});
|
||||
return list;
|
||||
}
|
||||
void recordInitialize(VkCommandBuffer cb) {
|
||||
std::vector<VkImage> images;
|
||||
for (auto* group : {&feat, &hist, &e, &mv}) for (auto& t : *group) images.push_back(t.handle);
|
||||
for (auto* t : {&rec, &ka, &kb, &kc, &half, &quarter, &out, &zero}) images.push_back(t->handle);
|
||||
for (auto* t : {&rec, &ka, &kb, &kc, &half, &quarter, &zero}) images.push_back(t->handle);
|
||||
for (auto& [format, t] : spare) images.push_back(t.handle);
|
||||
VkClearColorValue clear{};
|
||||
VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
@@ -483,6 +542,10 @@ struct R2Engine {
|
||||
pbNew = true;
|
||||
}
|
||||
}
|
||||
if (!finalOutput.view && !out.handle) {
|
||||
out.create(d, ow, oh, VK_FORMAT_R16G16B16A16_SFLOAT);
|
||||
outNew = true;
|
||||
}
|
||||
exposure.reset = rst ? 1u : 0u;
|
||||
reset = rst;
|
||||
auto same = [&](const Bindings& b) {
|
||||
@@ -540,10 +603,11 @@ struct R2Engine {
|
||||
bound = &*found;
|
||||
// texture and surface handles: 1 where something is bound, 0 where NGX leaves the argument null (the kernels
|
||||
// test optional inputs' handles); buffer pointers other than the arena's are zero
|
||||
const VkImageView output = finalOutput.view ? finalOutput.view : out.view;
|
||||
auto handles = [&](unsigned stage, std::vector<uint8_t>& block) {
|
||||
const auto& st = model.stages[stage];
|
||||
for (uint32_t offset : st.textures) put64(block, offset, resolve(stage, false, offset, 0, images, out.view).view ? 1u : 0u);
|
||||
for (const auto& [offset, format] : st.surfaces) put64(block, offset, resolve(stage, true, offset, 0, images, out.view).view ? 1u : 0u);
|
||||
for (uint32_t offset : st.textures) put64(block, offset, resolve(stage, false, offset, 0, images, output).view ? 1u : 0u);
|
||||
for (const auto& [offset, format] : st.surfaces) put64(block, offset, resolve(stage, true, offset, 0, images, output).view ? 1u : 0u);
|
||||
for (uint32_t offset : st.buffers) put64(block, offset, 0);
|
||||
};
|
||||
for (unsigned stage = 0; stage < R2Network; ++stage) handles(stage, blocks[stage]);
|
||||
@@ -570,17 +634,20 @@ struct R2Engine {
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
|
||||
VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
};
|
||||
if (pbNew) { // the particle history starts from zero, like the rest of the temporal state
|
||||
if (pbNew || outNew) { // images created on first use start from zero, like the rest of the temporal state
|
||||
VkClearColorValue clear{};
|
||||
VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1};
|
||||
for (auto& t : pb) {
|
||||
imageBarrier(cb, t.handle, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
|
||||
std::vector<VkImage> fresh;
|
||||
if (pbNew) for (auto& t : pb) fresh.push_back(t.handle);
|
||||
if (outNew) fresh.push_back(out.handle);
|
||||
for (VkImage image : fresh) {
|
||||
imageBarrier(cb, image, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
|
||||
VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
vkCmdClearColorImage(cb, t.handle, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
|
||||
vkCmdClearColorImage(cb, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range);
|
||||
}
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||
VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||
pbNew = false;
|
||||
pbNew = outNew = false;
|
||||
}
|
||||
stamp(0);
|
||||
memoryBarrier(cb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT);
|
||||
|
||||
Reference in new issue
Block a user