engine/rr: preset F's exposure in one pass with 64-bit atomics
exposure_f.comp ran NGX's auto exposure in two passes: the 16x16 blocks wrote their fixed-point luma sums, then a single workgroup added them after a barrier and finished (11 us of a 25 us stage). PASS 2 does what NVIDIA's kernel does: each block adds its sum to a 64-bit total with an atomic add and counts itself in with an acquire-release add; the last block reads the total, finishes and leaves both words zero for the next frame. The total is an integer, the same in any order, so the exposure image and the values are the two passes' bit for bit. compile_rr2.py builds exposure_f2.spv next to the two passes' shaders (the block sum and the finish are shared functions now; their arithmetic is unchanged). 64-bit buffer atomics are a device feature (shaderBufferInt64Atomics), so RModel takes whether the device has it enabled (default no) and preset F uses the one pass only then and only with a model that has exposure_f2.spv; otherwise the two passes as before. vkd3d-proton creates its device with every feature the GPU reports except a list it turns off, which does not include this one (it backs D3D12's 64-bit atomics with it); the shim reads the device's features through the interop interface (GetDeviceFeatures) and logs the result. d4r-rr enables every feature the GPU has, and D4R_RR_TEST_NO_INT64_ATOMICS=1 turns this one off to run the two passes. Ray Reconstruction never runs on the native Vulkan frontend's device. Every image of every frame byte-identical to the previous build (1280x720 -> 1440p, 1114x626 -> 1080p, 960x540 -> 1080p, the stress inputs, with D4R_RR_TEST_ROBUST2=1, at levels 0 and 3, with D4R_RR2_SPLIT=1), and so is the two-pass fallback of the new build; presets D and E unchanged. Controls: the last block finishing with twice the total changes 64 of the 90 images of the 1280x720 sequence (and nothing under the fallback), finishing without its own block's sum 58 of 90. RX 9070 XT, d4r-rr 720p -> 1440p, interleaved: exposure 25.2 -> 14.9 us, frame 3.536 -> 3.526 ms (the fallback 25.1 us). Signed-off-by: itchyfeetleech <[email protected]>
This commit is contained in:
1 parent
f70adc8202
commit
6c7f881d9c
8 files changed
+131
-52
No files matched your search
@@ -12,7 +12,7 @@ D4R_CUDA_REPLAY_DUMP_FILTER=rr2 D4R_CUDA_LAUNCH_STATS_FILTER=rr2 and preset hint
|
|||||||
table of its PTX module; enc0-aux.bin is encoder 0's second table (argument 104)
|
table of its PTX module; enc0-aux.bin is encoder 0's second table (argument 104)
|
||||||
<stage>.args the argument block of the capture's frame, pointers and handles cleared: the runtime keeps its
|
<stage>.args the argument block of the capture's frame, pointers and handles cleared: the runtime keeps its
|
||||||
constants and fills what depends on the frame or the geometry (engine/runtime_rr2.inc)
|
constants and fills what depends on the frame or the geometry (engine/runtime_rr2.inc)
|
||||||
exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp
|
exposure_f0.spv, exposure_f1.spv engine/rr/exposure_f.comp (exposure_f2.spv: both in one, 64-bit atomics)
|
||||||
rr.txt the manifest RModel reads ("D4RR0002")
|
rr.txt the manifest RModel reads ("D4RR0002")
|
||||||
With --native, the stages engine/rr/rr2_m.comp implements (enc1 ... enc5, dec4 ... dec1, dec0's first kernel) are that
|
With --native, the stages engine/rr/rr2_m.comp implements (enc1 ... enc5, dec4 ... dec1, dec0's first kernel) are that
|
||||||
shader with weights read from the DLL itself (engine/rr/rr2_pack.py) instead of the translation: bit-identical to
|
shader with weights read from the DLL itself (engine/rr/rr2_pack.py) instead of the translation: bit-identical to
|
||||||
@@ -249,7 +249,7 @@ def main():
|
|||||||
if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges
|
if hybrid and split: # the same block, bindings and regions; "rN": N words per window record, "s3": in three ranges
|
||||||
lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3')
|
lines.append(f'stage enc0g enc0g.spv 32 - {stage}.args {tex} {surf} {buf} {",".join(ptrs) or "-"} r{record} s3')
|
||||||
print(f'{stage}: {kernel}, {len(args)} argument bytes' + (' (native block)' if hybrid else ''))
|
print(f'{stage}: {kernel}, {len(args)} argument bytes' + (' (native block)' if hybrid else ''))
|
||||||
for p in (0, 1):
|
for p in (0, 1, 2):
|
||||||
subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}',
|
subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', f'-DPASS={p}',
|
||||||
str(HERE / 'rr' / 'exposure_f.comp'), '-o', str(out / f'exposure_f{p}.spv')],
|
str(HERE / 'rr' / 'exposure_f.comp'), '-o', str(out / f'exposure_f{p}.spv')],
|
||||||
check=True, stdout=subprocess.DEVNULL)
|
check=True, stdout=subprocess.DEVNULL)
|
||||||
|
|||||||
@@ -158,6 +158,8 @@ int main(int argc, char** argv) try {
|
|||||||
if (std::getenv("D4R_RR_TEST_NO_ROBUST")) { f2.features.robustBufferAccess = VK_FALSE; v13.robustImageAccess = VK_FALSE; }
|
if (std::getenv("D4R_RR_TEST_NO_ROBUST")) { f2.features.robustBufferAccess = VK_FALSE; v13.robustImageAccess = VK_FALSE; }
|
||||||
// D4R_RR_TEST_NO_COOPMAT=1: a device like vkd3d-proton's, without the cooperative-matrix and float8 features
|
// D4R_RR_TEST_NO_COOPMAT=1: a device like vkd3d-proton's, without the cooperative-matrix and float8 features
|
||||||
if (std::getenv("D4R_RR_TEST_NO_COOPMAT")) { di.enabledExtensionCount = 0; v13.pNext = nullptr; }
|
if (std::getenv("D4R_RR_TEST_NO_COOPMAT")) { di.enabledExtensionCount = 0; v13.pNext = nullptr; }
|
||||||
|
// D4R_RR_TEST_NO_INT64_ATOMICS=1: no 64-bit buffer atomics (preset F's exposure then takes its two passes)
|
||||||
|
if (std::getenv("D4R_RR_TEST_NO_INT64_ATOMICS")) v12.shaderBufferInt64Atomics = v12.shaderSharedInt64Atomics = VK_FALSE;
|
||||||
CK(vkCreateDevice(c.physical, &di, nullptr, &c.device));
|
CK(vkCreateDevice(c.physical, &di, nullptr, &c.device));
|
||||||
vkGetDeviceQueue(c.device, family, 0, &c.queue);
|
vkGetDeviceQueue(c.device, family, 0, &c.queue);
|
||||||
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pi.queueFamilyIndex = family;
|
VkCommandPoolCreateInfo pi{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pi.queueFamilyIndex = family;
|
||||||
@@ -169,7 +171,7 @@ int main(int argc, char** argv) try {
|
|||||||
std::string key;
|
std::string key;
|
||||||
uint32_t rw, rh, ow, oh;
|
uint32_t rw, rh, ow, oh;
|
||||||
frames >> key >> rw >> rh >> ow >> oh;
|
frames >> key >> rw >> rh >> ow >> oh;
|
||||||
d4r::RModel model(c.physical, c.device, modelDir);
|
d4r::RModel model(c.physical, c.device, modelDir, VK_NULL_HANDLE, v12.shaderBufferInt64Atomics == VK_TRUE);
|
||||||
d4r::REngine engine(model, rw, rh, ow, oh);
|
d4r::REngine engine(model, rw, rh, ow, oh);
|
||||||
const char* names[] = {"color", "motion", "depth", "normals", "roughness", "diffuse", "specular"};
|
const char* names[] = {"color", "motion", "depth", "normals", "roughness", "diffuse", "specular"};
|
||||||
const VkFormat formats[] = {VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R32_SFLOAT,
|
const VkFormat formats[] = {VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_R32_SFLOAT,
|
||||||
|
|||||||
+66
-27
@@ -1,33 +1,39 @@
|
|||||||
#version 460
|
#version 460
|
||||||
// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel, in two passes.
|
// Ray Reconstruction preset F (rr2_*): the auto exposure of rr2_histogram_auto_exposure_basic_kernel.
|
||||||
//
|
//
|
||||||
// NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block
|
// NVIDIA's kernel sums a clamped luma per 16x16 block (a pairwise tree over the block's threads), converts each block
|
||||||
// sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish
|
// sum to fixed point (x 2^32 / 10000, round to nearest) and adds them with a 64-bit atomic; the last block to finish
|
||||||
// turns the total into the exposure. The integer total does not depend on the order of the blocks, so PASS 0 writes
|
// turns the total into the exposure:
|
||||||
// every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last block does:
|
|
||||||
// exposure = 0.23547088 / max(mean luma, 0.005) -> the 1x1 exposure image of this frame
|
// exposure = 0.23547088 / max(mean luma, 0.005) -> the 1x1 exposure image of this frame
|
||||||
// values = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 } -> the 16-byte buffer encoder 0 and the output passes read
|
// values = { e, p, 1 / e, clamp(log2(e / p), -6, 6) / 6 } -> the 16-byte buffer encoder 0 and the output passes read
|
||||||
// with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero).
|
// with e the exposure truncated to binary16 and p the previous frame's (1 where either is zero).
|
||||||
|
// PASS 2 is that kernel (64-bit atomics on a storage buffer: shaderBufferInt64Atomics). Without them the runtime runs
|
||||||
|
// two passes: PASS 0 writes every block's fixed-point sum and PASS 1 (one workgroup) adds them and does what the last
|
||||||
|
// block does. The total is an integer: it does not depend on the order of the blocks, so both give the same values.
|
||||||
#extension GL_EXT_shader_explicit_arithmetic_types : require
|
#extension GL_EXT_shader_explicit_arithmetic_types : require
|
||||||
#extension GL_EXT_control_flow_attributes : require
|
#extension GL_EXT_control_flow_attributes : require
|
||||||
#ifndef PASS
|
#ifndef PASS
|
||||||
#define PASS 0
|
#define PASS 0
|
||||||
#endif
|
#endif
|
||||||
|
#if PASS == 2
|
||||||
|
#extension GL_EXT_shader_atomic_int64 : require
|
||||||
|
#extension GL_KHR_memory_scope_semantics : require
|
||||||
|
#endif
|
||||||
layout(set = 0, binding = 0, std430) readonly buffer Params {
|
layout(set = 0, binding = 0, std430) readonly buffer Params {
|
||||||
ivec2 last; // render size - 1
|
ivec2 last; // render size - 1
|
||||||
uint blocksX, blocks;
|
uint blocksX, blocks;
|
||||||
float invCount; // 10000 / (2^32 * render pixels)
|
float invCount; // 10000 / (2^32 * render pixels)
|
||||||
uint reset; // the kernel's argument 32: no log ratio when set
|
uint reset; // the kernel's argument 32: no log ratio when set
|
||||||
} p;
|
} p;
|
||||||
#if PASS == 0
|
#if PASS != 1
|
||||||
// One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128,
|
// One thread per column of the 16x16 block. NVIDIA's tree over the block's 256 threads (index 16 y + x: i += i + 128,
|
||||||
// then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the
|
// then 64, 32, ... 1) is, per column, ((r0 + r8) + (r4 + r12)) + ((r2 + r10) + (r6 + r14)) of the even rows plus the
|
||||||
// same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1).
|
// same of the odd rows, and then the columns (c + c8, + 4, + 2, + 1).
|
||||||
layout(local_size_x = 16) in;
|
layout(local_size_x = 16) in;
|
||||||
layout(set = 0, binding = 1) uniform sampler2D color;
|
layout(set = 0, binding = 1) uniform sampler2D color;
|
||||||
layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
|
|
||||||
shared float col[16];
|
shared float col[16];
|
||||||
void main()
|
// the block's fixed-point sum, in thread 0
|
||||||
|
uint64_t blockSum()
|
||||||
{
|
{
|
||||||
const uint tx = gl_LocalInvocationID.x;
|
const uint tx = gl_LocalInvocationID.x;
|
||||||
const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16;
|
const int x = int(gl_GlobalInvocationID.x), y0 = int(gl_WorkGroupID.y) * 16;
|
||||||
@@ -49,33 +55,37 @@ void main()
|
|||||||
precise float c = w1 + w0;
|
precise float c = w1 + w0;
|
||||||
col[tx] = c;
|
col[tx] = c;
|
||||||
barrier();
|
barrier();
|
||||||
if (tx == 0u) {
|
if (tx != 0u) return 0ul;
|
||||||
precise float d[8];
|
precise float d[8];
|
||||||
[[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
|
[[unroll]] for (int k = 0; k < 8; ++k) d[k] = col[k + 8] + col[k];
|
||||||
precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
|
precise float e0 = d[4] + d[0], e1 = d[5] + d[1], e2 = d[6] + d[2], e3 = d[7] + d[3];
|
||||||
precise float f0 = e2 + e0, f1 = e3 + e1;
|
precise float f0 = e2 + e0, f1 = e3 + e1;
|
||||||
precise float g = f1 + f0;
|
precise float g = f1 + f0;
|
||||||
sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = uint64_t(roundEven(g * 429496.71875));
|
return uint64_t(roundEven(g * 429496.71875));
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
#if PASS == 0
|
||||||
|
layout(set = 0, binding = 2, std430) writeonly buffer Sums { uint64_t s[]; } sums;
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
const uint64_t s = blockSum();
|
||||||
|
if (gl_LocalInvocationID.x == 0u) sums.s[gl_WorkGroupID.y * p.blocksX + gl_WorkGroupID.x] = s;
|
||||||
}
|
}
|
||||||
#else
|
#else
|
||||||
|
#if PASS == 1
|
||||||
layout(local_size_x = 64) in;
|
layout(local_size_x = 64) in;
|
||||||
shared uint64_t part[64];
|
shared uint64_t part[64];
|
||||||
layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums;
|
layout(set = 0, binding = 1, std430) readonly buffer Sums { uint64_t s[]; } sums;
|
||||||
layout(set = 0, binding = 2, std430) writeonly buffer Values { vec4 v; } values;
|
#else
|
||||||
layout(set = 0, binding = 3) uniform sampler2D previous;
|
// s[0]: the total, s[1]: the blocks done; the last block leaves both 0 for the next frame
|
||||||
layout(set = 0, binding = 4, r16f) uniform writeonly image2D current;
|
layout(set = 0, binding = 2, std430) buffer Sums { uint64_t s[]; } sums;
|
||||||
void main()
|
#endif
|
||||||
|
layout(set = 0, binding = PASS + 1, std430) writeonly buffer Values { vec4 v; } values;
|
||||||
|
layout(set = 0, binding = PASS + 2) uniform sampler2D previous;
|
||||||
|
layout(set = 0, binding = PASS + 3, r16f) uniform writeonly image2D current;
|
||||||
|
// the last block's work, from the frame's total
|
||||||
|
void finish(uint64_t total)
|
||||||
{
|
{
|
||||||
// the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
|
|
||||||
const uint tid = gl_LocalInvocationID.x;
|
|
||||||
uint64_t mine = 0ul;
|
|
||||||
for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
|
|
||||||
part[tid] = mine;
|
|
||||||
barrier();
|
|
||||||
if (tid != 0u) return;
|
|
||||||
uint64_t total = 0ul;
|
|
||||||
for (uint b = 0u; b < 64u; ++b) total += part[b];
|
|
||||||
precise float mean = p.invCount * float(total);
|
precise float mean = p.invCount * float(total);
|
||||||
precise float e = 0.23547088 / max(mean, 0.005);
|
precise float e = 0.23547088 / max(mean, 0.005);
|
||||||
imageStore(current, ivec2(0), vec4(e));
|
imageStore(current, ivec2(0), vec4(e));
|
||||||
@@ -90,4 +100,33 @@ void main()
|
|||||||
if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667;
|
if (p.reset == 0u) ratio = clamp(log2(max(cur, 1e-8) / max(prev, 1e-8)), -6.0, 6.0) * 0.16666667;
|
||||||
values.v = vec4(cur, prev, 1.0 / cur, ratio);
|
values.v = vec4(cur, prev, 1.0 / cur, ratio);
|
||||||
}
|
}
|
||||||
|
#if PASS == 1
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
// the block sums are integers: any order. 64 strided partial sums, then thread 0 finishes
|
||||||
|
const uint tid = gl_LocalInvocationID.x;
|
||||||
|
uint64_t mine = 0ul;
|
||||||
|
for (uint b = tid; b < p.blocks; b += 64u) mine += sums.s[b];
|
||||||
|
part[tid] = mine;
|
||||||
|
barrier();
|
||||||
|
if (tid != 0u) return;
|
||||||
|
uint64_t total = 0ul;
|
||||||
|
for (uint b = 0u; b < 64u; ++b) total += part[b];
|
||||||
|
finish(total);
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
void main()
|
||||||
|
{
|
||||||
|
const uint64_t s = blockSum();
|
||||||
|
if (gl_LocalInvocationID.x != 0u) return;
|
||||||
|
atomicAdd(sums.s[0], s, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
|
||||||
|
// the release orders this block's add before its count, and the last block's acquire every block's add before
|
||||||
|
// its read of the total
|
||||||
|
if (atomicAdd(sums.s[1], 1ul, gl_ScopeDevice, gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease) != uint64_t(p.blocks - 1u))
|
||||||
|
return;
|
||||||
|
const uint64_t total = atomicExchange(sums.s[0], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
|
||||||
|
atomicExchange(sums.s[1], 0ul, gl_ScopeDevice, 0, gl_SemanticsRelaxed);
|
||||||
|
finish(total);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#endif
|
#endif
|
||||||
+4
-1
@@ -273,7 +273,10 @@ struct RFrameSettings {
|
|||||||
};
|
};
|
||||||
class RModel {
|
class RModel {
|
||||||
public:
|
public:
|
||||||
RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE);
|
// int64Atomics: the device has shaderBufferInt64Atomics enabled; preset F then measures the exposure in one pass
|
||||||
|
// (engine/rr/exposure_f.comp PASS 2) instead of two
|
||||||
|
RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache = VK_NULL_HANDLE,
|
||||||
|
bool int64Atomics = false);
|
||||||
~RModel();
|
~RModel();
|
||||||
RModel(const RModel&) = delete;
|
RModel(const RModel&) = delete;
|
||||||
RModel& operator=(const RModel&) = delete;
|
RModel& operator=(const RModel&) = delete;
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ struct RModel::Impl {
|
|||||||
bool uploaded = false;
|
bool uploaded = false;
|
||||||
Impl(VkPhysicalDevice physical, VkDevice dev) : device(physical, dev) {}
|
Impl(VkPhysicalDevice physical, VkDevice dev) : device(physical, dev) {}
|
||||||
};
|
};
|
||||||
RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache)
|
RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& directory, VkPipelineCache cache, bool int64Atomics)
|
||||||
: impl(std::make_unique<Impl>(physical, device)) {
|
: impl(std::make_unique<Impl>(physical, device)) {
|
||||||
auto& m = *impl;
|
auto& m = *impl;
|
||||||
// rr.txt: "D4RR0001", "preset N", one "stage NAME SHADER.spv SUBGROUP WEIGHTS TEXTURES SURFACES" line per stage in
|
// rr.txt: "D4RR0001", "preset N", one "stage NAME SHADER.spv SUBGROUP WEIGHTS TEXTURES SURFACES" line per stage in
|
||||||
@@ -103,7 +103,7 @@ RModel::RModel(VkPhysicalDevice physical, VkDevice device, const std::string& di
|
|||||||
if (line.rfind("D4RR0002", 0) == 0) {
|
if (line.rfind("D4RR0002", 0) == 0) {
|
||||||
m.preset = 6;
|
m.preset = 6;
|
||||||
m.v2 = std::make_unique<R2Model>();
|
m.v2 = std::make_unique<R2Model>();
|
||||||
m.v2->load(m.device, directory, lines, cache);
|
m.v2->load(m.device, directory, lines, cache, int64Atomics);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (line.rfind("D4RR0001", 0) != 0) throw std::runtime_error("not a Ray Reconstruction model (rr.txt D4RR0001)");
|
if (line.rfind("D4RR0001", 0) != 0) throw std::runtime_error("not a Ray Reconstruction model (rr.txt D4RR0001)");
|
||||||
|
|||||||
+38
-17
@@ -44,9 +44,10 @@ struct R2Model {
|
|||||||
Buffer aux; // encoder 0's second table (argument 104)
|
Buffer aux; // encoder 0's second table (argument 104)
|
||||||
std::vector<char> auxPending;
|
std::vector<char> auxPending;
|
||||||
std::array<Pipeline, 2> exposure;
|
std::array<Pipeline, 2> exposure;
|
||||||
|
Pipeline exposureOne; // both exposure passes in one (exposure_f2.spv, 64-bit atomics); else null
|
||||||
bool particles = false; // the model has R2DownP
|
bool particles = false; // the model has R2DownP
|
||||||
bool gather = false; // the model has R2Gather
|
bool gather = false; // the model has R2Gather
|
||||||
void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache) {
|
void load(const Device& d, const std::string& directory, std::istringstream& lines, VkPipelineCache cache, bool int64Atomics) {
|
||||||
const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
const auto b = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, t = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, s = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE;
|
||||||
auto split = [](const std::string& list) {
|
auto split = [](const std::string& list) {
|
||||||
std::vector<std::string> items;
|
std::vector<std::string> items;
|
||||||
@@ -108,6 +109,8 @@ struct R2Model {
|
|||||||
blob(aux, auxPending, read(directory + "/enc0-aux.bin"));
|
blob(aux, auxPending, read(directory + "/enc0-aux.bin"));
|
||||||
exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache);
|
exposure[0].create(d.handle, directory + "/exposure_f0.spv", {b, t, b}, 64, cache);
|
||||||
exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache);
|
exposure[1].create(d.handle, directory + "/exposure_f1.spv", {b, b, b, t, s}, 64, cache);
|
||||||
|
if (int64Atomics && exists(directory + "/exposure_f2.spv"))
|
||||||
|
exposureOne.create(d.handle, directory + "/exposure_f2.spv", {b, t, b, b, t, s}, 64, cache);
|
||||||
}
|
}
|
||||||
void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents)
|
void recordUpload(VkCommandBuffer cb) { // the weights into the command buffer (recordContents)
|
||||||
for (auto& st : stages) recordContents(cb, st.weights, st.pending);
|
for (auto& st : stages) recordContents(cb, st.weights, st.pending);
|
||||||
@@ -172,7 +175,7 @@ struct R2Engine {
|
|||||||
std::vector<VkDescriptorSet> down;
|
std::vector<VkDescriptorSet> down;
|
||||||
std::vector<std::array<VkDescriptorSet, 2>> downP;
|
std::vector<std::array<VkDescriptorSet, 2>> downP;
|
||||||
VkDescriptorSet pooled = VK_NULL_HANDLE, exposure0 = VK_NULL_HANDLE;
|
VkDescriptorSet pooled = VK_NULL_HANDLE, exposure0 = VK_NULL_HANDLE;
|
||||||
std::array<VkDescriptorSet, 2> exposure1{};
|
std::array<VkDescriptorSet, 2> exposure1{}, exposureOne{};
|
||||||
};
|
};
|
||||||
std::vector<Bindings> bindings;
|
std::vector<Bindings> bindings;
|
||||||
const Bindings* bound = nullptr;
|
const Bindings* bound = nullptr;
|
||||||
@@ -337,7 +340,7 @@ struct R2Engine {
|
|||||||
exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16);
|
exposure.blocksX = (rw + 15) / 16; exposure.blocks = exposure.blocksX * ((rh + 15) / 16);
|
||||||
exposure.last[0] = static_cast<int32_t>(rw - 1); exposure.last[1] = static_cast<int32_t>(rh - 1);
|
exposure.last[0] = static_cast<int32_t>(rw - 1); exposure.last[1] = static_cast<int32_t>(rh - 1);
|
||||||
exposure.invCount = static_cast<float>(10000.0 / (4294967296.0 * rw * rh));
|
exposure.invCount = static_cast<float>(10000.0 / (4294967296.0 * rw * rh));
|
||||||
sums.create(d, exposure.blocks * 8, storage);
|
sums.create(d, std::max(exposure.blocks, 2u) * 8, storage); // the one-pass exposure: the total and the count
|
||||||
const size_t tiles = src.size();
|
const size_t tiles = src.size();
|
||||||
auto place = [&](size_t bytes) { const VkDeviceSize at = paramBytes; paramBytes += (bytes + 255) & ~VkDeviceSize(255); return at; };
|
auto place = [&](size_t bytes) { const VkDeviceSize at = paramBytes; paramBytes += (bytes + 255) & ~VkDeviceSize(255); return at; };
|
||||||
for (unsigned i = 0; i < R2Network; ++i) { blocks[i] = model.stages[i].args; paramAt[i] = place(blocks[i].size()); }
|
for (unsigned i = 0; i < R2Network; ++i) { blocks[i] = model.stages[i].args; paramAt[i] = place(blocks[i].size()); }
|
||||||
@@ -593,17 +596,29 @@ struct R2Engine {
|
|||||||
b.downP.push_back({stageSet(b, R2DownP, downPAt[i], 0, output), stageSet(b, R2DownP, downPAt[i], 1, output)});
|
b.downP.push_back({stageSet(b, R2DownP, downPAt[i], 0, output), stageSet(b, R2DownP, downPAt[i], 1, output)});
|
||||||
}
|
}
|
||||||
b.pooled = stageSet(b, R2Pooled, pooledAt, 0, output);
|
b.pooled = stageSet(b, R2Pooled, pooledAt, 0, output);
|
||||||
b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors);
|
if (model.exposureOne.handle)
|
||||||
bindRange(d.handle, b.exposure0, 0, paramAll, expAt);
|
for (uint32_t par = 0; par < 2; ++par) {
|
||||||
bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout);
|
VkDescriptorSet set = b.exposureOne[par] = descriptorSet(d.handle, b.pool, model.exposureOne.descriptors);
|
||||||
bindBuffer(d.handle, b.exposure0, 2, sums);
|
bindRange(d.handle, set, 0, paramAll, expAt);
|
||||||
for (uint32_t par = 0; par < 2; ++par) {
|
bindImage(d.handle, set, 1, images.color, nearest, images.layout);
|
||||||
VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors);
|
bindBuffer(d.handle, set, 2, sums);
|
||||||
bindRange(d.handle, set, 0, paramAll, expAt);
|
bindBuffer(d.handle, set, 3, values);
|
||||||
bindBuffer(d.handle, set, 1, sums);
|
bindImage(d.handle, set, 4, e[par ^ 1].view, nearest);
|
||||||
bindBuffer(d.handle, set, 2, values);
|
bindImage(d.handle, set, 5, e[par].view);
|
||||||
bindImage(d.handle, set, 3, e[par ^ 1].view, nearest);
|
}
|
||||||
bindImage(d.handle, set, 4, e[par].view);
|
else {
|
||||||
|
b.exposure0 = descriptorSet(d.handle, b.pool, model.exposure[0].descriptors);
|
||||||
|
bindRange(d.handle, b.exposure0, 0, paramAll, expAt);
|
||||||
|
bindImage(d.handle, b.exposure0, 1, images.color, nearest, images.layout);
|
||||||
|
bindBuffer(d.handle, b.exposure0, 2, sums);
|
||||||
|
for (uint32_t par = 0; par < 2; ++par) {
|
||||||
|
VkDescriptorSet set = b.exposure1[par] = descriptorSet(d.handle, b.pool, model.exposure[1].descriptors);
|
||||||
|
bindRange(d.handle, set, 0, paramAll, expAt);
|
||||||
|
bindBuffer(d.handle, set, 1, sums);
|
||||||
|
bindBuffer(d.handle, set, 2, values);
|
||||||
|
bindImage(d.handle, set, 3, e[par ^ 1].view, nearest);
|
||||||
|
bindImage(d.handle, set, 4, e[par].view);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
bindings.push_back(std::move(b));
|
bindings.push_back(std::move(b));
|
||||||
found = bindings.end() - 1;
|
found = bindings.end() - 1;
|
||||||
@@ -675,9 +690,15 @@ struct R2Engine {
|
|||||||
vkCmdUpdateBuffer(cb, paramAll.handle, at, std::min<VkDeviceSize>(65536, paramBytes - at), all.data() + at);
|
vkCmdUpdateBuffer(cb, paramAll.handle, at, std::min<VkDeviceSize>(65536, paramBytes - at), all.data() + at);
|
||||||
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
|
memoryBarrier(cb, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT,
|
||||||
VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT);
|
||||||
dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX);
|
// the exposure: one pass with 64-bit atomics (the last block finishes, as NGX's kernel), or the block sums and
|
||||||
step();
|
// one workgroup that adds them
|
||||||
dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1);
|
if (model.exposureOne.handle)
|
||||||
|
dispatch(cb, model.exposureOne, b.exposureOne[cur], exposure.blocksX, exposure.blocks / exposure.blocksX);
|
||||||
|
else {
|
||||||
|
dispatch(cb, model.exposure[0], b.exposure0, exposure.blocksX, exposure.blocks / exposure.blocksX);
|
||||||
|
step();
|
||||||
|
dispatch(cb, model.exposure[1], b.exposure1[cur], 1, 1);
|
||||||
|
}
|
||||||
step();
|
step();
|
||||||
stamp(1);
|
stamp(1);
|
||||||
for (unsigned stage = 0; stage < R2Network; ++stage) {
|
for (unsigned stage = 0; stage < R2Network; ++stage) {
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ if (( DO_RR )); then
|
|||||||
done
|
done
|
||||||
run ptx-f.log python3 "$KIT/kernels/tools/extract_dlss_ptx.py" "$DLL" "$WORK/ptx-f"
|
run ptx-f.log python3 "$KIT/kernels/tools/extract_dlss_ptx.py" "$DLL" "$WORK/ptx-f"
|
||||||
run rr-f.log python3 "$KIT/engine/compile_rr2.py" --dll "$DLL" "$WORK/rr-f" --ptx-dir "$WORK/ptx-f"
|
run rr-f.log python3 "$KIT/engine/compile_rr2.py" --dll "$DLL" "$WORK/rr-f" --ptx-dir "$WORK/ptx-f"
|
||||||
place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv
|
place rr-f "$WORK/rr-f" $(model_files "$WORK/rr-f") enc0-aux.bin exposure_f0.spv exposure_f1.spv exposure_f2.spv
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if (( DO_SR )); then
|
if (( DO_SR )); then
|
||||||
|
|||||||
@@ -3385,6 +3385,9 @@ struct VulkanInterop
|
|||||||
// VK_EXT_shader_float8 (vkd3d-proton enables it when the GPU has it): M's and L's engine layers are FP8
|
// VK_EXT_shader_float8 (vkd3d-proton enables it when the GPU has it): M's and L's engine layers are FP8
|
||||||
// cooperative-matrix shaders, which RADV's compiler crashes on without it (RDNA3)
|
// cooperative-matrix shaders, which RADV's compiler crashes on without it (RDNA3)
|
||||||
bool float8 = false;
|
bool float8 = false;
|
||||||
|
// shaderBufferInt64Atomics is enabled on vkd3d-proton's device (it enables it when the GPU has it): Ray
|
||||||
|
// Reconstruction preset F then measures the exposure in one pass, as NGX does, instead of two
|
||||||
|
bool int64Atomics = false;
|
||||||
// The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects.
|
// The engine runs on the game's own Vulkan device (native Vulkan NGX frontend): no vkd3d interop objects.
|
||||||
bool native = false;
|
bool native = false;
|
||||||
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
|
PFN_vkGetPhysicalDeviceFormatProperties formatProperties = nullptr;
|
||||||
@@ -9322,6 +9325,17 @@ static void init_engine_vulkan()
|
|||||||
}
|
}
|
||||||
logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: "
|
logf("engine backend: VK_EXT_shader_float8 %s", g_vk.float8 ? "available (M and L engine layers)" : "unavailable: "
|
||||||
"M and L run on the CUDA backend");
|
"M and L run on the CUDA backend");
|
||||||
|
// the features vkd3d-proton created its device with
|
||||||
|
const VkPhysicalDeviceFeatures2* features = nullptr;
|
||||||
|
if (SUCCEEDED(g_vk.interop->GetDeviceFeatures(&features)) && features != nullptr)
|
||||||
|
for (auto* next = static_cast<const VkBaseInStructure*>(features->pNext); next != nullptr; next = next->pNext)
|
||||||
|
{
|
||||||
|
if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES)
|
||||||
|
g_vk.int64Atomics = reinterpret_cast<const VkPhysicalDeviceVulkan12Features*>(next)->shaderBufferInt64Atomics == VK_TRUE;
|
||||||
|
if (next->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES)
|
||||||
|
g_vk.int64Atomics = reinterpret_cast<const VkPhysicalDeviceShaderAtomicInt64Features*>(next)->shaderBufferInt64Atomics == VK_TRUE;
|
||||||
|
}
|
||||||
|
logf("engine backend: 64-bit buffer atomics %s", g_vk.int64Atomics ? "enabled" : "not enabled: preset F's exposure in two passes");
|
||||||
}
|
}
|
||||||
|
|
||||||
static void init_engine_backend()
|
static void init_engine_backend()
|
||||||
@@ -10316,7 +10330,7 @@ static void init_engine_backend_rr(int slot)
|
|||||||
const auto start = std::chrono::steady_clock::now();
|
const auto start = std::chrono::steady_clock::now();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
auto model = std::make_unique<d4r::RModel>(g_vk.physical, g_vk.device, path);
|
auto model = std::make_unique<d4r::RModel>(g_vk.physical, g_vk.device, path, VK_NULL_HANDLE, g_vk.int64Atomics);
|
||||||
if (model->preset() != g_rrPreset[slot])
|
if (model->preset() != g_rrPreset[slot])
|
||||||
throw std::runtime_error("the directory holds a model of another preset");
|
throw std::runtime_error("the directory holds a model of another preset");
|
||||||
g_engineR.models[slot] = std::move(model);
|
g_engineR.models[slot] = std::move(model);
|
||||||
|
|||||||
Reference in new issue
Block a user