shim: adaptive marker polling, opt-in GPU-side input wait/completion and RGBA8 output
- [Interop] MarkerPollUs = auto is the new default: the input-marker wait learns when each feature's inputs become ready, sleeps in <=200 us chunks, yields near the prediction and backs off when late (d4r_input_wait.h); a number keeps fixed pacing (0 yields). d4r_config.py no longer exports 200 when the key is unset - D4R_SHIM_GPU_INPUT_WAIT=1 (experimental, split VRAM, age 0): NGX's null stream waits on a coherent Vulkan word imported into HIP (d4rStreamWaitValue32Eq) instead of CPU polling. The original readback marker stays the authority; a CPU watchdog releases discarded lists and timeouts. Denoiser, verify, capture and dilation frames keep the CPU wait - D4R_SHIM_GPU_COMPLETION=1 (needs GPU_INPUT_WAIT): the bridge writes a completion word right after the native output kernel (d4rNgxWriteValue32), and the game's queue polls it with a bounded compute wait (kernels/support/vulkan_wait.comp), replacing the split list's CPU timeline signal. frame_retired writes the same word on every failure path - D4R_SHIM_NATIVE_COMPLETION=1: a null-stream callback in the bridge signals the split timeline (d4rNgxTimeline*); D4rFrameCompletion::can_signal only releases the oldest outstanding frame - RGBA8 output: tex_common.h adds an 'R4' redirect tag, so K's native output kernel stores packed RGBA8 directly (D4R_SHIM_OUTPUT_RGBA8=1, checked against the kernel manifest by d4rOutputKernelRgba8). Host packing handles RGBA8 too. patches/vkd3d-proton/0004 (VKD3D_D4R_LINEAR_RGBA8=WxH) makes such outputs linear and exportable so the result can be written in place - D4R_SHIM_FUSE_RGBA8_INPUT=1: one Vulkan pass converts and copies RGBA8 color input (vulkan_read.comp mode 5 matches the blit's toward-zero halves, plus a row stride) - profiling: thread CPU time from the Unix bridge (Wine's GetThreadTimes ticks at 10 ms); game-queue D3D12 timestamps (vk_input/vk_dlss_wait/vk_output/vk_total); gpu_eval split into input_gpu_wait and dlss_gpu; D4R_PROFILE_GPU_EVENTS, _TIMESTAMPS, _LOG and _CPU_TIME=0 switch off individual parts for bisection - bridge: D4R_NGX_STREAM_PRIORITY=high|normal runs NGX on a priority stream (off by default; no gain measured) - release_feature: a feature already marked retiring by a failed drain can still be released normally - harness: RGBA8 with motion and jitter scenes (re-quantized each frame), D4R_HARNESS_RGBA8_RAMP, D4R_HARNESS_THRASH_MB to evict L2/Infinity Cache before each evaluation - kernels/native: in_hoist.py and split_output_paths.py (opt-in scheduling experiments); scripts/build_k_ldr_output.sh builds K's LDR output with an optional D4R_LDR_OUTPUT_MAX_VGPR cap - tests: test_input_wait.py and test_latency_optimizations.py; MarkerPollUs and frame-completion tests updated
This commit is contained in:
ayo committed
1 parent
f0b55a850a
commit
162bbd77c8
22 files changed
+1989
-109
No files matched your search
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue independent texture fetches of the translated hiluma input kernel earlier (opt-in experiment).
|
||||
|
||||
usage: in_hoist.py KERNEL.hip ANCHOR_REG HANDLE_REG [HANDLE_REG ...] > OUT.hip
|
||||
|
||||
The input kernel (ptx2hip --ifconv --sink + in_lds2reg.py) is one straight-line body. Its 3x3 color neighbourhood
|
||||
fetches depend only on the thread's pixel position and kernel parameters, yet the compiler issues them after the
|
||||
depth -> motion -> history chain, adding serial memory round trips to a latency-bound kernel. This pass moves every
|
||||
fetch block `{ ::f4 t_ = tex_fetch(rdX, ...); ... }` whose handle register is one of HANDLE_REG's copies, together
|
||||
with the backward slice of statements computing its operands, to right after the statement that assigns
|
||||
ANCHOR_REG, followed by a scheduling barrier (instruction order only). Nothing else moves and no statement
|
||||
changes, so every value is computed exactly as before.
|
||||
|
||||
Refuses unless: every register the moved statements assign is assigned exactly once in the kernel; every register
|
||||
they read is assigned before the anchor (or by another moved statement, earlier in order); there is no memory store
|
||||
and no control flow (label/goto/return) between the anchor and the last moved statement.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
REG = re.compile(r'\b(?:rd|rs|r|f|p|shv\d+_|dpv\d+_)\d+\b|\bl\d+_\w+\b')
|
||||
ASSIGN = re.compile(r'^\s*([A-Za-z_]\w*)\s*=(?!=)')
|
||||
|
||||
|
||||
def stmts(body_lines):
|
||||
"""group lines into statements: fetch blocks '{ ... }' become one statement"""
|
||||
out, i = [], 0
|
||||
while i < len(body_lines):
|
||||
line = body_lines[i]
|
||||
if line.strip().startswith('{ ::f4 t_ ='):
|
||||
j = i
|
||||
while body_lines[j].strip() != '}':
|
||||
j += 1
|
||||
out.append(body_lines[i:j + 1])
|
||||
i = j + 1
|
||||
else:
|
||||
out.append([line])
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def defs_uses(st):
|
||||
text = ''.join(st)
|
||||
if text.strip().startswith('{ ::f4 t_ ='):
|
||||
defs = set(re.findall(r'^\s*([A-Za-z_]\w*) = t_\[\d\];', text, re.M))
|
||||
call = text.split(';', 1)[0].replace('::f4', '')
|
||||
return defs, set(REG.findall(call))
|
||||
m = ASSIGN.match(text)
|
||||
if m:
|
||||
lhs = m.group(1)
|
||||
rhs = text[m.end():]
|
||||
return {lhs}, set(REG.findall(rhs))
|
||||
return set(), set(REG.findall(text))
|
||||
|
||||
|
||||
def main():
|
||||
src = open(sys.argv[1]).read().split('\n')
|
||||
anchor, handles = sys.argv[2], set(sys.argv[3:])
|
||||
start = next(i for i, l in enumerate(src) if l.startswith('extern "C" __attribute__((global))')) + 2
|
||||
end = max(i for i, l in enumerate(src) if l.strip() == '}')
|
||||
S = stmts([l + '\n' for l in src[start:end]])
|
||||
info = [defs_uses(s) for s in S]
|
||||
assigned = {}
|
||||
for k, (d, _) in enumerate(info):
|
||||
for r in d:
|
||||
assigned.setdefault(r, []).append(k)
|
||||
# handle copies: rdY = (uint64_t)(rdX) chains from the given handle registers
|
||||
alias = set(handles)
|
||||
for k, s in enumerate(S):
|
||||
m = re.match(r'^\s*(rd\d+) = \(uint64_t\)\((rd\d+)\);$', s[0])
|
||||
if m and m.group(2) in alias:
|
||||
alias.add(m.group(1))
|
||||
ia = [k for k, (d, _) in enumerate(info) if anchor in d]
|
||||
if len(ia) != 1:
|
||||
raise SystemExit(f'anchor {anchor} assigned {len(ia)} times')
|
||||
ia = ia[0]
|
||||
fetches = [k for k, s in enumerate(S) if s[0].strip().startswith('{ ::f4 t_ = tex_fetch(')
|
||||
and re.search(r'tex_fetch\((rd\d+)', s[0]).group(1) in alias and k > ia]
|
||||
if not fetches:
|
||||
raise SystemExit('no fetch to move')
|
||||
need, move = set(), set(fetches)
|
||||
for k in fetches:
|
||||
need |= info[k][1]
|
||||
while need:
|
||||
r = need.pop()
|
||||
ks = assigned.get(r, [])
|
||||
if not ks:
|
||||
continue # parameters / builtins
|
||||
if len(ks) != 1:
|
||||
raise SystemExit(f'register {r} is assigned {len(ks)} times')
|
||||
k = ks[0]
|
||||
if k <= ia or k in move:
|
||||
continue
|
||||
move.add(k)
|
||||
need |= info[k][1]
|
||||
last = max(move)
|
||||
for k in range(ia + 1, last + 1):
|
||||
text = ''.join(S[k])
|
||||
if re.search(r'^\s*\*\(|\bgoto\b|\breturn\b|^L__BB', text, re.M):
|
||||
raise SystemExit(f'store or control flow between anchor and moved code: {text.strip()[:80]}')
|
||||
for k in sorted(move):
|
||||
for r in info[k][0]:
|
||||
if len(assigned[r]) != 1:
|
||||
raise SystemExit(f'moved register {r} assigned more than once')
|
||||
for r in info[k][1]:
|
||||
ks = assigned.get(r, [])
|
||||
if ks and not (ks[0] <= ia or ks[0] in move and ks[0] < k):
|
||||
raise SystemExit(f'moved statement reads {r} defined later')
|
||||
# a scheduling barrier keeps the compiler from sinking the fetches back to their uses (order only)
|
||||
barrier = [' __builtin_amdgcn_sched_barrier(0); // in_hoist: fetches above are issued here\n']
|
||||
body = ''.join(''.join(S[k]) for k in range(0, ia + 1)) + ''.join(''.join(S[k]) for k in sorted(move)) + \
|
||||
''.join(barrier) + ''.join(''.join(S[k]) for k in range(ia + 1, len(S)) if k not in move)
|
||||
sys.stderr.write(f'in_hoist: moved {len(move)} statements ({len(fetches)} fetches) after {anchor}\n')
|
||||
sys.stdout.write('\n'.join(src[:start]) + '\n' + body + '\n'.join(src[end:]))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split a translated hiluma output kernel into its two per-block paths (opt-in experiment).
|
||||
|
||||
usage: split_output_paths.py KERNEL.hip > KERNEL_split.hip
|
||||
|
||||
NVIDIA's hiluma_engine_output_* kernels pick one of two code paths per block: path B (an LDS tile of the
|
||||
block's render footprint) when the footprint is under 24 texels in both axes, which holds for every block at
|
||||
Quality .. Ultra Performance, else path A (direct fetches, DLAA). The choice depends only on the block index and
|
||||
kernel parameters, so it is uniform per block, and blocks share nothing. Path A needs more registers than path B
|
||||
(LDR mvlo: 101 vs 88 VGPRs), and one kernel carries the larger allocation for every block.
|
||||
|
||||
This pass emits the same translated body twice:
|
||||
NAME runs path B and returns where path A would start;
|
||||
NAME_post1 runs path A and returns where path B would start.
|
||||
The ZLUDA/MicroCUDA native hook launches NAME_post1 right after NAME on the same stream with the same parameters
|
||||
and grid (d4r_post1_grid_x/_y = 0), with the 16x16 blocks the PTX hard-codes (tid.y << 4). Each block executes
|
||||
exactly the instructions it executed before; only the block order across the two paths changes, which no output
|
||||
depends on (blocks never read what other blocks write in the same launch).
|
||||
|
||||
The pass refuses anything but the expected entry structure:
|
||||
pA = ((int32_t)rX < (int32_t)24); pB = ((int32_t)rY < (int32_t)24); pC = (pA && pB);
|
||||
if (pC) goto LB;
|
||||
goto LA;
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def split(source):
|
||||
head, sep, main = source.partition('extern "C" __attribute__((global))')
|
||||
if not sep or 'extern "C" __attribute__((global))' in main:
|
||||
raise SystemExit('expected exactly one kernel')
|
||||
m = re.match(r'\s*__attribute__\(\(amdgpu_flat_work_group_size\(1, 256\)\)\) void (\w+)\((\w+)\)', main)
|
||||
if not m:
|
||||
raise SystemExit('unexpected kernel signature')
|
||||
name = m.group(1)
|
||||
pattern = re.compile(r'^ (p\d+) = \(\(int32_t\)r\d+ < \(int32_t\)24\);\n'
|
||||
r' (p\d+) = \(\(int32_t\)r\d+ < \(int32_t\)24\);\n'
|
||||
r' (p\d+) = \((p\d+) && (p\d+)\);\n'
|
||||
r' if \((p\d+)\) goto (L__BB0_\d+);\n'
|
||||
r' goto (L__BB0_\d+);\n', re.M)
|
||||
found = list(pattern.finditer(main))
|
||||
if len(found) != 1:
|
||||
raise SystemExit(f'path selection found {len(found)} times, expected once')
|
||||
g = found[0]
|
||||
pa, pb, pc, x, y, pc2, path_b, path_a = g.groups()
|
||||
if {x, y} != {pa, pb} or pc != pc2:
|
||||
raise SystemExit('path selection predicate has an unexpected form')
|
||||
# the predicate must not be reassigned anywhere (it is the block's path choice)
|
||||
if len(re.findall(rf'^ {pc} = ', main, re.M)) != 1:
|
||||
raise SystemExit(f'{pc} is assigned more than once')
|
||||
sel = g.group(0)
|
||||
lines = sel.splitlines(keepends=True)
|
||||
main_b = main.replace(sel, ''.join(lines[:4]) + ' return; // path A blocks: NAME_post1\n', 1)
|
||||
main_a = main.replace(sel, ''.join(lines[:3]) + f' if ({pc}) return; // path B blocks: NAME\n'
|
||||
+ f' goto {path_a};\n', 1)
|
||||
main_a = main_a.replace(f' void {name}(', f' void {name}_post1(', 1)
|
||||
globals_ = ('\n// d4r native hook: launch NAME_post1 after NAME with the launch grid and the PTX\'s 16x16 blocks\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_grid_x = 0;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_grid_y = 0;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_x = 16;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_y = 16;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_z = 1;\n')
|
||||
banner = '// split_output_paths.py: NAME = path B only, NAME_post1 = path A only (per-block choice unchanged)\n'
|
||||
return (banner + head + globals_ + sep + main_b + '\n' + sep + main_a, path_b, path_a)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
text, b, a = split(open(sys.argv[1]).read())
|
||||
sys.stderr.write(f'split: path B {b}, path A {a}\n')
|
||||
sys.stdout.write(text)
|
||||
@@ -2,7 +2,7 @@
|
||||
layout(local_size_x=8, local_size_y=8) in;
|
||||
layout(set=0, binding=0) uniform sampler2D sourceImage;
|
||||
layout(std430, set=0, binding=1) writeonly buffer Pixels { uint pixels[]; };
|
||||
layout(push_constant) uniform Size { uint width; uint height; uint mode; } size;
|
||||
layout(push_constant) uniform Size { uint width; uint height; uint mode; uint stride; } size;
|
||||
// Match the CPU canonical conversion, including round-to-even and subnormals.
|
||||
uint halfBits(float value) {
|
||||
uint bits = floatBitsToUint(value);
|
||||
@@ -26,14 +26,24 @@ uint halfBits(float value) {
|
||||
if (remainder > halfway || (remainder == halfway && (m & 1u) != 0u)) ++result;
|
||||
return result;
|
||||
}
|
||||
// Mode 5 is only for RGBA8 UNORM: values are zero or normal positive halves. Preserve
|
||||
// the blit's toward-zero conversion; other modes keep the canonical round-to-even path.
|
||||
uint halfRtz(float value) {
|
||||
uint b = floatBitsToUint(value);
|
||||
if (value == 0.0) return 0u;
|
||||
return (((b >> 23) - 112u) << 10) | ((b & 0x7fffffu) >> 13);
|
||||
}
|
||||
void main() {
|
||||
uvec2 p = gl_GlobalInvocationID.xy;
|
||||
if (p.x >= size.width || p.y >= size.height) return;
|
||||
ivec2 extent = textureSize(sourceImage, 0);
|
||||
uint index = p.y * size.width + p.x;
|
||||
uint index = p.y * size.stride + p.x;
|
||||
vec4 value = all(lessThan(ivec2(p), extent))
|
||||
? texelFetch(sourceImage, ivec2(p), 0) : vec4(0.0);
|
||||
if (size.mode == 1u) { // canonical RGBA16F color
|
||||
if (size.mode == 5u) {
|
||||
pixels[index * 2u] = halfRtz(value.x) | (halfRtz(value.y) << 16);
|
||||
pixels[index * 2u + 1u] = halfRtz(value.z) | (halfRtz(value.w) << 16);
|
||||
} else if (size.mode == 1u) { // canonical RGBA16F color
|
||||
pixels[index * 2u] = halfBits(value.x) | (halfBits(value.y) << 16);
|
||||
pixels[index * 2u + 1u] = halfBits(value.z) | (halfBits(value.w) << 16);
|
||||
} else if (size.mode == 3u) { // canonical RG16F motion
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
#version 450
|
||||
// D4R_SHIM_GPU_COMPLETION: holds the game's queue until DLSS has written this frame's completion word (HIP
|
||||
// hipStreamWriteValue32 after the output, or the CPU on every failure path). Bounded: after `limit` polls the
|
||||
// shader returns regardless, so a lost write can never hang the GPU (the frame then shows its previous output).
|
||||
layout(local_size_x = 1) in;
|
||||
layout(std430, set = 0, binding = 0) coherent volatile readonly buffer Words { uint words[]; };
|
||||
layout(push_constant) uniform Wait { uint index; uint value; uint limit; } wait;
|
||||
void main() {
|
||||
for (uint poll = 0u; poll < wait.limit; ++poll)
|
||||
if (words[wait.index] == wait.value)
|
||||
return;
|
||||
}
|
||||
@@ -371,15 +371,19 @@ extern "C" __attribute__((device)) int __ockl_image_width_2D(tsharp_t* image);
|
||||
extern "C" __attribute__((device)) int __ockl_image_height_2D(tsharp_t* image);
|
||||
__attribute__((device)) static inline uint32_t redirect_pitch(uint64_t surface)
|
||||
{
|
||||
// 'R2' (RGBA16F texels) or 'R3' (R10G10B10A2_UNORM texels) tag in the high half (other handles may have
|
||||
// 'R2' (RGBA16F), 'R3' (RGB10A2) or 'R4' (RGBA8) tag in the high half (other handles may have
|
||||
// data there), row pitch / 8 in the low half
|
||||
const uint32_t word = ((const __attribute__((address_space(4))) uint32_t*)surface)[21];
|
||||
return (word >> 16) == 0x5232u || (word >> 16) == 0x5233u ? (word & 0xffffu) << 3 : 0u;
|
||||
return (word >> 16) >= 0x5232u && (word >> 16) <= 0x5234u ? (word & 0xffffu) << 3 : 0u;
|
||||
}
|
||||
__attribute__((device)) static inline bool redirect_rgb10a2(uint64_t surface)
|
||||
{
|
||||
return (((const __attribute__((address_space(4))) uint32_t*)surface)[21] >> 16) == 0x5233u;
|
||||
}
|
||||
__attribute__((device)) static inline bool redirect_rgba8(uint64_t surface)
|
||||
{
|
||||
return (((const __attribute__((address_space(4))) uint32_t*)surface)[21] >> 16) == 0x5234u;
|
||||
}
|
||||
// One channel as the game-side path produces it: the f32 value rounded toward zero to a half (the RGBA16F
|
||||
// store), then converted to UNORM by a blit (NaN and negatives 0, clamped at 1, rounded to nearest).
|
||||
__attribute__((device)) static inline uint32_t redirect_unorm(uint16_t half, float scale)
|
||||
@@ -422,6 +426,10 @@ DEV __attribute__((always_inline)) void d4r_sust_p_v4b32(uint64_t surface, int32
|
||||
*D4R_REDIRECT_PTR(uint32_t, redirect_row(surface, y) + 4 * x) =
|
||||
redirect_unorm((uint16_t)lo, 1023.0f) | redirect_unorm((uint16_t)(lo >> 16), 1023.0f) << 10 |
|
||||
redirect_unorm((uint16_t)hi, 1023.0f) << 20 | redirect_unorm((uint16_t)(hi >> 16), 3.0f) << 30;
|
||||
else if (redirect_rgba8(surface))
|
||||
*D4R_REDIRECT_PTR(uint32_t, redirect_row(surface, y) + 4 * x) =
|
||||
redirect_unorm((uint16_t)lo, 255.0f) | redirect_unorm((uint16_t)(lo >> 16), 255.0f) << 8 |
|
||||
redirect_unorm((uint16_t)hi, 255.0f) << 16 | redirect_unorm((uint16_t)(hi >> 16), 255.0f) << 24;
|
||||
else
|
||||
*D4R_REDIRECT_PTR(uint2_t, redirect_row(surface, y) + 8 * x) = (uint2_t){lo, hi};
|
||||
}
|
||||
@@ -526,9 +534,15 @@ DEV __attribute__((always_inline)) void d4r_sust_b32(uint64_t surface, int32_t x
|
||||
tsharp_t* image = (tsharp_t*)surface;
|
||||
if (redirect_pitch(surface) != 0)
|
||||
{
|
||||
// Raw RGBA16F stores carry two half channels at each four-byte offset. Packing those channels
|
||||
// separately keeps the original store ownership (no read-modify-write or cross-thread race).
|
||||
if (redirect_rgba8(surface) && x >= 0 && (x & 3) == 0 && redirect_inside(surface, x >> 3, y))
|
||||
*D4R_REDIRECT_PTR(uint16_t, redirect_row(surface, y) + (x >> 1)) =
|
||||
(uint16_t)(redirect_unorm((uint16_t)data, 255.0f) |
|
||||
redirect_unorm((uint16_t)(data >> 16), 255.0f) << 8);
|
||||
// raw bits at byte offset x (RGBA16F rows are 8 bytes per pixel); the shim only asks for packed
|
||||
// RGB10A2 texels from kernels without raw stores
|
||||
if (!redirect_rgb10a2(surface) && x >= 0 && (x & 3) == 0 && redirect_inside(surface, x >> 3, y))
|
||||
// RGB10A2 texels from kernels without raw stores; RGBA8 preserves each half-pair store
|
||||
if (!redirect_rgb10a2(surface) && !redirect_rgba8(surface) && x >= 0 && (x & 3) == 0 && redirect_inside(surface, x >> 3, y))
|
||||
*D4R_REDIRECT_PTR(uint32_t, redirect_row(surface, y) + x) = data;
|
||||
return;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user