shim: adaptive marker polling, opt-in GPU-side input wait/completion and RGBA8 output
- [Interop] MarkerPollUs = auto is the new default: the input-marker wait learns when each feature's inputs become ready, sleeps in <=200 us chunks, yields near the prediction and backs off when late (d4r_input_wait.h); a number keeps fixed pacing (0 yields). d4r_config.py no longer exports 200 when the key is unset - D4R_SHIM_GPU_INPUT_WAIT=1 (experimental, split VRAM, age 0): NGX's null stream waits on a coherent Vulkan word imported into HIP (d4rStreamWaitValue32Eq) instead of CPU polling. The original readback marker stays the authority; a CPU watchdog releases discarded lists and timeouts. Denoiser, verify, capture and dilation frames keep the CPU wait - D4R_SHIM_GPU_COMPLETION=1 (needs GPU_INPUT_WAIT): the bridge writes a completion word right after the native output kernel (d4rNgxWriteValue32), and the game's queue polls it with a bounded compute wait (kernels/support/vulkan_wait.comp), replacing the split list's CPU timeline signal. frame_retired writes the same word on every failure path - D4R_SHIM_NATIVE_COMPLETION=1: a null-stream callback in the bridge signals the split timeline (d4rNgxTimeline*); D4rFrameCompletion::can_signal only releases the oldest outstanding frame - RGBA8 output: tex_common.h adds an 'R4' redirect tag, so K's native output kernel stores packed RGBA8 directly (D4R_SHIM_OUTPUT_RGBA8=1, checked against the kernel manifest by d4rOutputKernelRgba8). Host packing handles RGBA8 too. patches/vkd3d-proton/0004 (VKD3D_D4R_LINEAR_RGBA8=WxH) makes such outputs linear and exportable so the result can be written in place - D4R_SHIM_FUSE_RGBA8_INPUT=1: one Vulkan pass converts and copies RGBA8 color input (vulkan_read.comp mode 5 matches the blit's toward-zero halves, plus a row stride) - profiling: thread CPU time from the Unix bridge (Wine's GetThreadTimes ticks at 10 ms); game-queue D3D12 timestamps (vk_input/vk_dlss_wait/vk_output/vk_total); gpu_eval split into input_gpu_wait and dlss_gpu; D4R_PROFILE_GPU_EVENTS, _TIMESTAMPS, _LOG and _CPU_TIME=0 switch off individual parts for bisection - bridge: D4R_NGX_STREAM_PRIORITY=high|normal runs NGX on a priority stream (off by default; no gain measured) - release_feature: a feature already marked retiring by a failed drain can still be released normally - harness: RGBA8 with motion and jitter scenes (re-quantized each frame), D4R_HARNESS_RGBA8_RAMP, D4R_HARNESS_THRASH_MB to evict L2/Infinity Cache before each evaluation - kernels/native: in_hoist.py and split_output_paths.py (opt-in scheduling experiments); scripts/build_k_ldr_output.sh builds K's LDR output with an optional D4R_LDR_OUTPUT_MAX_VGPR cap - tests: test_input_wait.py and test_latency_optimizations.py; MarkerPollUs and frame-completion tests updated
This commit is contained in:
ayo committed
1 parent
f0b55a850a
commit
162bbd77c8
22 files changed
+1989
-109
No files matched your search
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Issue independent texture fetches of the translated hiluma input kernel earlier (opt-in experiment).
|
||||
|
||||
usage: in_hoist.py KERNEL.hip ANCHOR_REG HANDLE_REG [HANDLE_REG ...] > OUT.hip
|
||||
|
||||
The input kernel (ptx2hip --ifconv --sink + in_lds2reg.py) is one straight-line body. Its 3x3 color neighbourhood
|
||||
fetches depend only on the thread's pixel position and kernel parameters, yet the compiler issues them after the
|
||||
depth -> motion -> history chain, adding serial memory round trips to a latency-bound kernel. This pass moves every
|
||||
fetch block `{ ::f4 t_ = tex_fetch(rdX, ...); ... }` whose handle register is one of HANDLE_REG's copies, together
|
||||
with the backward slice of statements computing its operands, to right after the statement that assigns
|
||||
ANCHOR_REG, followed by a scheduling barrier (instruction order only). Nothing else moves and no statement
|
||||
changes, so every value is computed exactly as before.
|
||||
|
||||
Refuses unless: every register the moved statements assign is assigned exactly once in the kernel; every register
|
||||
they read is assigned before the anchor (or by another moved statement, earlier in order); there is no memory store
|
||||
and no control flow (label/goto/return) between the anchor and the last moved statement.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
REG = re.compile(r'\b(?:rd|rs|r|f|p|shv\d+_|dpv\d+_)\d+\b|\bl\d+_\w+\b')
|
||||
ASSIGN = re.compile(r'^\s*([A-Za-z_]\w*)\s*=(?!=)')
|
||||
|
||||
|
||||
def stmts(body_lines):
|
||||
"""group lines into statements: fetch blocks '{ ... }' become one statement"""
|
||||
out, i = [], 0
|
||||
while i < len(body_lines):
|
||||
line = body_lines[i]
|
||||
if line.strip().startswith('{ ::f4 t_ ='):
|
||||
j = i
|
||||
while body_lines[j].strip() != '}':
|
||||
j += 1
|
||||
out.append(body_lines[i:j + 1])
|
||||
i = j + 1
|
||||
else:
|
||||
out.append([line])
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def defs_uses(st):
|
||||
text = ''.join(st)
|
||||
if text.strip().startswith('{ ::f4 t_ ='):
|
||||
defs = set(re.findall(r'^\s*([A-Za-z_]\w*) = t_\[\d\];', text, re.M))
|
||||
call = text.split(';', 1)[0].replace('::f4', '')
|
||||
return defs, set(REG.findall(call))
|
||||
m = ASSIGN.match(text)
|
||||
if m:
|
||||
lhs = m.group(1)
|
||||
rhs = text[m.end():]
|
||||
return {lhs}, set(REG.findall(rhs))
|
||||
return set(), set(REG.findall(text))
|
||||
|
||||
|
||||
def main():
|
||||
src = open(sys.argv[1]).read().split('\n')
|
||||
anchor, handles = sys.argv[2], set(sys.argv[3:])
|
||||
start = next(i for i, l in enumerate(src) if l.startswith('extern "C" __attribute__((global))')) + 2
|
||||
end = max(i for i, l in enumerate(src) if l.strip() == '}')
|
||||
S = stmts([l + '\n' for l in src[start:end]])
|
||||
info = [defs_uses(s) for s in S]
|
||||
assigned = {}
|
||||
for k, (d, _) in enumerate(info):
|
||||
for r in d:
|
||||
assigned.setdefault(r, []).append(k)
|
||||
# handle copies: rdY = (uint64_t)(rdX) chains from the given handle registers
|
||||
alias = set(handles)
|
||||
for k, s in enumerate(S):
|
||||
m = re.match(r'^\s*(rd\d+) = \(uint64_t\)\((rd\d+)\);$', s[0])
|
||||
if m and m.group(2) in alias:
|
||||
alias.add(m.group(1))
|
||||
ia = [k for k, (d, _) in enumerate(info) if anchor in d]
|
||||
if len(ia) != 1:
|
||||
raise SystemExit(f'anchor {anchor} assigned {len(ia)} times')
|
||||
ia = ia[0]
|
||||
fetches = [k for k, s in enumerate(S) if s[0].strip().startswith('{ ::f4 t_ = tex_fetch(')
|
||||
and re.search(r'tex_fetch\((rd\d+)', s[0]).group(1) in alias and k > ia]
|
||||
if not fetches:
|
||||
raise SystemExit('no fetch to move')
|
||||
need, move = set(), set(fetches)
|
||||
for k in fetches:
|
||||
need |= info[k][1]
|
||||
while need:
|
||||
r = need.pop()
|
||||
ks = assigned.get(r, [])
|
||||
if not ks:
|
||||
continue # parameters / builtins
|
||||
if len(ks) != 1:
|
||||
raise SystemExit(f'register {r} is assigned {len(ks)} times')
|
||||
k = ks[0]
|
||||
if k <= ia or k in move:
|
||||
continue
|
||||
move.add(k)
|
||||
need |= info[k][1]
|
||||
last = max(move)
|
||||
for k in range(ia + 1, last + 1):
|
||||
text = ''.join(S[k])
|
||||
if re.search(r'^\s*\*\(|\bgoto\b|\breturn\b|^L__BB', text, re.M):
|
||||
raise SystemExit(f'store or control flow between anchor and moved code: {text.strip()[:80]}')
|
||||
for k in sorted(move):
|
||||
for r in info[k][0]:
|
||||
if len(assigned[r]) != 1:
|
||||
raise SystemExit(f'moved register {r} assigned more than once')
|
||||
for r in info[k][1]:
|
||||
ks = assigned.get(r, [])
|
||||
if ks and not (ks[0] <= ia or ks[0] in move and ks[0] < k):
|
||||
raise SystemExit(f'moved statement reads {r} defined later')
|
||||
# a scheduling barrier keeps the compiler from sinking the fetches back to their uses (order only)
|
||||
barrier = [' __builtin_amdgcn_sched_barrier(0); // in_hoist: fetches above are issued here\n']
|
||||
body = ''.join(''.join(S[k]) for k in range(0, ia + 1)) + ''.join(''.join(S[k]) for k in sorted(move)) + \
|
||||
''.join(barrier) + ''.join(''.join(S[k]) for k in range(ia + 1, len(S)) if k not in move)
|
||||
sys.stderr.write(f'in_hoist: moved {len(move)} statements ({len(fetches)} fetches) after {anchor}\n')
|
||||
sys.stdout.write('\n'.join(src[:start]) + '\n' + body + '\n'.join(src[end:]))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split a translated hiluma output kernel into its two per-block paths (opt-in experiment).
|
||||
|
||||
usage: split_output_paths.py KERNEL.hip > KERNEL_split.hip
|
||||
|
||||
NVIDIA's hiluma_engine_output_* kernels pick one of two code paths per block: path B (an LDS tile of the
|
||||
block's render footprint) when the footprint is under 24 texels in both axes, which holds for every block at
|
||||
Quality .. Ultra Performance, else path A (direct fetches, DLAA). The choice depends only on the block index and
|
||||
kernel parameters, so it is uniform per block, and blocks share nothing. Path A needs more registers than path B
|
||||
(LDR mvlo: 101 vs 88 VGPRs), and one kernel carries the larger allocation for every block.
|
||||
|
||||
This pass emits the same translated body twice:
|
||||
NAME runs path B and returns where path A would start;
|
||||
NAME_post1 runs path A and returns where path B would start.
|
||||
The ZLUDA/MicroCUDA native hook launches NAME_post1 right after NAME on the same stream with the same parameters
|
||||
and grid (d4r_post1_grid_x/_y = 0), with the 16x16 blocks the PTX hard-codes (tid.y << 4). Each block executes
|
||||
exactly the instructions it executed before; only the block order across the two paths changes, which no output
|
||||
depends on (blocks never read what other blocks write in the same launch).
|
||||
|
||||
The pass refuses anything but the expected entry structure:
|
||||
pA = ((int32_t)rX < (int32_t)24); pB = ((int32_t)rY < (int32_t)24); pC = (pA && pB);
|
||||
if (pC) goto LB;
|
||||
goto LA;
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
def split(source):
|
||||
head, sep, main = source.partition('extern "C" __attribute__((global))')
|
||||
if not sep or 'extern "C" __attribute__((global))' in main:
|
||||
raise SystemExit('expected exactly one kernel')
|
||||
m = re.match(r'\s*__attribute__\(\(amdgpu_flat_work_group_size\(1, 256\)\)\) void (\w+)\((\w+)\)', main)
|
||||
if not m:
|
||||
raise SystemExit('unexpected kernel signature')
|
||||
name = m.group(1)
|
||||
pattern = re.compile(r'^ (p\d+) = \(\(int32_t\)r\d+ < \(int32_t\)24\);\n'
|
||||
r' (p\d+) = \(\(int32_t\)r\d+ < \(int32_t\)24\);\n'
|
||||
r' (p\d+) = \((p\d+) && (p\d+)\);\n'
|
||||
r' if \((p\d+)\) goto (L__BB0_\d+);\n'
|
||||
r' goto (L__BB0_\d+);\n', re.M)
|
||||
found = list(pattern.finditer(main))
|
||||
if len(found) != 1:
|
||||
raise SystemExit(f'path selection found {len(found)} times, expected once')
|
||||
g = found[0]
|
||||
pa, pb, pc, x, y, pc2, path_b, path_a = g.groups()
|
||||
if {x, y} != {pa, pb} or pc != pc2:
|
||||
raise SystemExit('path selection predicate has an unexpected form')
|
||||
# the predicate must not be reassigned anywhere (it is the block's path choice)
|
||||
if len(re.findall(rf'^ {pc} = ', main, re.M)) != 1:
|
||||
raise SystemExit(f'{pc} is assigned more than once')
|
||||
sel = g.group(0)
|
||||
lines = sel.splitlines(keepends=True)
|
||||
main_b = main.replace(sel, ''.join(lines[:4]) + ' return; // path A blocks: NAME_post1\n', 1)
|
||||
main_a = main.replace(sel, ''.join(lines[:3]) + f' if ({pc}) return; // path B blocks: NAME\n'
|
||||
+ f' goto {path_a};\n', 1)
|
||||
main_a = main_a.replace(f' void {name}(', f' void {name}_post1(', 1)
|
||||
globals_ = ('\n// d4r native hook: launch NAME_post1 after NAME with the launch grid and the PTX\'s 16x16 blocks\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_grid_x = 0;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_grid_y = 0;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_x = 16;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_y = 16;\n'
|
||||
'extern "C" __attribute__((device, used)) uint32_t d4r_post1_block_z = 1;\n')
|
||||
banner = '// split_output_paths.py: NAME = path B only, NAME_post1 = path A only (per-block choice unchanged)\n'
|
||||
return (banner + head + globals_ + sep + main_b + '\n' + sep + main_a, path_b, path_a)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
text, b, a = split(open(sys.argv[1]).read())
|
||||
sys.stderr.write(f'split: path B {b}, path A {a}\n')
|
||||
sys.stdout.write(text)
|
||||
Reference in new issue
Block a user