diff --git a/engine/README.md b/engine/README.md index adf16b1..6ce22c0 100644 --- a/engine/README.md +++ b/engine/README.md @@ -1110,6 +1110,15 @@ reconstruction tile tiers and the stress inputs). The network layers are separated by an execution-only barrier (`emptyStage`). A memory barrier there gives the same output and costs about 60 us (M 1520 -> 1592), so it stays. +### `M_RESIDUAL_F32` (opt-in, 2026-10-19) + +`M_RESIDUAL_F32=1 python3 engine/compile_m.py ...` keeps the residual stream in f32 across the MLP's 32-channel chunks +and rounds it to binary16 once after them, instead of after every chunk. It needs the relaxed build (not +`M_EXACT=1`) and is off by default. 1280x720 -> 1440p: M 1521 -> 1410 us, L 2195 -> 2077. Against the default build, +final RGB, peak 1: 43 - 46 dB (L 45 - 49) on smooth frames, 17 - 18 dB on noise. The exact build is as far from the +default, so this moves no further from NVIDIA's arithmetic than `M_RELAXED` already does. Not checked against an +NVIDIA card. + ## Preset L (runtime and shim path exist; experimental) Preset L (render preset 12, NVIDIA's model for Ultra Performance) is M's pipeline with its own input, expansion and diff --git a/engine/compile_m.py b/engine/compile_m.py index 66b4ac7..a5293bf 100644 --- a/engine/compile_m.py +++ b/engine/compile_m.py @@ -34,6 +34,11 @@ FAST = [] if EXACT or '-DM_RELAXED' in os.environ.get('M_DEFS', '').split() else FAST_WPW = [] if EXACT or os.environ.get('M_WPW') or 'WPW=' in os.environ.get('M_DEFS', '') else ['-DWPW=2'] # preset L's expansion stage (dec0l) two windows per wave as well; its input stage (enc0l) is slower that way L_WPW = [] if EXACT or 'WPW=' in os.environ.get('M_DEFS', '') + os.environ.get('M_DEFS_DEC0L', '') else ['-DWPW=2'] +# M_RESIDUAL_F32=1 (opt-in, not with M_EXACT=1): every layer_m.comp stage keeps the residual stream f32 across the MLP's +# 32-channel chunks, one binary16 rounding after them instead of one per chunk (engine/README.md) +RESF32 = ['-DM_RESIDUAL_F32'] if os.environ.get('M_RESIDUAL_F32') == '1' else [] +if RESF32 and EXACT: + raise SystemExit('M_RESIDUAL_F32=1 is part of the relaxed arithmetic: not with M_EXACT=1') SHADERS = {'enc1': 'm_enc1', 'enc2': 'm_enc2', 'enc3_tube': 'm_tube', 'dec2': 'm_dec2', 'dec1': 'm_dec1'} @@ -107,7 +112,7 @@ def main(): # the network in the arithmetic of ZLUDA's lowering of NVIDIA's PTX (TEXL): byte-identical to the CUDA backend # with NVIDIA's translated kernels in the harness (2026-10-11), where the native accuracy kernels' K32_MIX order # drifted to 36-41 dB through the FP8 tube - for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW) + for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW + RESF32) for s in ('enc0_m', 'dec0_m', 'post_m', 'down_m'): glsl(s + '.comp', s + '.spv', ['-DLDR=1'] if a.ldr and s != 'dec0_m' else []) glsl('down_m.comp', 'down_m_rgb10.spv', ['-DOUTPUT_RGB10=1'] + (['-DLDR=1'] if a.ldr else [])) glsl('post_m.comp', 'post_m_q.spv', ['-DTILE_W=15', '-DTILE_H=15'] + (['-DLDR=1'] if a.ldr else [])) @@ -158,10 +163,10 @@ def compile_l(a, one, R): subprocess.run(['glslangValidator', '-V', '--target-env', 'vulkan1.3', *flags, str(HERE / src), '-o', str(a.output / out)], check=True, stdout=subprocess.DEVNULL) ldr = ['-DLDR=1'] if a.ldr else [] # the network in the arithmetic of ZLUDA's lowering of NVIDIA's PTX (TEXL), as for preset M (L_NOTES.md) - for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW) + for name, out in SHADERS.items(): glsl('layer_m.comp', out + '.spv', defines(name) + ['-DTEXL=1'] + FAST + FAST_WPW + RESF32) for planes, suffix in ((0, ''), (1, '_planes')): - glsl('layer_m.comp', f'enc0_l{suffix}.spv', defines('enc0l') + ldr + [f'-DPLANES={planes}'] + FAST) - glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST + L_WPW) + glsl('layer_m.comp', f'enc0_l{suffix}.spv', defines('enc0l') + ldr + [f'-DPLANES={planes}'] + FAST + RESF32) + glsl('layer_m.comp', f'dec0_l{suffix}.spv', defines('dec0l') + [f'-DPLANES={planes}'] + FAST + L_WPW + RESF32) # the reconstruction one render texel wider per tier than M's (4x4 taps): 11 (reconstruction/render >= 3), # 16 (the whole post_3_2 range), 12 (the compact tier: render/reconstruction <= 0.45, jitter within half a texel) for out, tile in (('post_l', 11), ('post_l_q', 16), ('post_l_c', 12)): diff --git a/engine/compile_rr2.py b/engine/compile_rr2.py index 455ea45..0d8fed8 100644 --- a/engine/compile_rr2.py +++ b/engine/compile_rr2.py @@ -80,10 +80,18 @@ GROUP = int(os.environ.get('D4R_RR2_GROUP', '1')) # D4R_RR2_SPLIT=1 / 0 forces the gather pass / the single shader. SPLIT = os.environ.get('D4R_RR2_SPLIT', '0' if RELAXED >= 3 and 'enc0' in ONE_WAVE else '1') != '0' # The pooled histories as RGBA32F holding q / 1023, as ZLUDA backs them (16 bytes a pixel). D4R_RR2_POOLED=rgb10_a2 -# makes them real UNORM 10:10:10:2 images (0.1 ms faster at 4K), but the hardware's bilinear filter then rounds -# differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13) +# makes them real UNORM 10:10:10:2 images (0.1 ms faster at 4K, 70 MB less), but the hardware's bilinear filter then +# rounds differently: identical at 3x and 2x scaling, 44 - 51 dB on temporal frames at 1.5x (capFQ, 2026-10-13); on +# random-motion noise also at 2x (29 dB on the history frames, engine/rr/README.md) POOLED = os.environ.get('D4R_RR2_POOLED', 'rgba32f') SURFACES = {'dec0_1': {176: 'rgba8_snorm', 184: 'rgba8', 192: 'rgba8'}, 'post': {120: POOLED}, 'pooled': {8: POOLED}} +# D4R_RR2_UNGUARDED=1 (opt-in, D4R_RR_RELAXED=3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, +# the output passes) without its exactness guards: no `precise` registers, no opaque zero OR-ed into binary16 stores and +# conversions (ptx2rr.py --unguarded) and no NoContraction pass (spv_exact_convert.py) on those modules, so the driver may +# contract, fold and fuse there as it does in the relaxed native stages (engine/rr/README.md) +UNGUARDED = os.environ.get('D4R_RR2_UNGUARDED') == '1' +if UNGUARDED and RELAXED < 3: + raise SystemExit('D4R_RR2_UNGUARDED=1 is for D4R_RR_RELAXED=3') def find_module(ptx_dir, kernel): @@ -170,8 +178,8 @@ def main(): cmd += [f'--surface={off}={fmt}' for off, fmt in SURFACES.get(stage, {}).items()] # the hybrid stages' and the output passes' modules go through spv_exact_convert.py, so their conversions need # only the lighter guard (ptx2rr.py --exact-convert) - exact = (bool(a.native) and stage in HYBRID) or not network - cmd += ['--exact-convert'] if exact else [] + exact = ((bool(a.native) and stage in HYBRID) or not network) and not UNGUARDED + cmd += ['--exact-convert'] if exact else ['--unguarded'] if UNGUARDED else [] subprocess.run(cmd, check=True, stderr=subprocess.DEVNULL) info = json.loads(generated.with_suffix('.json').read_text()) weights = '-' @@ -195,8 +203,9 @@ def main(): subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', *defs, f'-I{work}', f'-I{HERE / "rr"}', str(work / 'enc0_g.comp'), '-o', str(out / 'enc0g.spv')], check=True, stdout=subprocess.DEVNULL) - subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / 'enc0g.spv')], - check=True, stdout=subprocess.DEVNULL) + if not UNGUARDED: + subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / 'enc0g.spv')], + check=True, stdout=subprocess.DEVNULL) record = int((work / 'enc0_g.record').read_text()) if RELAXED >= 3 and stage in ONE_WAVE and (split or GROUP == 1): how = how + ['--one-wave'] @@ -206,8 +215,9 @@ def main(): subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', *defs, f'-I{work}', f'-I{HERE / "rr"}', str(work / f'{stage}_m.comp'), '-o', str(out / f'{stage}.spv')], check=True, stdout=subprocess.DEVNULL) - subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / f'{stage}.spv')], - check=True, stdout=subprocess.DEVNULL) + if not UNGUARDED: + subprocess.run([sys.executable, str(HERE / 'rr' / 'spv_exact_convert.py'), str(out / f'{stage}.spv')], + check=True, stdout=subprocess.DEVNULL) else: subprocess.run(capped + ['glslangValidator', '-V', '--target-env', 'vulkan1.3', str(generated), '-o', str(out / f'{stage}.spv')], check=True, stdout=subprocess.DEVNULL) diff --git a/engine/layer_m.comp b/engine/layer_m.comp index 84e673c..2d85faa 100644 --- a/engine/layer_m.comp +++ b/engine/layer_m.comp @@ -725,6 +725,20 @@ void main() EACH_T(N2) } // ------------------------------------------------------------ the MLP, 32 hidden channels at a time + // M_RESIDUAL_F32 (opt-in, part of M_RELAXED): the residual stays f32 across the MLP's chunks and is rounded to + // binary16 once after them, instead of after every chunk +#ifdef M_RESIDUAL_F32 +#ifndef M_RELAXED +#error layer_m.comp: M_RESIDUAL_F32 needs M_RELAXED +#endif + ACC RF_[WPW][NT]; +#define RF_IN(N) RF_[w][N] = ACC(res[N]); + FORW { EACH_T(RF_IN) } +#undef RF_IN +#define K32R(N, a0, b0, a1, b1) { RF_[w][N] = coopMatMulAdd((a1), (b1), coopMatMulAdd((a0), (b0), RF_[w][N])); } +#else +#define K32R(N, a0, b0, a1, b1) K32H(res[N], a0, b0, a1, b1) +#endif for (uint c = 0u; c < UC / 8u; ++c) { const uint ow1 = p.o[20u + 3u * c], ob1 = p.o[21u + 3u * c], ow2 = p.o[22u + 3u * c]; @@ -759,9 +773,15 @@ void main() A8 a0, a1; \ LDA8(a0, ow2, 0u, UC, 16u * (N)); \ LDA8(a1, ow2, 1u, UC, 16u * (N)); \ - FORW { K32H(res[N], a0, G0_[w], a1, G1_[w]) } } + FORW { K32R(N, a0, G0_[w], a1, G1_[w]) } } EACH_T(MLP2_N) } +#undef K32R +#ifdef M_RESIDUAL_F32 +#define RF_OUT(N) res[N] = HAC(RF_[w][N]); + FORW { EACH_T(RF_OUT) } +#undef RF_OUT +#endif #if KIND == 4 // ------------------------------------------------------------ L's expansion head: per-pixel reconstruction inputs SYNC // every wave is done with the V tiles in HO diff --git a/engine/rr/README.md b/engine/rr/README.md index e0ac464..3e1f6a9 100644 --- a/engine/rr/README.md +++ b/engine/rr/README.md @@ -629,3 +629,16 @@ byte-identical to the previous model on the replay and stress inputs, at levels total with an atomic add and the last block finishing. It needs `shaderBufferInt64Atomics`, which vkd3d-proton enables; the shim reads the device's features through `GetDeviceFeatures`. Without it, or with a model that has no `exposure_f2.spv`, the two passes run (`D4R_RR_TEST_NO_INT64_ATOMICS=1` forces them). Exposure 25.2 -> 14.9 us. + +### Opt-in builds for preset F (2026-10-19) + +Against the default build, presented image, RGB, peak 1, 1280x720 -> 1440p. For scale, the exact build +(`D4R_RR_RELAXED=0`) is 43.0 - 44.4 dB from the default. + +- `D4R_RR2_UNGUARDED=1` (level 3 only): the translated code (encoder 0's and decoder 0's per-pixel parts, the output + passes) without its exactness guards: no `precise`, no opaque zero on binary16 stores, no `spv_exact_convert.py`. + 56.0 dB on the reset frame, then 43.0 - 45.6 dB, as far from the exact build as the default is. Frame 3.526 -> 3.501 + ms. +- `D4R_RR2_POOLED=rgb10_a2` (since 2026-10-13): the pooled histories as UNORM 10:10:10:2 images, 31 MB less at 1440p. + Identical at 1280x720 -> 1440p and on reset frames; 43.7 - 47.2 dB on the history frames at 1114x626 -> 1080p and + 28.8 dB with random motion. Frame 3.526 -> 3.484 ms, 3.456 with `D4R_RR2_UNGUARDED=1` as well. diff --git a/engine/rr/ptx2rr.py b/engine/rr/ptx2rr.py index 515982d..a279e36 100755 --- a/engine/rr/ptx2rr.py +++ b/engine/rr/ptx2rr.py @@ -950,6 +950,9 @@ class RRT(base.T): w(f'shared uint mms[{384 * (self.bx * self.by * self.bz // 32)}]; // mma_e4m3_f16 scratch, per subgroup') def runtime(self): + if getattr(self, 'unguarded', False): # --unguarded: no opaque zero + assert '#define SUST_Z (gl_NumWorkGroups.z - 1u)' in RR_RUNTIME + return RR_RUNTIME.replace('#define SUST_Z (gl_NumWorkGroups.z - 1u)', '#define SUST_Z 0u') return RR_RUNTIME @@ -976,6 +979,9 @@ def main(): ap.add_argument('--oob-report', action='store_true', help='debugging: count out-of-range buffer accesses') ap.add_argument('--exact-convert', action='store_true', help='the module goes through spv_exact_convert.py: lighter guards on binary16 conversions (CVT_GUARD)') + ap.add_argument('--unguarded', action='store_true', + help="none of the exactness guards: no 'precise' registers and no opaque zero OR-ed into binary16 " + "stores and conversions, so the driver may contract, fold and fuse (compile_rr2.py D4R_RR2_UNGUARDED)") ns = ap.parse_args() block = tuple(int(x) for x in ns.block.split(',')) if ns.block else default_block(ns.kernel) formats = {int(k): v for k, v in (x.split('=') for x in ns.surface)} @@ -991,8 +997,9 @@ def main(): t.probes = probes t.probe_block = tuple(int(x) for x in ns.probe_block.split(',')) t.oob = ns.oob_report + t.unguarded = ns.unguarded return t - t = base.build(make, False, False, False) + t = base.build(make, False, ns.unguarded, False) for off, fmt in formats.items(): if off in t.surfaces: t.surfaces[off] = fmt