wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

kernels: build the denoiser's output kernel natively, authorized separately

Browse Source 
Ray Reconstruction's eleven DLSS-D Swin layers still run ZLUDA's compile of
NVIDIA's PTX; at 1280x720 -> 3840x2160 on gfx1201 they are 30.2 of a 31.6 ms
frame. The denoiser's high-fidelity output kernel is the first one with a native
replacement: make_ptx.py replaces its whole-texel RGBA16F store
(sust.b.2d.v4.b16.zero) through d4r_sust_v4b16, kernels/build.sh builds the pair
as the `rr` family, and packaging builds it into every target folder including
the RDNA4 FP8 ones.

Both libraries define modules named cuda_dldn_engine_*, with different text, so
one manifest cannot take the union of their hashes: kernel_manifest.py now takes
the denoiser DLL as --rr-dll and lets only its hashes authorize a
cuda_dldn_engine_* binary, and package_release.sh passes it when the denoiser is
bundled.

Verified in the D3D12 harness with Ray Reconstruction at 1280x720 -> 3840x2160:
the output kernel drops from 1.44 to 1.00 ms/frame and the 66 MB raw output is
byte-identical to the run with every kernel translated.

docs/native-kernels.md gains the denoiser family, the per-kernel cost table and
the instruction counts that show why the Swin layers are the remaining work.
This commit is contained in:
woutermans committed 2026-10-12 12:47:53 +02:00
1 parent 56e1b76622
commit 74ce9d8f9a
7 files changed
+200 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernels/tex/make_ptx.py
+28 -3
View File
@@ -99,6 +99,13 @@ SUST_EXTERN = """.extern .func d4r_sust_v2b16
)
;
"""
SUST4_EXTERN = """.extern .func d4r_sust_v4b16
(
.param .b64 d4r_h0, .param .b32 d4r_h1, .param .b32 d4r_h2, .param .b16 d4r_h3,
.param .b16 d4r_h4, .param .b16 d4r_h5, .param .b16 d4r_h6
)
;
"""
SUSTP_EXTERN = """.extern .func d4r_sust_p_v4b32
(
.param .b64 d4r_t0, .param .b32 d4r_t1, .param .b32 d4r_t2, .param .b32 d4r_t3, .param .b32 d4r_t4,
@@ -115,14 +122,31 @@ SUSTB32_EXTERN = """.extern .func d4r_sust_b32
"""
SUSTB32_RE = re.compile(r"^sust\.b\.2d\.b32\.zero \[(%rd\d+), \{(%r\d+),(%r\d+)\}\], \{(%r\d+)\};$")
SUST_RE = re.compile(r"^sust\.b\.2d\.v2\.b16\.zero \[(%rd\d+), \{(%r\d+),(%r\d+)\}\], \{(%rs\d+),(%rs\d+)\};$")
SUST4_RE = re.compile(r"^sust\.b\.2d\.v4\.b16\.zero \[(%rd\d+), \{(%r\d+),\s*(%r\d+)\}\], \{(%rs\d+),\s*(%rs\d+),\s*(%rs\d+),\s*(%rs\d+)\};$")
SUST_KINDS = set(os.environ.get("D4R_SUST_KINDS", "b16,p,b32").split(","))
def replace_sust(lines):
out, n = [], {"b16": 0, "p": 0, "b32": 0}
out, n = [], {"b16": 0, "v4b16": 0, "p": 0, "b32": 0}
for line in lines:
mh = SUST4_RE.match(line.strip()) if "b16" in SUST_KINDS else None
if mh:
s, x, y, *v = mh.groups()
st = "\n".join(f"\t.param .b16 v{i};\n\tst.param.b16 [v{i}+0], {r};" for i, r in enumerate(v))
out.append(f"""{{
.param .b64 u0;
st.param.b64 [u0+0], {s};
.param .b32 u1;
st.param.b32 [u1+0], {x};
.param .b32 u2;
st.param.b32 [u2+0], {y};
{st}
call d4r_sust_v4b16, (u0, u1, u2, v0, v1, v2, v3);
}}""")
n["v4b16"] += 1
continue
mb = SUSTB32_RE.match(line.strip()) if "b32" in SUST_KINDS else None
if mb:
s, x, y, v = mb.groups()
@@ -414,7 +438,8 @@ ENC0_REF = "rrlite_enc0_4x4_mvhi_hdr_folded"
SUST_ONLY_RE = re.compile(r"^(hiluma_engine_output_depth(inv|reg)_mv(hi|lo)_(hdr|ldr)(_max)?_v[12]_rel|"
r"rrlite_post_3_[12]_mv(hi|lo)_(hdr|ldr)(_folded)?|"
r"rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)|rrlite_dec0_4x4|"
r"rrlite_downsample_kernel_(static|dynamic)_(hdr|ldr))$")
r"rrlite_downsample_kernel_(static|dynamic)_(hdr|ldr)|"
r"cuda_dldn_engine_hkpn_output_kernel_transformer(_diamond_wallaby)?)$")
ENC0_RE = re.compile(r"^rrlite_enc0_4x4_mv(hi|lo)_(hdr|ldr)_folded$")
@@ -536,7 +561,7 @@ def main():
if n["dec8"] != k["dec8"]:
sys.exit(f"{name}: {n['dec8']} e4m3 decodes, expected {k['dec8']}")
k = dict(k, extern=k["extern"] + DEC8_EXTERN)
extern = k["extern"] + (SUST_EXTERN if n["b16"] else "") + (SUSTP_EXTERN if n["p"] else "") + (SUSTB32_EXTERN if n["b32"] else "")
extern = k["extern"] + (SUST_EXTERN if n["b16"] else "") + (SUST4_EXTERN if n.get("v4b16") else "") + (SUSTP_EXTERN if n["p"] else "") + (SUSTB32_EXTERN if n["b32"] else "")
text = "\n".join(head[:entry]) + "\n" + extern + "\n".join(body + tail)
out.write_text(text)
print(f"{out}: {len(text.splitlines())} lines, surface stores replaced: {n}")
kernels/tex/tex_common.h
+33
View File
@@ -261,6 +261,39 @@ DEV __attribute__((always_inline)) void d4r_sust_v2b16(uint64_t surface, int32_t
sust_v2b16_generic(surface, x, y, a, b);
}
typedef int16_t v4s __attribute__((ext_vector_type(4)));
extern "C" __attribute__((device)) void __zluda_ptx_impl_surfobj_b_2d_v4_b16_zero(uint64_t surface, v2i coord, v4s value);
__attribute__((device, noinline)) static void sust_v4b16_generic(uint64_t surface, int32_t x, int32_t y,
uint16_t a, uint16_t b, uint16_t c, uint16_t d)
{
__zluda_ptx_impl_surfobj_b_2d_v4_b16_zero(surface, (v2i){x, y},
(v4s){(int16_t)a, (int16_t)b, (int16_t)c, (int16_t)d});
}
FP_BODY void sust_rgba16f_body(tsharp_t* image, int32_t px, int32_t y,
uint16_t a, uint16_t b, uint16_t c, uint16_t d)
{
__ockl_image_store_2D(image, (v2i){px, y},
(v4f){half_bits_to_float(a), half_bits_to_float(b),
half_bits_to_float(c), half_bits_to_float(d)});
}
// RR stores complete RGBA16F texels. No channel reads or partial-pixel merges
// are needed; other layouts retain the generic byte-addressed store.
DEV __attribute__((always_inline)) void d4r_sust_v4b16(uint64_t surface, int32_t x, int32_t y,
uint16_t a, uint16_t b, uint16_t c, uint16_t d)
{
tsharp_t* image = (tsharp_t*)surface;
if (__ockl_image_channel_data_type_2D(image) == 14 && __ockl_image_channel_order_2D(image) == 8 && (x & 7) == 0)
{
if (x >= 0)
sust_rgba16f_body(image, x >> 3, y, a, b, c, d);
}
else
sust_v4b16_generic(surface, x, y, a, b, c, d);
}
// f32 comparisons on the bits (no fcmp: kernels ZLUDA compiles in strict-FP mode cannot lower
// constrained fcmp); same results as the float compares, -0.0 included