Hardware e4m3 decode in dec0's tail on the native-FP8 build

make_ptx.py turns the 32 cvt.rn.f16x2.e4m3x2 of rrlite_dec0_4x4_folded into calls of d4r_dec_e4m3x2 (tex_common.h) when D4R_TEX_FP8=1: v_cvt_pk_f32_fp8 and two conversions to f16 instead of ZLUDA's integer lowering (about 25 instructions each; the kernel goes from 4058 to 3418 instructions). The conversion equals ZLUDA's e4m3_to_f16_bits for every code except the NaN codes 0x7f / 0xff, checked over all 65536 code pairs on an RX 9070 XT, and the head never stages those codes (it clamps to +-448 before encoding). Harness frames are byte-identical in the accuracy and the fast set, over a 16-frame moving sequence in the accuracy set. dec0 at 2227x1253 render: 0.43 -> 0.38 ms (accuracy), 0.36 -> 0.30 ms (fast). -DD4R_TEX_SOFT_E4M3 gives ZLUDA's lowering; builds without D4R_TEX_FP8 keep the PTX instruction.

kernels/tex/make_ptx.py

@@ -56,6 +56,7 @@ KERNELS = {
         delete=(139, 587),
         delete_check=("mov.u32 %r17, %laneid;", "st.shared.v2.u16 [%r570+1168], {%rs63, %rs64};"),
         sust=True,
+        dec8=32,
         extern=""".extern .func d4r_dec0_head
 (
 	.param .b32 d4r_a0, .param .b64 d4r_a1, .param .b64 d4r_a2, .param .b32 d4r_a3,
@@ -176,6 +177,36 @@ def replace_sust(lines):
     return out, n
 
 
+# dec0 on the native-FP8 build (D4R_TEX_FP8=1): the tail's e4m3 decodes call d4r_dec_e4m3x2 (tex_common.h, the
+# hardware conversion) instead of ZLUDA's integer lowering of cvt.rn.f16x2.e4m3x2
+DEC8_EXTERN = """.extern .func (.param .b32 d4r_dr) d4r_dec_e4m3x2
+(
+	.param .b16 d4r_dc
+)
+;
+"""
+DEC8_RE = re.compile(r"^cvt\.rn\.f16x2\.e4m3x2 (%r\d+), (%rs\d+);$")
+
+
+def replace_dec8(lines):
+    out, n = [], 0
+    for line in lines:
+        m = DEC8_RE.match(line.strip())
+        if not m:
+            out.append(line)
+            continue
+        dst, src = m.groups()
+        out.append(f"""{{
+	.param .b16 c0;
+	st.param.b16 [c0+0], {src};
+	.param .b32 d0;
+	call (d0), d4r_dec_e4m3x2, (c0);
+	ld.param.b32 {dst}, [d0+0];
+}}""")
+        n += 1
+    return out, n
+
+
 RZ_RE = re.compile(r"^add\.rz\.ftz\.f32 (%f\d+), (%f\d+), (%f\d+);$")
 CVT_RE = re.compile(r"^cvt\.rzi\.f32\.f32 (%f\d+), (%f\d+);$")
 
@@ -318,6 +349,11 @@ def main():
         at = next(i for i, l in enumerate(body) if l.startswith(".reg"))
         body = body[:at] + [f".reg .f32 %rq<{4 * nrz + 1}>;", f".reg .pred %pq<{nrz + 1}>;"] + body[at:]
         n["rz"] = nrz
+    if k.get("dec8") and os.environ.get("D4R_TEX_FP8", "0") == "1":
+        body, n["dec8"] = replace_dec8(body)
+        if n["dec8"] != k["dec8"]:
+            sys.exit(f"{name}: {n['dec8']} e4m3 decodes, expected {k['dec8']}")
+        k = dict(k, extern=k["extern"] + DEC8_EXTERN)
     extern = k["extern"] + (SUST_EXTERN if n["b16"] else "") + (SUSTP_EXTERN if n["p"] else "") + (SUSTB32_EXTERN if n["b32"] else "")
     text = "\n".join(head[:entry]) + "\n" + extern + "\n".join(body + tail)
     out.write_text(text)