Port K kernel tuning and M gfx1101 flags
Preset K, byte-identical to the previous output in all captured buffers and the harness image, measured on an RX 7700 XT: - LDS row padding so WMMA operand loads hit distinct banks - WGP mode for enc1/enc2/dec1/dec2, ds_bpermute half-wave exchange for enc0/dec0/dec3, more waves per window in enc4/dec4/dec5 - cheaper operand plumbing in common/wmma_layout.h (packed f16 conversions, no dynamic vector indexing), raw buffer weight loads, accumulator-init vectors prepared once - enc1 as a single-buffer recompute layer, dec1 Q/K/V chains split - dec5 as four dependent phase kernels
This commit is contained in:
ayo committed
1 parent
8ce8618181
commit
22026ede12
45 files changed
+4514
-354
No files matched your search
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
# build_native.sh SRC.hip OUT.hsaco [extra clang flags]: natively compiled DLSS texture kernel (gfx1101, wave64 default)
|
||||
set -euo pipefail
|
||||
ND=$(cd "$(dirname "$0")" && pwd)
|
||||
SRC=$(realpath "$1") OUT=$(realpath -m "$2"); shift 2
|
||||
[[ -s "$SRC" ]] || { echo "empty native source: $SRC" >&2; exit 2; }
|
||||
R=${D4R_ROCM_DIR:-/opt/rocm}
|
||||
ARCH=${D4R_GPU_ARCH:-gfx1101}
|
||||
W=${WAVE:-64}
|
||||
WF=$([[ $W == 64 ]] && echo -mwavefrontsize64 || echo -mno-wavefrontsize64)
|
||||
D=$(cd "$(dirname "$SRC")" && pwd)
|
||||
T=$(mktemp -d); trap 'rm -rf "$T"' EXIT
|
||||
(cd "$T" && "$R/lib/llvm/bin/clang++" -x hip -std=c++20 --offload-arch=$ARCH --offload-device-only -O3 $WF -mcumode \
|
||||
-ffp-contract=off -fgpu-flush-denormals-to-zero -fno-fast-math -I"$D" -I"$ND" -I"$R/include" --rocm-path="$R" \
|
||||
--rocm-device-lib-path="$R/amdgcn/bitcode" "$@" -o "$OUT" "$SRC" -save-temps=cwd --no-gpu-bundle-output)
|
||||
S=$(ls "$T"/*.s | head -1)
|
||||
cp "$S" "${OUT%.hsaco}.s"
|
||||
printf '%s: %s\n' "$(basename "$OUT")" "$(grep -hE '^\s+\.(vgpr_count|sgpr_count|private_segment_fixed_size|vgpr_spill_count|group_segment_fixed_size):' "$S" | tr -s ' ' | paste -sd' ')"
|
||||
Executable
+411
@@ -0,0 +1,411 @@
|
||||
"""Color-tap LDS tile pass for the hiluma output kernels (used by ptx2hip.py --ctile).
|
||||
|
||||
Path A of hiluma_engine_output_* (taken when a block's render footprint is wider than path B's 24-texel LDS tile,
|
||||
e.g. at Quality) fetches 16 color texels per thread and converts each one (YCoCg, exposure, tonemap weight) right
|
||||
after the fetch. The conversion of a texel depends only on the texel and the exposure %f1, and neighbouring
|
||||
threads convert the same texels (~9x at ratio 1.5). This pass:
|
||||
|
||||
1. finds every tex.base on the color texture inside path A,
|
||||
2. slices the instructions that compute the tap's packed result (cvt.rn.f16x2 x2 or cvt.rn.f16.f32) from the
|
||||
fetched texel (forward taint + backward need, including the tonemap branch diamond), and checks that no other
|
||||
value escapes the slice,
|
||||
3. replaces the slice with `d4r.ctap.G` (tile lookup, falling back to fetch + the same conversion), and
|
||||
4. emits one conversion function per slice shape (generated from the slice's own instructions, so the tile holds
|
||||
bit-identical values) and a cooperative tile fill at the start of path A.
|
||||
|
||||
The tile is keyed by the final (clamped) texel coordinates, so any tap outside it just takes the fallback.
|
||||
"""
|
||||
import re
|
||||
|
||||
REG = re.compile(r'%[A-Za-z_$][\w$]*')
|
||||
CAP = 1024 # tile texels (TW*TH); a larger footprint falls back to direct fetches
|
||||
|
||||
|
||||
class CtileErr(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def parse(body, tokenize):
|
||||
"""-> list of dicts: kind ('stmt'|'label'|'open'|'close'), text, plus op/dst/src/pred for stmts"""
|
||||
items = []
|
||||
for kind, text in tokenize(body):
|
||||
it = {'kind': kind, 'text': text}
|
||||
if kind == 'stmt':
|
||||
s = text
|
||||
m = re.match(r'@(!?)(%p\d+)\s+(.*)$', s)
|
||||
it['pred'] = m.group(2) if m else None
|
||||
if m:
|
||||
s = m.group(3)
|
||||
parts = s.split(None, 1)
|
||||
it['op'] = parts[0]
|
||||
rest = parts[1] if len(parts) > 1 else ''
|
||||
it['rest'] = rest
|
||||
dst, src = [], []
|
||||
op = it['op']
|
||||
if op.startswith('.') or op.startswith('bra') or op.startswith('bar') or op.startswith('ret'):
|
||||
src = REG.findall(rest)
|
||||
elif op.startswith('st.') or op.startswith('sust'):
|
||||
src = REG.findall(rest)
|
||||
else:
|
||||
# first operand = destination(s)
|
||||
depth, i = 0, 0
|
||||
while i < len(rest):
|
||||
c = rest[i]
|
||||
if c in '{[':
|
||||
depth += 1
|
||||
elif c in '}]':
|
||||
depth -= 1
|
||||
elif c == ',' and depth == 0:
|
||||
break
|
||||
i += 1
|
||||
dst = REG.findall(rest[:i])
|
||||
src = REG.findall(rest[i:])
|
||||
if it['pred']:
|
||||
src = src + [it['pred']]
|
||||
it['dst'], it['src'] = dst, src
|
||||
items.append(it)
|
||||
return items
|
||||
|
||||
|
||||
def branch_target(it):
|
||||
return it['rest'].strip().rstrip(';').strip().lstrip('$') if it['kind'] == 'stmt' and it['op'].startswith('bra') else None
|
||||
|
||||
|
||||
def find_taps(items, color_aliases):
|
||||
return [i for i, it in enumerate(items) if it['kind'] == 'stmt' and it['op'] == 'tex.base.2d.v4.f32.s32'
|
||||
and re.search(r'\[\s*(%rd\d+)', it['rest']).group(1) in color_aliases]
|
||||
|
||||
|
||||
def slice_tap(items, t):
|
||||
"""-> (removed index set, results list, kind 'in'|'out', externals set)"""
|
||||
tex = items[t]
|
||||
taint = set(tex['dst'])
|
||||
tainted_idx = [t]
|
||||
results, kind = [], None
|
||||
i = t + 1
|
||||
while i < len(items):
|
||||
it = items[i]
|
||||
if it['kind'] == 'stmt' and any(s in taint for s in it['src']):
|
||||
op = it['op']
|
||||
if op.startswith('cvt.rn.f16x2.f32') and kind in (None, 'in'):
|
||||
kind = 'in'
|
||||
results.append((i, it['dst'][0]))
|
||||
tainted_idx.append(i)
|
||||
if len(results) == 2:
|
||||
break
|
||||
elif op.startswith('cvt.rn.f16.f32') and kind is None:
|
||||
kind = 'out'
|
||||
results.append((i, it['dst'][0]))
|
||||
tainted_idx.append(i)
|
||||
break
|
||||
else:
|
||||
taint.update(it['dst'])
|
||||
tainted_idx.append(i)
|
||||
i += 1
|
||||
else:
|
||||
raise CtileErr(f'tap at {t}: no packed result found')
|
||||
last = results[-1][0]
|
||||
# backward need from the results (within [t, last])
|
||||
need_regs = set()
|
||||
for ri, _ in results:
|
||||
need_regs.update(items[ri]['src'])
|
||||
needed = {ri for ri, _ in results}
|
||||
# conditional branches on tainted predicates (the tonemap branch) are part of the slice
|
||||
for j in tainted_idx:
|
||||
it = items[j]
|
||||
if it['op'].startswith('bra') and it['pred'] is not None:
|
||||
needed.add(j)
|
||||
need_regs.add(it['pred'])
|
||||
for j in range(last, t - 1, -1):
|
||||
it = items[j]
|
||||
if j in needed:
|
||||
continue
|
||||
if it['kind'] == 'stmt' and j in tainted_idx and any(d in need_regs for d in it['dst']):
|
||||
needed.add(j)
|
||||
need_regs.update(it['src'])
|
||||
extra = [j for j in tainted_idx if j not in needed]
|
||||
if extra:
|
||||
raise CtileErr(f'tap at {t}: tainted but not needed: {[items[j]["text"] for j in extra]}')
|
||||
removed = set(needed)
|
||||
# branch diamonds: region from each conditional branch to its merge label
|
||||
for j in sorted(needed):
|
||||
it = items[j]
|
||||
if not it['op'].startswith('bra'):
|
||||
continue
|
||||
targets, seen = {branch_target(it)}, set()
|
||||
k = j + 1
|
||||
while True:
|
||||
x = items[k]
|
||||
if x['kind'] == 'label':
|
||||
lab = x['text'].lstrip('$')
|
||||
if lab in targets and targets - seen == {lab} and lab not in {branch_target(it)}:
|
||||
removed.add(k)
|
||||
break
|
||||
seen.add(lab)
|
||||
removed.add(k)
|
||||
elif x['kind'] == 'stmt':
|
||||
if x['op'].startswith('bra'):
|
||||
if x['pred'] is not None and k not in needed:
|
||||
raise CtileErr(f'tap at {t}: foreign conditional branch in diamond')
|
||||
targets.add(branch_target(x))
|
||||
removed.add(k)
|
||||
elif k not in needed:
|
||||
# constant moves used only inside the slice are part of it
|
||||
if x['op'].startswith('mov') and not any(REG.match(s) for s in x['src']):
|
||||
removed.add(k)
|
||||
else:
|
||||
raise CtileErr(f'tap at {t}: foreign instruction in diamond: {x["text"]}')
|
||||
k += 1
|
||||
if k > last:
|
||||
raise CtileErr(f'tap at {t}: diamond does not close before the result')
|
||||
# escape check: registers defined in the slice (except results) used outside it
|
||||
defs = set()
|
||||
for j in removed:
|
||||
if items[j]['kind'] == 'stmt':
|
||||
defs.update(items[j]['dst'])
|
||||
res_regs = {r for _, r in results}
|
||||
inner_defs = defs - res_regs
|
||||
for j, it in enumerate(items):
|
||||
if j in removed or it['kind'] != 'stmt':
|
||||
continue
|
||||
bad = inner_defs.intersection(it['src'])
|
||||
if bad:
|
||||
raise CtileErr(f'tap at {t}: {bad} escapes to {it["text"]}')
|
||||
if defs.intersection(it['dst']) - res_regs or res_regs.intersection(it['dst']):
|
||||
raise CtileErr(f'tap at {t}: slice register redefined by {it["text"]}')
|
||||
# labels removed must not be targeted from outside
|
||||
labels = {items[j]['text'].lstrip('$') for j in removed if items[j]['kind'] == 'label'}
|
||||
for j, it in enumerate(items):
|
||||
if j not in removed and branch_target(it) in labels:
|
||||
raise CtileErr(f'tap at {t}: label targeted from outside the slice')
|
||||
used = set()
|
||||
for j in removed:
|
||||
if items[j]['kind'] == 'stmt' and j != t:
|
||||
used.update(items[j]['src'])
|
||||
externals = used - defs - set(tex['dst'])
|
||||
externals.discard(None)
|
||||
return removed, results, kind, externals
|
||||
|
||||
|
||||
def canon(items, removed, t, results):
|
||||
"""canonical text of the slice (registers by first appearance, texel by position)"""
|
||||
names, labels = {}, {}
|
||||
for k, r in enumerate(items[t]['dst']):
|
||||
names[r] = f'T{k}'
|
||||
for k, (_, r) in enumerate(results):
|
||||
names[r] = f'R{k}'
|
||||
out = []
|
||||
for j in sorted(removed):
|
||||
if j == t:
|
||||
continue
|
||||
it = items[j]
|
||||
if it['kind'] == 'label':
|
||||
lab = it['text'].lstrip('$')
|
||||
labels.setdefault(lab, f'L{len(labels)}')
|
||||
out.append(labels[lab] + ':')
|
||||
continue
|
||||
if it['kind'] != 'stmt':
|
||||
continue
|
||||
s = it['text']
|
||||
|
||||
def rn(m):
|
||||
r = m.group(0)
|
||||
if r not in names:
|
||||
names[r] = f'V{len(names)}'
|
||||
return names[r]
|
||||
s = REG.sub(rn, s)
|
||||
s = re.sub(r'\$(L__\w+)', lambda m: labels.setdefault(m.group(1), f'L{len(labels)}'), s)
|
||||
out.append(s)
|
||||
return '\n'.join(out), names
|
||||
|
||||
|
||||
def apply(tr, tokenize):
|
||||
"""Rewrite tr.body in place for the color tile. Returns the HIP prelude (conversion functions) and info."""
|
||||
body = tr.body
|
||||
items = parse(body, tokenize)
|
||||
# color texture handle: ld.param.u64 [%rd1+-56] (param+200, %rd10 / %rd69); taps use it or its mov aliases
|
||||
color = {it['dst'][0] for it in items if it['kind'] == 'stmt' and it['op'] == 'ld.param.u64'
|
||||
and re.search(r'\[%rd1\+-56\]', it['rest'])}
|
||||
aliases = set(color)
|
||||
for it in items:
|
||||
if it['kind'] == 'stmt' and it['op'] == 'mov.u64' and len(it['src']) == 1 and it['src'][0] in color:
|
||||
aliases.add(it['dst'][0])
|
||||
# path A's color taps lie between $L__BB0_1 and $L__BB0_77 (the per-tap blocks); path B's are elsewhere
|
||||
labels = {it['text'].lstrip('$'): i for i, it in enumerate(items) if it['kind'] == 'label'}
|
||||
a0, a1 = labels['L__BB0_1'], labels['L__BB0_77']
|
||||
taps = [t for t in find_taps(items, aliases) if a0 < t < a1]
|
||||
if len(taps) != 16:
|
||||
raise CtileErr(f'expected 16 path A color taps, found {len(taps)}')
|
||||
groups, per_tap = {}, []
|
||||
all_removed = set()
|
||||
for t in taps:
|
||||
removed, results, kind, ext = slice_tap(items, t)
|
||||
if removed & all_removed:
|
||||
raise CtileErr('overlapping slices')
|
||||
all_removed |= removed
|
||||
text, names = canon(items, removed, t, results)
|
||||
ext_names = sorted(ext, key=lambda r: names.get(r, r))
|
||||
key = (kind, text, tuple(names.get(r) for r in ext_names))
|
||||
g = groups.setdefault(key, len(groups))
|
||||
per_tap.append((t, removed, results, kind, ext_names, g))
|
||||
if len(groups) > 3:
|
||||
raise CtileErr(f'{len(groups)} slice shapes')
|
||||
# externals: %f1 (exposure) or constant registers (single mov-immediate definition)
|
||||
const_def = {}
|
||||
for it in items:
|
||||
if it['kind'] == 'stmt' and it['op'].startswith('mov') and len(it['dst']) == 1 and not it['src']:
|
||||
const_def.setdefault(it['dst'][0], []).append(it)
|
||||
group_info = {}
|
||||
for (t, removed, results, kind, ext_names, g) in per_tap:
|
||||
if g in group_info:
|
||||
continue
|
||||
params, consts = [], []
|
||||
for r in ext_names:
|
||||
if r == '%f1':
|
||||
params.append(r)
|
||||
elif r in const_def and len(const_def[r]) == 1:
|
||||
consts.append(const_def[r][0]['text'])
|
||||
else:
|
||||
raise CtileErr(f'external {r} of tap {t}')
|
||||
group_info[g] = dict(t=t, removed=removed, results=results, kind=kind, params=params, consts=consts)
|
||||
return items, per_tap, group_info, color
|
||||
|
||||
|
||||
def emit(tr, tokenize, Translator):
|
||||
"""Apply the pass to translator tr (before translate()). Installs the pseudo-op handler; returns prelude text."""
|
||||
items, per_tap, group_info, color = apply(tr, tokenize)
|
||||
# ---- conversion functions, one per group, translated from the group's first slice
|
||||
prelude = []
|
||||
for g, info in sorted(group_info.items()):
|
||||
t, removed, results = info['t'], info['removed'], info['results']
|
||||
sub = Translator.__new__(Translator)
|
||||
sub.kernel = tr.kernel
|
||||
sub.param_name = tr.param_name
|
||||
sub.regs, sub.shared, sub.locals, sub.lines = {}, [], [], []
|
||||
sub.scope_names, sub.scope_id = [{}], 0
|
||||
tex = items[t]
|
||||
# texel inputs
|
||||
for k, r in enumerate(tex['dst']):
|
||||
sub.reg(r)
|
||||
sub.lines.append(f' {sub.reg(r)} = t_[{k}];')
|
||||
for c in info['consts']:
|
||||
sub.stmt(c)
|
||||
for j in sorted(removed):
|
||||
if j == t:
|
||||
continue
|
||||
it = items[j]
|
||||
if it['kind'] == 'label':
|
||||
sub.lines.append(f'{it["text"].replace("$", "")}:;')
|
||||
elif it['kind'] == 'stmt':
|
||||
sub.stmt(it['text'])
|
||||
outs = [sub.reg(r) for _, r in results]
|
||||
decl = []
|
||||
for name, ty in sorted(sub.regs.items()):
|
||||
if name in [sub.reg(p) for p in info['params']]:
|
||||
continue
|
||||
decl.append(f' {ty} {name};')
|
||||
args = ', '.join(['f4 t_'] + [f'float {sub.reg(p)}' for p in info['params']] + [f'uint32_t* o{k}' for k in range(len(outs))])
|
||||
prelude.append(f'RT void d4r_ctap_conv{g}({args})\n{{')
|
||||
prelude.extend(decl)
|
||||
prelude.extend(sub.lines)
|
||||
for k, o in enumerate(outs):
|
||||
prelude.append(f' *o{k} = (uint32_t){o};')
|
||||
prelude.append('}')
|
||||
# ---- tile storage (aliases the kernel's LDS through d4r_lds)
|
||||
offs, off = {}, 0
|
||||
for g, info in sorted(group_info.items()):
|
||||
for k in range(len(info['results'])):
|
||||
offs[(g, k)] = off
|
||||
off += CAP * 4
|
||||
# ---- rewrite the body: removed statements dropped, pseudo-op at each tap, fill at the start of path A
|
||||
tap_at = {t: (results, ext, g) for (t, removed, results, kind, ext, g) in per_tap}
|
||||
removed_all = set()
|
||||
for (t, removed, *_r) in per_tap:
|
||||
removed_all |= removed
|
||||
out = []
|
||||
for i, it in enumerate(items):
|
||||
if it['kind'] == 'open':
|
||||
out.append('{'); continue
|
||||
if it['kind'] == 'close':
|
||||
out.append('}'); continue
|
||||
if it['kind'] == 'label':
|
||||
if i in removed_all:
|
||||
continue
|
||||
out.append(it['text'] + ':')
|
||||
if it['text'].lstrip('$') == 'L__BB0_1':
|
||||
out.append('d4r.ctile_fill;')
|
||||
continue
|
||||
if i in tap_at:
|
||||
results, ext, g = tap_at[i]
|
||||
m = re.search(r'\[\s*(%rd\d+)\s*,\s*\{([^}]*)\}\s*\]', it['rest'])
|
||||
regs = ', '.join(r for _, r in results)
|
||||
out.append(f'd4r.ctap.{g} {regs}, {m.group(1)}, {m.group(2)};')
|
||||
continue
|
||||
if i in removed_all:
|
||||
continue
|
||||
out.append(it['text'] + ';')
|
||||
tr.body = '\n'.join(out)
|
||||
summary = dict(groups=group_info, offs=offs, lds_bytes=off, taps=len(per_tap))
|
||||
|
||||
orig_op = tr.op
|
||||
|
||||
def op(opname, a):
|
||||
if opname == 'd4r.ctile_fill':
|
||||
tr.emit('d4r_ctile_fill(KARGP, (int32_t)r3, (int32_t)r4, (int32_t)r5, (int32_t)r6, f1, &ct_x0, &ct_y0, &ct_w, &ct_h);')
|
||||
return
|
||||
if opname.startswith('d4r.ctap.'):
|
||||
g = int(opname.split('.')[2])
|
||||
nres = len(group_info[g]['results'])
|
||||
res = [tr.reg(x) for x in a[:nres]]
|
||||
h = tr.val(a[nres], 'u64')
|
||||
x, y = tr.val(a[nres + 1], 'u32'), tr.val(a[nres + 2], 'u32')
|
||||
prm = ''.join(f', {tr.val(p, "f32")}' for p in group_info[g]['params'])
|
||||
tr.emit(f'{{ int32_t dx_ = (int32_t){x} - ct_x0, dy_ = (int32_t){y} - ct_y0;')
|
||||
tr.emit(f' if ((uint32_t)dx_ < (uint32_t)ct_w && (uint32_t)dy_ < (uint32_t)ct_h) {{ uint32_t i_ = (uint32_t)(dy_ * ct_w + dx_);')
|
||||
for k, r in enumerate(res):
|
||||
tr.emit(f' {r} = (uint32_t)((const AS3 uint32_t*)(d4r_lds + {offs[(g, k)]}))[i_];')
|
||||
outs = ', '.join(f'o_ + {k}' for k in range(nres))
|
||||
tr.emit(f' }} else {{ uint32_t o_[2]; d4r_ctap_conv{g}(tex_fetch({h}, (int32_t){x}, (int32_t){y}){prm}, {outs});')
|
||||
for k, r in enumerate(res):
|
||||
tr.emit(f' {r} = o_[{k}];')
|
||||
tr.emit(' } }')
|
||||
return
|
||||
orig_op(opname, a)
|
||||
tr.op = op
|
||||
# result register types: f16x2 packs are u32 regs, f16 packs are u16 regs: both assigned via (uint32_t) casts
|
||||
# fill function (needs the conversion functions and offsets)
|
||||
fill = ['RT void d4r_ctile_fill(const AS4 uint8_t* KARGP, int32_t x0l, int32_t y0l, int32_t x1l, int32_t y1l, float f1,',
|
||||
' int32_t* ox0, int32_t* oy0, int32_t* ow, int32_t* oh)',
|
||||
'{',
|
||||
' // final texel coordinate of a tile-relative logical coordinate u: u > 0 ? min(lo + u, hi) : lo',
|
||||
' // (the clamp every color tap applies; params +208/+212 = lo, +216/+220 = hi)',
|
||||
' int32_t lox = *(const AS4 int32_t*)(KARGP + 208), loy = *(const AS4 int32_t*)(KARGP + 212);',
|
||||
' int32_t hix = *(const AS4 int32_t*)(KARGP + 216), hiy = *(const AS4 int32_t*)(KARGP + 220);',
|
||||
' int32_t fx0 = x0l > 0 ? __builtin_elementwise_min(lox + x0l, hix) : lox;',
|
||||
' int32_t fy0 = y0l > 0 ? __builtin_elementwise_min(loy + y0l, hiy) : loy;',
|
||||
' int32_t fx1 = x1l > 0 ? __builtin_elementwise_min(lox + x1l, hix) : lox;',
|
||||
' int32_t fy1 = y1l > 0 ? __builtin_elementwise_min(loy + y1l, hiy) : loy;',
|
||||
' int32_t w = fx1 - fx0 + 1, h = fy1 - fy0 + 1;',
|
||||
f' if (w <= 0 || h <= 0 || w * h > {CAP}) {{ w = 0; h = 0; }}',
|
||||
' uint64_t tex = *(const AS4 uint64_t*)(KARGP + 200);',
|
||||
' int32_t n = w * h;',
|
||||
' for (int32_t i = (int32_t)(__builtin_amdgcn_workitem_id_y() * 16u + __builtin_amdgcn_workitem_id_x()); i < n; i += 256)',
|
||||
' {',
|
||||
' int32_t ty = i / w, tx = i - ty * w;',
|
||||
' f4 t_ = tex_fetch(tex, fx0 + tx, fy0 + ty);']
|
||||
for g, info in sorted(group_info.items()):
|
||||
nres = len(info['results'])
|
||||
prm = ''.join(', f1' for _ in info['params'])
|
||||
outs = ', '.join(f'o_ + {k}' for k in range(nres))
|
||||
fill.append(f' {{ uint32_t o_[2]; d4r_ctap_conv{g}(t_{prm}, {outs});')
|
||||
for k in range(nres):
|
||||
fill.append(f' ((AS3 uint32_t*)(d4r_lds + {offs[(g, k)]}))[i] = o_[{k}];')
|
||||
fill.append(' }')
|
||||
fill += [' }',
|
||||
' bar_sync();',
|
||||
' *ox0 = fx0; *oy0 = fy0; *ow = w; *oh = h;',
|
||||
'}']
|
||||
for p in sum((info['params'] for info in group_info.values()), []):
|
||||
if p != '%f1':
|
||||
raise CtileErr('only %f1 is supported as a conversion parameter')
|
||||
return prelude, fill, summary
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
"""If-conversion of pure branch diamonds in PTX (used by ptx2hip.py --ifconv).
|
||||
|
||||
NVIDIA's DLSS kernels branch around small pure arithmetic alternatives (e.g. the inverse tonemap
|
||||
`v < 0.998 ? 1 / (1 - v) : (exp2(...) - 1) / (9 v)`), laid out as
|
||||
|
||||
@%p bra $A; (or @!%p)
|
||||
bra.uni $B;
|
||||
$A: <pure ops> bra.uni $M;
|
||||
$B: <pure ops> (falls through)
|
||||
$M:
|
||||
|
||||
Branches like these split the code into basic blocks that no compiler schedules memory operations across, so
|
||||
independent fetch chains on either side are serialized. This pass rewrites such a diamond into straight-line code:
|
||||
both arms compute into fresh registers and every register an arm assigns is then selected by the predicate. Each lane
|
||||
keeps exactly the values of the arm it would have taken (GPU arithmetic has no traps), so results are bit-identical.
|
||||
Only arms made of register arithmetic are converted (no memory, texture, barrier, shuffle, branch or return; no
|
||||
integer division, which is undefined behaviour in C for some inputs).
|
||||
"""
|
||||
import re
|
||||
|
||||
REG = re.compile(r'%([a-zA-Z]+)(\d+)\b')
|
||||
PURE_BASES = {'add', 'sub', 'mul', 'fma', 'mad', 'div', 'rcp', 'sqrt', 'rsqrt', 'ex2', 'lg2', 'min', 'max', 'neg',
|
||||
'abs', 'cvt', 'mov', 'selp', 'setp', 'and', 'or', 'xor', 'not', 'shl', 'shr', 'set', 'bfi', 'copysign',
|
||||
'sin', 'cos', 'cvta'}
|
||||
SEL = {'f': 'selp.f32', 'r': 'selp.b32', 'rd': 'selp.b64', 'rs': 'selp.b16'}
|
||||
|
||||
|
||||
def _pure(it):
|
||||
if it['kind'] != 'stmt' or it['pred'] is not None:
|
||||
return False
|
||||
op = it['op']
|
||||
base = op.split('.')[0]
|
||||
if base not in PURE_BASES:
|
||||
return False
|
||||
if base == 'div' and not op.endswith('.f32') and 'approx' not in op:
|
||||
return False # integer division (and IEEE f32 division, which the translator rejects anyway)
|
||||
if base == 'cvta':
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _target(it):
|
||||
if it['kind'] == 'stmt' and it['op'].startswith('bra'):
|
||||
return it['rest'].strip().rstrip(';').strip()
|
||||
return None
|
||||
|
||||
|
||||
def apply(body, tokenize, parse, split_operands):
|
||||
items = parse(body, tokenize)
|
||||
# label -> number of branches targeting it
|
||||
refs = {}
|
||||
for it in items:
|
||||
t = _target(it)
|
||||
if t:
|
||||
refs[t] = refs.get(t, 0) + 1
|
||||
# fresh register numbers per kind
|
||||
top = {}
|
||||
for m in REG.finditer(body):
|
||||
k, n = m.group(1), int(m.group(2))
|
||||
top[k] = max(top.get(k, 0), n)
|
||||
fresh = {k: v + 100000 for k, v in top.items()}
|
||||
|
||||
def new_reg(kind):
|
||||
fresh[kind] = fresh.get(kind, 100000) + 1
|
||||
return f'%{kind}{fresh[kind]}'
|
||||
|
||||
use_sites = {}
|
||||
for x in items:
|
||||
if x['kind'] == 'stmt':
|
||||
for r in x['src']:
|
||||
use_sites.setdefault(r, []).append(id(x))
|
||||
|
||||
def uses_outside(r, arm_ids):
|
||||
return any(u not in arm_ids for u in use_sites.get(r, []))
|
||||
|
||||
defined_before = set()
|
||||
out, i, n_conv = [], 0, 0
|
||||
while i < len(items):
|
||||
it = items[i]
|
||||
conv = None
|
||||
if it['kind'] == 'stmt' and it['op'].startswith('bra') and it['pred'] is not None:
|
||||
conv = _match(items, i, refs)
|
||||
if conv is None:
|
||||
if it['kind'] == 'stmt':
|
||||
defined_before.update(it['dst'])
|
||||
out.append(_text(it))
|
||||
i += 1
|
||||
continue
|
||||
neg, pred, arm_a, arm_b, merge_idx = conv
|
||||
lines, last = [], [{}, {}]
|
||||
for side, arm in enumerate((arm_a, arm_b)):
|
||||
ren = {}
|
||||
for st in arm:
|
||||
op, rest = st['text'].split(None, 1)
|
||||
ops = split_operands(rest)
|
||||
if len(st['dst']) != 1 or ops[0] != st['dst'][0]:
|
||||
raise ValueError('unexpected destination form: ' + st['text'])
|
||||
srcs = [REG.sub(lambda mm: ren.get(mm.group(0), mm.group(0)), o) for o in ops[1:]]
|
||||
d = st['dst'][0]
|
||||
kind = REG.match(d).group(1)
|
||||
if kind not in SEL:
|
||||
raise ValueError('predicate assigned in a diamond arm: ' + st['text'])
|
||||
nr = new_reg(kind)
|
||||
ren[d] = nr
|
||||
lines.append(f"{op} {', '.join([nr] + srcs)}")
|
||||
last[side] = ren
|
||||
# only registers read outside the two arms need a select (arm temporaries are dropped)
|
||||
arm_ids = {id(x) for x in arm_a} | {id(x) for x in arm_b}
|
||||
assigned = sorted(r for r in set(last[0]) | set(last[1]) if uses_outside(r, arm_ids))
|
||||
cond = pred
|
||||
for r in assigned:
|
||||
kind = REG.match(r).group(1)
|
||||
a = last[0].get(r, r if r in defined_before else last[1].get(r))
|
||||
b = last[1].get(r, r if r in defined_before else last[0].get(r))
|
||||
# selp d, a, b, p: d = p ? a : b ; arm A is taken when (p xor neg)
|
||||
if neg:
|
||||
a, b = b, a
|
||||
lines.append(f'{SEL[kind]} {r}, {a}, {b}, {cond}')
|
||||
defined_before.update(assigned)
|
||||
out.extend(l + ';' for l in lines)
|
||||
n_conv += 1
|
||||
i = merge_idx # the merge label itself is emitted by the main loop
|
||||
return '\n'.join(out), n_conv
|
||||
|
||||
|
||||
def _text(it):
|
||||
if it['kind'] == 'open':
|
||||
return '{'
|
||||
if it['kind'] == 'close':
|
||||
return '}'
|
||||
if it['kind'] == 'label':
|
||||
return it['text'] + ':'
|
||||
return it['text'] + ';'
|
||||
|
||||
|
||||
def _match(items, i, refs):
|
||||
"""@p bra A; bra.uni B; A: ... bra.uni M; B: ... M: -> (neg, pred, armA, armB, index of M)"""
|
||||
br = items[i]
|
||||
m = re.match(r'@(!?)(%p\d+)\s', br['text'])
|
||||
if not m:
|
||||
return None
|
||||
neg, pred = m.group(1) == '!', m.group(2)
|
||||
a_lab = _target(br)
|
||||
if i + 2 >= len(items):
|
||||
return None
|
||||
j = items[i + 1]
|
||||
if not (j['kind'] == 'stmt' and j['op'].startswith('bra') and j['pred'] is None):
|
||||
return None
|
||||
b_lab = _target(j)
|
||||
if refs.get(a_lab) != 1 or refs.get(b_lab) != 1:
|
||||
return None
|
||||
|
||||
def arm(start, label):
|
||||
if not (items[start]['kind'] == 'label' and items[start]['text'] == label):
|
||||
return None
|
||||
body, k = [], start + 1
|
||||
while k < len(items):
|
||||
x = items[k]
|
||||
if x['kind'] == 'label':
|
||||
return body, k, None
|
||||
if x['kind'] in ('open', 'close'):
|
||||
return None
|
||||
if x['kind'] == 'stmt' and x['op'].startswith('bra'):
|
||||
if x['pred'] is not None:
|
||||
return None
|
||||
return body, k + 1, _target(x)
|
||||
if not _pure(x):
|
||||
return None
|
||||
body.append(x)
|
||||
k += 1
|
||||
return None
|
||||
|
||||
first = arm(i + 2, a_lab)
|
||||
order = 'AB'
|
||||
if first is None:
|
||||
first = arm(i + 2, b_lab)
|
||||
order = 'BA'
|
||||
if first is None:
|
||||
return None
|
||||
body1, k, jump1 = first
|
||||
if jump1 is None:
|
||||
return None
|
||||
second = arm(k, b_lab if order == 'AB' else a_lab)
|
||||
if second is None:
|
||||
return None
|
||||
body2, k2, jump2 = second
|
||||
if jump2 is not None and jump2 != jump1:
|
||||
return None # the second arm may also jump to the merge label explicitly
|
||||
if not (k2 < len(items) and items[k2]['kind'] == 'label' and items[k2]['text'] == jump1):
|
||||
return None
|
||||
arm_a, arm_b = (body1, body2) if order == 'AB' else (body2, body1)
|
||||
return neg, pred, arm_a, arm_b, k2
|
||||
|
||||
|
||||
def sink_stores(body, tokenize, parse):
|
||||
"""Drop labels nothing branches to, then move each st.shared / st.local down to just before the next statement
|
||||
that could observe or reorder it: a shared/local load or any other store, a barrier, shuffle, atomic, label,
|
||||
branch or return, or a statement that writes one of the store's registers. Stores keep their relative order, so
|
||||
every memory location sees the same sequence of writes; texture loads (read-only data) and arithmetic move ahead
|
||||
of them, which lets independent fetch chains issue together. Returns (body, number of stores moved)."""
|
||||
items = parse(body, tokenize)
|
||||
refs = set()
|
||||
for it in items:
|
||||
t = _target(it)
|
||||
if t:
|
||||
refs.add(t)
|
||||
items = [it for it in items if not (it['kind'] == 'label' and it['text'] not in refs)]
|
||||
|
||||
def is_fence(it):
|
||||
if it['kind'] != 'stmt':
|
||||
return True # labels and scope braces
|
||||
op = it['op']
|
||||
base = op.split('.')[0]
|
||||
if base in ('bar', 'shfl', 'atom', 'red', 'bra', 'ret', 'membar', 'fence', 'st', 'sust', 'd4r'):
|
||||
return True
|
||||
if base == 'ld' and ('.shared' in op or '.local' in op):
|
||||
return True
|
||||
return False
|
||||
|
||||
moved = 0
|
||||
i = len(items) - 1
|
||||
while i >= 0:
|
||||
it = items[i]
|
||||
if it['kind'] == 'stmt' and it['pred'] is None and it['op'].startswith('st.') and ('.shared' in it['op'] or '.local' in it['op']):
|
||||
regs = set(it['src'])
|
||||
j = i + 1
|
||||
while j < len(items) and not is_fence(items[j]) and not (regs & set(items[j]['dst'])):
|
||||
j += 1
|
||||
if j > i + 1:
|
||||
# after the pop the fence that stopped the scan is at j - 1: the store goes right before it
|
||||
items.insert(j - 1, items.pop(i))
|
||||
moved += 1
|
||||
i -= 1
|
||||
return '\n'.join(_text(it) for it in items), moved
|
||||
Executable
+123
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""in_fuse.py IN.hip > in_unit.h: the hiluma input kernel (translated by ptx2hip.py) as a device function
|
||||
|
||||
in_unit(KARGP, gx, gy, out[4])
|
||||
|
||||
computing what thread (gx, gy) of the original launch (blocks of 16x16) writes: the 4 halves at network pixel
|
||||
(gx >> 2, gy), channels 4 (gx & 3) .. 4 (gx & 3) + 3 (used by in_regs.hip).
|
||||
|
||||
Changes, all value-preserving:
|
||||
- thread ids: workgroup/workitem ids -> gx >> 4, gx & 15, gy >> 4, gy & 15 (the kernel only uses gx = 16 wg_x + wi_x,
|
||||
gy = 16 wg_y + wi_y and gx & 1, gx & 2);
|
||||
- the per-thread LDS scratch (4 planes of 3 floats at slot 12 * (16 wi_y + wi_x), written in straight-line code
|
||||
before any branch, read at fixed offsets and once at plane r14 in 0..3) -> 12 registers + a 4-way select;
|
||||
- the 32-byte local depot (4 float pairs written in straight-line code, one pair read at plane r14) -> 8 registers
|
||||
+ a 4-way select;
|
||||
- the 4 final f16 stores -> out[0..3].
|
||||
Refuses any input that does not match these patterns exactly.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
src = open(sys.argv[1]).read()
|
||||
lines = src.split('\n')
|
||||
k0 = next(i for i, l in enumerate(lines) if l.startswith('extern "C" __attribute__((global))'))
|
||||
assert lines[k0 + 1] == '{'
|
||||
end = len(lines) - 1
|
||||
while lines[end].strip() != '}':
|
||||
end -= 1
|
||||
body = lines[k0 + 2:end]
|
||||
first_label = next(i for i, l in enumerate(body) if re.match(r'^L__BB0_\d+:;$', l))
|
||||
|
||||
out = []
|
||||
SUBS = {
|
||||
'r39 = (uint32_t)((uint32_t)__builtin_amdgcn_workgroup_id_x());': 'r39 = gx >> 4;',
|
||||
'r41 = (uint32_t)((uint32_t)__builtin_amdgcn_workitem_id_x());': 'r41 = gx & 15u;',
|
||||
'r46 = (uint32_t)((uint32_t)__builtin_amdgcn_workgroup_id_y());': 'r46 = gy >> 4;',
|
||||
'r48 = (uint32_t)((uint32_t)__builtin_amdgcn_workitem_id_y());': 'r48 = gy & 15u;',
|
||||
'r191 = (uint32_t)((uint32_t)__builtin_amdgcn_workitem_id_y());': 'r191 = 0u;',
|
||||
'r190 = (uint32_t)((uint32_t)__builtin_amdgcn_workitem_id_x());': 'r190 = 0u;',
|
||||
'r90 = (uint32_t)((uint32_t)(uintptr_t)(AS3 uint8_t*)sharedWarpedSpaceHistoryYcocg);': 'r90 = 0u; // (LDS scratch replaced by registers)',
|
||||
}
|
||||
seen = set()
|
||||
ST = re.compile(r'^ \*\(AS3 float\*\)\(uintptr_t\)\(r20 \+ (\d+)u\) = \(float\)\((f\d+)\);$')
|
||||
LD = re.compile(r'^ (f\d+) = \*\(AS3 float\*\)\(uintptr_t\)\(r20 \+ (\d+)u\);$')
|
||||
LD153 = re.compile(r'^ (f\d+) = \*\(AS3 float\*\)\(uintptr_t\)\(r153 \+ 0u\);$')
|
||||
STORE = re.compile(r'^ \*\(uint16_t\*\)\(uintptr_t\)\(rd48 \+ (\d)\) = \(uint16_t\)\((rs\d+)\);$')
|
||||
stored = set()
|
||||
nst = 0
|
||||
# the 32-byte local depot: 4 float pairs (offsets 0..28) written through rd6 / rd59 / rd61 = depot + 0 in
|
||||
# straight-line code, read once as the pair at rd61 + 8 r14 (r14 in 0..3) -> 8 registers + a 4-way select
|
||||
DST = re.compile(r'^ \*\(AS5 float\*\)\(uintptr_t\)\(uint32_t\)\((rd6|rd59|rd61) \+ (\d+)\) = \(float\)\((f\d+)\);$')
|
||||
DLD = re.compile(r'^ (f\d+) = \*\(AS5 float\*\)\(uintptr_t\)\(uint32_t\)\(rd44 \+ ([04])\);$')
|
||||
dstored = set()
|
||||
for i, l in enumerate(body):
|
||||
t = l.strip()
|
||||
if t.startswith('const AS4 uint8_t* KARGP =') or t.startswith('const uint64_t KARGU ='):
|
||||
continue
|
||||
if t in SUBS:
|
||||
out.append(' ' + SUBS[t])
|
||||
seen.add(t)
|
||||
continue
|
||||
m = ST.match(l)
|
||||
if m:
|
||||
off = int(m.group(1))
|
||||
assert i < first_label and off % 4 == 0 and off // 3072 < 4 and (off % 3072) // 4 < 3, l
|
||||
out.append(f' shv{off // 3072}_{(off % 3072) // 4} = {m.group(2)};')
|
||||
stored.add(off)
|
||||
continue
|
||||
m = LD.match(l)
|
||||
if m:
|
||||
off = int(m.group(2))
|
||||
assert off in stored, f'load before store: {l}'
|
||||
out.append(f' {m.group(1)} = shv{off // 3072}_{(off % 3072) // 4};')
|
||||
continue
|
||||
m = LD153.match(l)
|
||||
if m:
|
||||
assert all(o in stored for o in (0, 3072, 6144, 9216)), 'plane loads need all planes stored'
|
||||
out.append(f' {m.group(1)} = r14 == 0u ? shv0_0 : r14 == 1u ? shv1_0 : r14 == 2u ? shv2_0 : shv3_0; // r14 in 0..3')
|
||||
continue
|
||||
m = DST.match(l)
|
||||
if m:
|
||||
off = int(m.group(2))
|
||||
assert i < first_label and off % 4 == 0 and off < 32, l
|
||||
out.append(f' dpv{off // 8}_{(off % 8) // 4} = {m.group(3)};')
|
||||
dstored.add(off)
|
||||
continue
|
||||
m = DLD.match(l)
|
||||
if m:
|
||||
assert len(dstored) == 8, 'depot read before all pairs are stored'
|
||||
j = int(m.group(2)) // 4
|
||||
out.append(f' {m.group(1)} = r14 == 0u ? dpv0_{j} : r14 == 1u ? dpv1_{j} : r14 == 2u ? dpv2_{j} : dpv3_{j}; // r14 in 0..3')
|
||||
continue
|
||||
m = STORE.match(l)
|
||||
if m:
|
||||
out.append(f' out[{int(m.group(1)) // 2}] = {m.group(2)};')
|
||||
nst += 1
|
||||
continue
|
||||
assert 'AS3' not in l and 'workitem_id' not in l and 'workgroup_id' not in l, f'unhandled: {l}'
|
||||
assert not re.search(r'\(rd48 \+', l), f'unhandled store: {l}'
|
||||
out.append(l)
|
||||
assert len(seen) == len(SUBS), set(SUBS) - seen
|
||||
assert nst == 4, nst
|
||||
assert len(stored) == 12, sorted(stored)
|
||||
assert len(dstored) == 8, sorted(dstored)
|
||||
for k in (' rd6 = (uint64_t)(l2_SPL + (uint64_t)0ull);', ' rd59 = (uint64_t)(l2_SPL + (uint64_t)0ull);',
|
||||
' rd61 = (uint64_t)(l2_SPL + (uint64_t)0ull);', ' rd43 = (uint64_t)((int64_t)(int32_t)r14 * (int64_t)(int32_t)8);',
|
||||
' rd44 = (uint64_t)(rd61 + rd43);'):
|
||||
assert k in body, k
|
||||
# r14 must be 2 r58 + (1 - r65) with r58, r65 parity bits (checked textually)
|
||||
assert ' r14 = (uint32_t)(r68 + r67);' in body and ' r68 = (uint32_t)(r58 << (1u & 31));' in body
|
||||
assert ' r67 = (uint32_t)(r26 - r65);' in body and ' r26 = (uint32_t)(1u);' in body
|
||||
|
||||
hdr = ['// Generated by kernels/native/in_fuse.py from ' + sys.argv[1].split('/')[-1] + ' (do not edit).',
|
||||
'#pragma once',
|
||||
'#include "native_rt.h"',
|
||||
'// Thread (gx, gy) of hiluma_engine_input_depthinv_mvhi_hdr_v2_rel (16x16 blocks): the 4 halves it stores at',
|
||||
'// network pixel (gx >> 2, gy), channels 4 (gx & 3) ... KARGP: the input kernel\'s 248-byte parameter block.',
|
||||
'__attribute__((device)) inline __attribute__((always_inline)) void in_unit(const AS4 uint8_t* KARGP, uint32_t gx, uint32_t gy, uint16_t out[4])',
|
||||
'{',
|
||||
' const uint64_t KARGU = (uint64_t)(uintptr_t)KARGP;',
|
||||
' float ' + ', '.join(f'shv{p}_{j}' for p in range(4) for j in range(3)) + ';',
|
||||
' float ' + ', '.join(f'dpv{p}_{j}' for p in range(4) for j in range(2)) + ';']
|
||||
print('\n'.join(hdr + out + ['}']))
|
||||
@@ -0,0 +1,22 @@
|
||||
// hiluma input kernel rebuilt from in_unit.h (kernels/native/in_fuse.py on in_ifs.hip): the translated kernel's
|
||||
// per-thread LDS scratch and 32-byte local array live in registers (the local array compiled to scratch-memory
|
||||
// flat stores/loads; -21 % in the replay bench). Same launch (16x16 blocks), same output.
|
||||
// Build: kernels/build.sh (generates in_unit.h from the extracted PTX); by hand: WAVE=32 build_native.sh in_regs.hip OUT.hsaco
|
||||
#include "in_unit.h"
|
||||
struct __attribute__((aligned(8))) hiluma_engine_input_depthinv_mvhi_hdr_v2_rel_params { uint8_t b[248]; };
|
||||
extern "C" __attribute__((global)) __attribute__((amdgpu_flat_work_group_size(1, 256))) void hiluma_engine_input_depthinv_mvhi_hdr_v2_rel(hiluma_engine_input_depthinv_mvhi_hdr_v2_rel_params)
|
||||
{
|
||||
const AS4 uint8_t* KARGP = (const AS4 uint8_t*)__builtin_amdgcn_kernarg_segment_ptr();
|
||||
const uint32_t gx = __builtin_amdgcn_workgroup_id_x() * 16u + __builtin_amdgcn_workitem_id_x();
|
||||
const uint32_t gy = __builtin_amdgcn_workgroup_id_y() * 16u + __builtin_amdgcn_workitem_id_y();
|
||||
uint16_t o[4];
|
||||
in_unit(KARGP, gx, gy, o);
|
||||
// the original's store: network pixel (gx >> 2, gy), channels 4 (gx & 3) .. + 3; row stride W (u32 at +64),
|
||||
// buffer at +240 (16 halves per pixel)
|
||||
const uint32_t W = *(const AS4 uint32_t*)(KARGP + 64);
|
||||
uint16_t* d = (uint16_t*)*(const AS4 uint64_t*)(KARGP + 240) + ((size_t)(W * gy + (gx >> 2)) * 16 + 4 * (gx & 3));
|
||||
d[0] = o[0];
|
||||
d[1] = o[1];
|
||||
d[2] = o[2];
|
||||
d[3] = o[3];
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// M texture-translation helpers. Kept separate from K runtime code.
|
||||
#pragma once
|
||||
#include "ptx_rt.h"
|
||||
extern "C" __attribute__((device)) h1 __ocml_tanh_f16(h1);
|
||||
extern "C" __attribute__((device)) h2 m_min2(h2, h2) __asm("llvm.minimumnum.v2f16");
|
||||
extern "C" __attribute__((device)) h2 m_max2(h2, h2) __asm("llvm.maximumnum.v2f16");
|
||||
RT uint32_t prmt32(uint32_t a, uint32_t b, uint32_t select)
|
||||
{
|
||||
uint32_t out = 0;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
uint32_t s = (select >> (4 * i)) & 15u;
|
||||
uint32_t byte = ((s & 4u) ? b : a) >> (8 * (s & 3u));
|
||||
byte = (s & 8u) ? ((byte & 128u) ? 255u : 0u) : (byte & 255u);
|
||||
out |= byte << (8 * i);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
RT uint32_t shfl_idx(uint32_t input, uint32_t index, uint32_t opts, bool* in_bounds)
|
||||
{
|
||||
uint32_t mask = (opts >> 8) & 31u;
|
||||
uint32_t subsection = laneid() & mask;
|
||||
uint32_t end = subsection | (opts & ~mask & 31u);
|
||||
uint32_t src = subsection | (index & ~mask & 31u);
|
||||
bool valid = src <= end;
|
||||
if (in_bounds) *in_bounds = valid;
|
||||
return __builtin_amdgcn_ds_bpermute(4u * (warp_base() | (valid ? src : laneid())), input);
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
// Runtime for natively compiled DLSS texture kernels: PTX semantics (ptx_rt.h) plus surface stores
|
||||
// (tex/tex_common.h, with ZLUDA's generic raw-store fallback ported here, since no ZLUDA bitcode is linked).
|
||||
#pragma once
|
||||
#include "ptx_rt.h"
|
||||
#include "../tex/tex_common.h"
|
||||
|
||||
extern "C" __attribute__((device)) v4f __ockl_image_load_2D(tsharp_t* image, v2i coord);
|
||||
|
||||
namespace zsurf {
|
||||
// zluda_ptx_impl.cpp surface_* helpers (raw sust.b stores), verbatim semantics
|
||||
RT int channel_bytes(int dt)
|
||||
{
|
||||
switch (dt)
|
||||
{
|
||||
case 0: case 2: case 8: case 11: return 1;
|
||||
case 1: case 3: case 9: case 12: case 14: return 2;
|
||||
case 10: case 13: case 15: return 4;
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
RT int channel_count(int order)
|
||||
{
|
||||
switch (order)
|
||||
{
|
||||
case 3: case 4: case 5: return 2;
|
||||
case 6: case 12: return 3;
|
||||
case 7: case 8: case 9: case 10: case 11: case 13: case 14: case 15: return 4;
|
||||
default: return 1;
|
||||
}
|
||||
}
|
||||
RT int memory_channel_lane(int order, int channel)
|
||||
{
|
||||
switch (order)
|
||||
{
|
||||
case 0: return 3;
|
||||
case 5: return channel == 0 ? 0 : 3;
|
||||
case 9: case 15: return channel == 0 ? 2 : channel == 2 ? 0 : channel;
|
||||
case 10: return channel == 0 ? 3 : channel - 1;
|
||||
case 11: return 3 - channel;
|
||||
default: return channel;
|
||||
}
|
||||
}
|
||||
RT uint32_t float_to_half_bits(float value)
|
||||
{
|
||||
uint32_t bits = __builtin_bit_cast(uint32_t, value);
|
||||
uint32_t sign = (bits >> 16) & 0x8000u, exponent = (bits >> 23) & 0xffu, mantissa = bits & 0x7fffffu;
|
||||
if (exponent == 0xff)
|
||||
return sign | 0x7c00u | (mantissa != 0 ? 0x200u | (mantissa >> 13) : 0u);
|
||||
int half_exponent = int(exponent) - 112;
|
||||
if (half_exponent >= 31)
|
||||
return sign | 0x7c00u;
|
||||
uint32_t shift = 13;
|
||||
if (half_exponent <= 0)
|
||||
{
|
||||
if (half_exponent < -10)
|
||||
return sign;
|
||||
mantissa |= 0x800000u;
|
||||
shift = uint32_t(14 - half_exponent);
|
||||
half_exponent = 0;
|
||||
}
|
||||
uint32_t half_mantissa = mantissa >> shift, remainder = mantissa & ((1u << shift) - 1u), halfway = 1u << (shift - 1u);
|
||||
uint32_t result = sign | (uint32_t(half_exponent) << 10);
|
||||
result += half_mantissa & (half_exponent == 0 ? 0x7ffu : 0x3ffu);
|
||||
if (remainder > halfway || (remainder == halfway && (half_mantissa & 1u)))
|
||||
result++;
|
||||
return result;
|
||||
}
|
||||
RT int32_t clampi(int32_t v, int32_t lo, int32_t hi) { return v < lo ? lo : (v > hi ? hi : v); }
|
||||
RT float decode(uint32_t bits, int dt)
|
||||
{
|
||||
switch (dt)
|
||||
{
|
||||
case 0: return float(clampi(int32_t(int8_t(bits)), -127, 127)) / 127.0f;
|
||||
case 1: return float(clampi(int32_t(int16_t(bits)), -32767, 32767)) / 32767.0f;
|
||||
case 2: return float(int32_t(bits & 0xffu)) / 255.0f;
|
||||
case 3: return float(int32_t(bits & 0xffffu)) / 65535.0f;
|
||||
case 8: return __builtin_bit_cast(float, int32_t(int8_t(bits)));
|
||||
case 9: return __builtin_bit_cast(float, int32_t(int16_t(bits)));
|
||||
case 14: return half_bits_to_float(bits);
|
||||
default: return __builtin_bit_cast(float, bits);
|
||||
}
|
||||
}
|
||||
RT uint32_t encode(float value, int dt)
|
||||
{
|
||||
switch (dt)
|
||||
{
|
||||
case 0: return uint32_t(clampi(int32_t(__builtin_floorf(value * 127.0f + 0.5f)), -127, 127)) & 0xffu;
|
||||
case 1: return uint32_t(clampi(int32_t(__builtin_floorf(value * 32767.0f + 0.5f)), -32767, 32767)) & 0xffffu;
|
||||
case 2: return uint32_t(clampi(int32_t(__builtin_floorf(value * 255.0f + 0.5f)), 0, 255));
|
||||
case 3: return uint32_t(clampi(int32_t(__builtin_floorf(value * 65535.0f + 0.5f)), 0, 65535));
|
||||
case 14: return float_to_half_bits(value);
|
||||
default: return __builtin_bit_cast(uint32_t, value);
|
||||
}
|
||||
}
|
||||
RT int lane_channel(int order, int lane, int channels)
|
||||
{
|
||||
switch (order)
|
||||
{
|
||||
case 0: return lane == 3 ? 0 : -1;
|
||||
case 5: return lane == 0 ? 0 : (lane == 3 ? 1 : -1);
|
||||
case 9: case 15: return lane == 0 ? 2 : (lane == 2 ? 0 : lane);
|
||||
case 10: return lane == 3 ? 0 : lane + 1;
|
||||
case 11: return 3 - lane;
|
||||
default: return lane < channels ? lane : -1;
|
||||
}
|
||||
}
|
||||
RT uint32_t sel_channel(uint32_t c0, uint32_t c1, uint32_t c2, uint32_t c3, int ch)
|
||||
{
|
||||
return ch == 0 ? c0 : (ch == 1 ? c1 : (ch == 2 ? c2 : c3));
|
||||
}
|
||||
RT float sel_lane(v4f v, int lane) { return lane == 0 ? v.x : (lane == 1 ? v.y : (lane == 2 ? v.z : v.w)); }
|
||||
RT uint32_t merge_channel(uint32_t bits, int channel, int cb, uint32_t data, int byte_offset, int byte_count)
|
||||
{
|
||||
for (int b = 0; b < 4; b++)
|
||||
{
|
||||
int relative = channel * cb + b - byte_offset;
|
||||
if (b < cb && relative >= 0 && relative < byte_count)
|
||||
{
|
||||
uint32_t byte = (data >> (8 * relative)) & 0xffu;
|
||||
bits = (bits & ~(0xffu << (8 * b))) | (byte << (8 * b));
|
||||
}
|
||||
}
|
||||
return bits;
|
||||
}
|
||||
RT float lane_value(uint32_t c0, uint32_t c1, uint32_t c2, uint32_t c3, int order, int lane, int channels, int dt)
|
||||
{
|
||||
int ch = lane_channel(order, lane, channels);
|
||||
return ch < 0 ? 0.0f : decode(sel_channel(c0, c1, c2, c3, ch), dt);
|
||||
}
|
||||
__attribute__((device, noinline)) static void store_raw_2d(tsharp_t* image, int32_t cx, int32_t cy, uint32_t data, int byte_count)
|
||||
{
|
||||
int dt = __ockl_image_channel_data_type_2D(image), order = __ockl_image_channel_order_2D(image);
|
||||
int cb = channel_bytes(dt);
|
||||
if (cb == 0 || cx < 0)
|
||||
return;
|
||||
int channels = channel_count(order), pixel_bytes = cb * channels;
|
||||
for (int consumed = 0; consumed < byte_count;)
|
||||
{
|
||||
int byte_x = cx + consumed, pixel_x = byte_x / pixel_bytes, byte_offset = byte_x - pixel_x * pixel_bytes;
|
||||
int chunk = pixel_bytes - byte_offset;
|
||||
if (chunk > byte_count - consumed)
|
||||
chunk = byte_count - consumed;
|
||||
uint32_t fragment = data >> (8 * consumed);
|
||||
uint32_t c0 = 0, c1 = 0, c2 = 0, c3 = 0;
|
||||
if (byte_offset != 0 || chunk < pixel_bytes)
|
||||
{
|
||||
v4f loaded = __ockl_image_load_2D(image, (v2i){pixel_x, cy});
|
||||
c0 = encode(sel_lane(loaded, memory_channel_lane(order, 0)), dt);
|
||||
c1 = encode(sel_lane(loaded, memory_channel_lane(order, 1)), dt);
|
||||
c2 = encode(sel_lane(loaded, memory_channel_lane(order, 2)), dt);
|
||||
c3 = encode(sel_lane(loaded, memory_channel_lane(order, 3)), dt);
|
||||
}
|
||||
c0 = merge_channel(c0, 0, cb, fragment, byte_offset, chunk);
|
||||
c1 = merge_channel(c1, 1, cb, fragment, byte_offset, chunk);
|
||||
c2 = merge_channel(c2, 2, cb, fragment, byte_offset, chunk);
|
||||
c3 = merge_channel(c3, 3, cb, fragment, byte_offset, chunk);
|
||||
__ockl_image_store_2D(image, (v2i){pixel_x, cy},
|
||||
(v4f){lane_value(c0, c1, c2, c3, order, 0, channels, dt), lane_value(c0, c1, c2, c3, order, 1, channels, dt),
|
||||
lane_value(c0, c1, c2, c3, order, 2, channels, dt), lane_value(c0, c1, c2, c3, order, 3, channels, dt)});
|
||||
consumed += chunk;
|
||||
}
|
||||
}
|
||||
} // namespace zsurf
|
||||
|
||||
extern "C" __attribute__((device)) void __zluda_ptx_impl_surfobj_b_2d_b32_zero(uint64_t surface, v2i coord, uint32_t value)
|
||||
{
|
||||
zsurf::store_raw_2d((tsharp_t*)(uintptr_t)surface, coord.x, coord.y, value, 4);
|
||||
}
|
||||
extern "C" __attribute__((device)) void __zluda_ptx_impl_surfobj_b_2d_v2_b16_zero(uint64_t surface, v2i coord, v2s value)
|
||||
{
|
||||
zsurf::store_raw_2d((tsharp_t*)(uintptr_t)surface, coord.x, coord.y, uint32_t(uint16_t(value.x)) | (uint32_t(uint16_t(value.y)) << 16), 4);
|
||||
}
|
||||
|
||||
// A sample that every lane takes with the same handle, coordinates and lod (e.g. a 1x1 exposure texture at (0.5,
|
||||
// 0.5)): only the first active lane samples and the result is broadcast with readfirstlane. Same values, but the
|
||||
// texture unit processes one lane instead of the whole wave.
|
||||
RT f4 tex_level_uniform(uint64_t t, float x, float y, float lod)
|
||||
{
|
||||
f4 r = (f4){0.0f, 0.0f, 0.0f, 0.0f};
|
||||
if (wave_lane() == (uint32_t)__builtin_ctzll(__builtin_amdgcn_read_exec()))
|
||||
r = tex_level(t, x, y, lod);
|
||||
return (f4){asf(__builtin_amdgcn_readfirstlane(asu(r[0]))), asf(__builtin_amdgcn_readfirstlane(asu(r[1]))),
|
||||
asf(__builtin_amdgcn_readfirstlane(asu(r[2]))), asf(__builtin_amdgcn_readfirstlane(asu(r[3])))};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// Structured path-B tile fill. Same color/depth values and 24-column LDS layout
|
||||
// as the original kernel; work-items traverse x/y directly without integer division.
|
||||
#pragma once
|
||||
|
||||
#ifndef D4R_OUTPUT_TILE_PITCH
|
||||
#define D4R_OUTPUT_TILE_PITCH 24
|
||||
#endif
|
||||
static_assert(D4R_OUTPUT_TILE_PITCH >= 24 && D4R_OUTPUT_TILE_PITCH <= 32, "LDS tile pitch capacity");
|
||||
|
||||
RT void output_color_pack(f4 color, float exposure, uint32_t* a, uint32_t* b)
|
||||
{
|
||||
const float r = fmaxn(color[0], 0.0f), g = fmaxn(color[1], 0.0f), blue = fmaxn(color[2], 0.0f);
|
||||
const float qr = r * 0.25f, hg = g * 0.5f, qb = blue * 0.25f;
|
||||
const float y = exposure * ((qr + hg) + qb);
|
||||
const float co = __builtin_fmaf(r, 0.5f, g * 0.0f) - blue * 0.5f;
|
||||
const float cg = (hg - qr) - qb;
|
||||
float weight;
|
||||
if (y < 1000.0f)
|
||||
weight = rcp_a(y + 1.0f);
|
||||
else
|
||||
{
|
||||
const float z = __builtin_fmaf(y, 9.0f, 1.0f);
|
||||
const float ln = lg2_a(z) * asf(0x3F317218u);
|
||||
const float q = div_a(ln, 9.0f) + asf(0xBC4FAE29u);
|
||||
weight = rcp_a(y) * q;
|
||||
}
|
||||
*a = pack16(ashu((h1)(y * weight)), ashu((h1)((exposure * co) * weight)));
|
||||
*b = pack16(ashu((h1)((exposure * cg) * weight)), ashu((h1)color[3]));
|
||||
}
|
||||
|
||||
RT int output_tile_coord(int relative, int lo, int hi)
|
||||
{
|
||||
return relative > 0 ? __builtin_elementwise_min(lo + relative, hi) : lo;
|
||||
}
|
||||
|
||||
RT void output_tile_fill(const AS4 uint8_t* args, int x0, int y0, int width, int height,
|
||||
float exposure, AS3 uint8_t* color_tile, AS3 uint8_t* depth_tile)
|
||||
{
|
||||
const uint64_t color = *(const AS4 uint64_t*)(args + 200);
|
||||
const uint64_t depth = *(const AS4 uint64_t*)(args + 248);
|
||||
const int cx0 = *(const AS4 int32_t*)(args + 208), cy0 = *(const AS4 int32_t*)(args + 212);
|
||||
const int cx1 = *(const AS4 int32_t*)(args + 216), cy1 = *(const AS4 int32_t*)(args + 220);
|
||||
const int dx0 = *(const AS4 int32_t*)(args + 96), dy0 = *(const AS4 int32_t*)(args + 100);
|
||||
const int dx1 = *(const AS4 int32_t*)(args + 104), dy1 = *(const AS4 int32_t*)(args + 108);
|
||||
bool fill_depth = true;
|
||||
#ifdef D4R_K_NO_DEPTH_TILE
|
||||
// Pinned K LUT has |slope|<=1. Its 4x3 color stencil stays in the four-texel
|
||||
// halo for positive upsampling scale<=1, so no color read aliases depth LDS.
|
||||
// Keep the original fill outside the proven geometry. Input depth is unchanged.
|
||||
const float scale_x = *(const AS4 float*)(args + 40), scale_y = *(const AS4 float*)(args + 44);
|
||||
fill_depth = !(scale_x > 0.0f && scale_x <= 1.0f && scale_y > 0.0f && scale_y <= 1.0f);
|
||||
#endif
|
||||
// Path B selects a footprint at most 24 x 24. At each iteration 0<=x<width,
|
||||
// 0<=y<height: color accesses stay within 4608 bytes, depth within 2304 bytes.
|
||||
for (int y = __builtin_amdgcn_workitem_id_y(); y < height; y += 16)
|
||||
for (int x = __builtin_amdgcn_workitem_id_x(); x < width; x += 16)
|
||||
{
|
||||
const int sx = x0 + x, sy = y0 + y;
|
||||
const f4 c = tex_fetch(color, output_tile_coord(sx, cx0, cx1), output_tile_coord(sy, cy0, cy1));
|
||||
uint32_t a, b;
|
||||
output_color_pack(c, exposure, &a, &b);
|
||||
AS3 uint32_t* dst = (AS3 uint32_t*)(color_tile + y * (8 * D4R_OUTPUT_TILE_PITCH) + x * 8);
|
||||
dst[0] = a;
|
||||
dst[1] = b;
|
||||
if (fill_depth)
|
||||
{
|
||||
const f4 d = tex_fetch(depth, output_tile_coord(sx, dx0, dx1), output_tile_coord(sy, dy0, dy1));
|
||||
*(AS3 float*)(depth_tile + y * (4 * D4R_OUTPUT_TILE_PITCH) + x * 4) = d[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+857
@@ -0,0 +1,857 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Translate one NVIDIA PTX kernel (DLSS texture kernels) into native HIP C++.
|
||||
|
||||
usage: ptx2hip.py MODULE.ptx KERNEL > KERNEL.hip
|
||||
|
||||
The output is a plain HIP kernel with the same name and parameter block. Every PTX instruction becomes
|
||||
the C expression of ZLUDA's lowering of it (see ptx_rt.h), so the translation computes what ZLUDA's own
|
||||
compile of the PTX computes, but it is compiled natively by clang and can then be optimised as C++.
|
||||
Registers become locals, basic blocks become labels, branches become gotos.
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
TYPES = {'f': 'float', 'r': 'uint32_t', 'rd': 'uint64_t', 'rs': 'uint16_t', 'p': 'bool', 'fd': 'double',
|
||||
'SP': 'uint64_t', 'SPL': 'uint64_t'}
|
||||
SREG = {'%tid.x': '__builtin_amdgcn_workitem_id_x()', '%tid.y': '__builtin_amdgcn_workitem_id_y()',
|
||||
'%tid.z': '__builtin_amdgcn_workitem_id_z()', '%ctaid.x': '__builtin_amdgcn_workgroup_id_x()',
|
||||
'%ctaid.y': '__builtin_amdgcn_workgroup_id_y()', '%ctaid.z': '__builtin_amdgcn_workgroup_id_z()',
|
||||
'%laneid': 'laneid()',
|
||||
'%ntid.x': '__builtin_amdgcn_workgroup_size_x()', '%ntid.y': '__builtin_amdgcn_workgroup_size_y()',
|
||||
'%nctaid.x': '(__builtin_amdgcn_grid_size_x() / __builtin_amdgcn_workgroup_size_x())',
|
||||
'%nctaid.y': '(__builtin_amdgcn_grid_size_y() / __builtin_amdgcn_workgroup_size_y())'}
|
||||
REG_RE = re.compile(r'%([a-zA-Z]+)(\d+)$')
|
||||
|
||||
|
||||
class Err(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def strip_comments(text):
|
||||
return re.sub(r'//[^\n]*', '', text)
|
||||
|
||||
|
||||
def tokenize(body):
|
||||
"""-> list of ('open'|'close'|'label'|'stmt', text)"""
|
||||
out, i, n = [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c.isspace():
|
||||
i += 1
|
||||
continue
|
||||
if c == '{':
|
||||
out.append(('open', None)); i += 1; continue
|
||||
if c == '}':
|
||||
out.append(('close', None)); i += 1; continue
|
||||
m = re.match(r'(\$?[A-Za-z_][\w$]*):', body[i:])
|
||||
if m and not body[i:].startswith('%'):
|
||||
out.append(('label', m.group(1))); i += m.end(); continue
|
||||
j = body.index(';', i)
|
||||
out.append(('stmt', ' '.join(body[i:j].split())))
|
||||
i = j + 1
|
||||
return out
|
||||
|
||||
|
||||
def split_operands(s):
|
||||
ops, depth, cur = [], 0, ''
|
||||
for ch in s:
|
||||
if ch in '{[':
|
||||
depth += 1
|
||||
elif ch in '}]':
|
||||
depth -= 1
|
||||
if ch == ',' and depth == 0:
|
||||
ops.append(cur.strip()); cur = ''
|
||||
else:
|
||||
cur += ch
|
||||
if cur.strip():
|
||||
ops.append(cur.strip())
|
||||
return ops
|
||||
|
||||
|
||||
class Translator:
|
||||
def __init__(self, ptx, kernel):
|
||||
self.kernel = kernel
|
||||
text = strip_comments(ptx)
|
||||
m = re.search(r'\.entry\s+' + re.escape(kernel) + r'\s*\((.*?)\)', text, re.S)
|
||||
if not m:
|
||||
raise Err('kernel not found')
|
||||
params = m.group(1)
|
||||
pm = re.search(r'\.param\s+\.align\s+(\d+)\s+\.b8\s+(\w+)\[(\d+)\]', params)
|
||||
self.param_name, self.param_size, self.param_align = pm.group(2), int(pm.group(3)), int(pm.group(1))
|
||||
rest = text[m.end():]
|
||||
self.attrs = rest[:rest.index('{')]
|
||||
start = rest.index('{')
|
||||
depth, k = 0, start
|
||||
while True:
|
||||
if rest[k] == '{':
|
||||
depth += 1
|
||||
elif rest[k] == '}':
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
k += 1
|
||||
self.body = rest[start + 1:k]
|
||||
self.globals_text = text[:m.start()]
|
||||
self.regs = {} # c name -> type
|
||||
self.shared = [] # (name, size, align)
|
||||
self.locals = []
|
||||
self.scope_names = [{}]
|
||||
self.scope_id = 0
|
||||
self.lines = []
|
||||
|
||||
# ------------------------------------------------------------ operands
|
||||
def reg(self, tok):
|
||||
tok = tok.strip()
|
||||
for scope in reversed(self.scope_names):
|
||||
if tok in scope:
|
||||
return scope[tok]
|
||||
m = REG_RE.match(tok)
|
||||
if not m:
|
||||
raise Err('not a register: ' + tok)
|
||||
kind, num = m.groups()
|
||||
name = f'{kind}{num}'
|
||||
if kind not in TYPES:
|
||||
raise Err('register kind ' + kind)
|
||||
self.regs[name] = TYPES[kind]
|
||||
return name
|
||||
|
||||
def rtype(self, name):
|
||||
return self.regs[name]
|
||||
|
||||
def val(self, tok, as_type):
|
||||
"""operand as a C expression of type as_type ('f32','u32','s32','u64','s64','u16','s16','pred','f16x2')"""
|
||||
tok = tok.strip()
|
||||
if tok in SREG:
|
||||
return f'(uint32_t){SREG[tok]}'
|
||||
if tok.startswith('%') or any(tok in s for s in self.scope_names):
|
||||
name = self.reg(tok)
|
||||
t = self.rtype(name)
|
||||
if as_type == 'f32':
|
||||
return name if t == 'float' else f'asf({name})'
|
||||
if as_type in ('u32', 'b32'):
|
||||
return f'asu({name})' if t == 'float' else name
|
||||
if as_type == 's32':
|
||||
return f'(int32_t)asu({name})' if t == 'float' else f'(int32_t){name}'
|
||||
if as_type in ('u64', 'b64'):
|
||||
return name
|
||||
if as_type == 's64':
|
||||
return f'(int64_t){name}'
|
||||
if as_type in ('u16', 'b16'):
|
||||
return name
|
||||
if as_type == 's16':
|
||||
return f'(int16_t){name}'
|
||||
if as_type == 'f16':
|
||||
return f'ash({name})'
|
||||
if as_type == 'f16x2':
|
||||
return f'ash2({name})'
|
||||
if as_type == 'pred':
|
||||
return name
|
||||
raise Err('type ' + as_type)
|
||||
if tok in self.shared_names():
|
||||
return f'(uint32_t)(uintptr_t)(AS3 uint8_t*){tok}'
|
||||
if tok == '__local_depot0':
|
||||
# private (address space 5) address of the depot, zero-extended: ld/st.local use it as an AS5 pointer
|
||||
return '(uint64_t)(uint32_t)(uintptr_t)(AS5 uint8_t*)__local_depot0'
|
||||
# immediates
|
||||
if re.match(r'0[fF][0-9A-Fa-f]{8}$', tok):
|
||||
bits = int(tok[2:], 16)
|
||||
return f'asf(0x{bits:08X}u)' if as_type == 'f32' else f'0x{bits:08X}u'
|
||||
if re.match(r'-?\d+$', tok) or re.match(r'0x[0-9a-fA-F]+$', tok):
|
||||
v = int(tok, 0)
|
||||
if as_type == 'f32':
|
||||
raise Err('int immediate as float')
|
||||
if as_type in ('u64', 's64', 'b64'):
|
||||
return f'(uint64_t){v}ull' if v >= 0 else f'(uint64_t)({v}ll)'
|
||||
if as_type in ('u16', 'b16', 's16'):
|
||||
return f'(uint16_t){v & 0xffff}'
|
||||
if as_type == 's32':
|
||||
v = v & 0xffffffff
|
||||
v = v - (1 << 32) if v >= (1 << 31) else v
|
||||
return f'(int32_t){v}' if v != -(1 << 31) else '(int32_t)0x80000000u'
|
||||
return f'{v & 0xffffffff}u'
|
||||
raise Err('operand ' + tok)
|
||||
|
||||
def shared_names(self):
|
||||
return [s[0] for s in self.shared]
|
||||
|
||||
def assign(self, dst, expr, src_type):
|
||||
"""dst = expr, expr of C type src_type ('float','u32','u64','u16','bool','f16x2','f16')"""
|
||||
name = self.reg(dst)
|
||||
t = self.rtype(name)
|
||||
if src_type == 'float':
|
||||
e = expr if t == 'float' else f'asu({expr})'
|
||||
elif src_type == 'u32':
|
||||
e = f'asf({expr})' if t == 'float' else f'(uint32_t)({expr})'
|
||||
elif src_type == 'u64':
|
||||
e = f'(uint64_t)({expr})'
|
||||
elif src_type == 'u16':
|
||||
e = f'(uint16_t)({expr})'
|
||||
elif src_type == 'f16':
|
||||
e = f'ashu({expr})'
|
||||
elif src_type == 'f16x2':
|
||||
e = f'ash2u({expr})'
|
||||
elif src_type == 'bool':
|
||||
e = f'({expr})'
|
||||
else:
|
||||
raise Err('src type')
|
||||
self.emit(f'{name} = {e};')
|
||||
|
||||
def emit(self, s):
|
||||
self.lines.append(' ' + s)
|
||||
|
||||
def addr(self, tok):
|
||||
"""[reg+off] / [name+off] -> (base_expr, offset_int, space_hint)"""
|
||||
m = re.match(r'\[\s*([^\]+]+?)\s*(?:\+\s*(-?\d+))?\s*\]$', tok)
|
||||
if not m:
|
||||
raise Err('address ' + tok)
|
||||
base, off = m.group(1), int(m.group(2) or 0)
|
||||
if base == self.param_name:
|
||||
return 'KARG', off
|
||||
if base in self.shared_names():
|
||||
return f'(uint32_t)(uintptr_t)(AS3 uint8_t*){base}', off
|
||||
return self.val(base, 'u64' if not base.startswith('%r') or base.startswith('%rd') else 'u32'), off
|
||||
|
||||
# ------------------------------------------------------------ instructions
|
||||
def translate(self):
|
||||
for kind, text in tokenize(self.body):
|
||||
if kind == 'open':
|
||||
self.scope_names.append({})
|
||||
continue
|
||||
if kind == 'close':
|
||||
self.scope_names.pop()
|
||||
continue
|
||||
if kind == 'label':
|
||||
self.lines.append(f'{text.replace("$", "")}:;')
|
||||
lab = text.replace('$', '')
|
||||
if lab in getattr(self, 'stamps', []):
|
||||
self.lines.append(f' D4R_STAMP({self.stamps.index(lab) + 1});')
|
||||
if getattr(self, 'bbcount', False):
|
||||
self.bblabels.append(lab)
|
||||
self.lines.append(f' D4R_BBC({len(self.bblabels) - 1});')
|
||||
continue
|
||||
self.stmt(text)
|
||||
|
||||
def stmt(self, s):
|
||||
if s.startswith('.reg'):
|
||||
m = re.match(r'\.reg\s+\.(\w+)\s+(.*)$', s)
|
||||
ty, names = m.groups()
|
||||
if '<' in names:
|
||||
return # numbered register banks: declared on use
|
||||
self.scope_id += 1
|
||||
for nm in names.split(','):
|
||||
nm = nm.strip()
|
||||
cname = f'l{self.scope_id}_{nm.lstrip("%")}'
|
||||
ctype = {'f16': 'uint16_t', 'b16': 'uint16_t', 'f32': 'float', 'b32': 'uint32_t', 'pred': 'bool',
|
||||
'f16x2': 'uint32_t', 'u32': 'uint32_t', 's32': 'uint32_t', 'b64': 'uint64_t', 'u64': 'uint64_t'}[ty]
|
||||
self.scope_names[-1][nm] = cname
|
||||
self.regs[cname] = ctype
|
||||
return
|
||||
if s.startswith('.shared'):
|
||||
m = re.match(r'\.shared\s+\.align\s+(\d+)\s+\.b8\s+(\w+)\[(\d+)\]', s)
|
||||
self.shared.append((m.group(2), int(m.group(3)), int(m.group(1))))
|
||||
return
|
||||
if s.startswith('.local'):
|
||||
m = re.match(r'\.local\s+\.align\s+(\d+)\s+\.b8\s+(\w+)\[(\d+)\]', s)
|
||||
self.locals.append((m.group(2), int(m.group(3)), int(m.group(1))))
|
||||
return
|
||||
if s.startswith('.pragma'):
|
||||
return
|
||||
pred = None
|
||||
m = re.match(r'@(!?)(%p\d+)\s+(.*)$', s)
|
||||
if m:
|
||||
pred = ('!' if m.group(1) else '') + self.reg(m.group(2))
|
||||
s = m.group(3)
|
||||
parts = s.split(None, 1)
|
||||
op = parts[0]
|
||||
args = split_operands(parts[1]) if len(parts) > 1 else []
|
||||
if pred is not None:
|
||||
if op.startswith('bra'):
|
||||
self.emit(f'if ({pred}) goto {args[0].replace("$", "")};')
|
||||
return
|
||||
self.emit(f'if ({pred}) {{')
|
||||
self.op(op, args)
|
||||
self.emit('}')
|
||||
return
|
||||
self.op(op, args)
|
||||
|
||||
def op(self, op, a):
|
||||
o = op.split('.')
|
||||
base = o[0]
|
||||
mods = set(o[1:])
|
||||
E, A = self.emit, self.assign
|
||||
V = self.val
|
||||
|
||||
def ftype():
|
||||
for t in ('f32', 'f16x2', 'f16', 'f64'):
|
||||
if t in mods:
|
||||
return t
|
||||
return None
|
||||
|
||||
if base == 'bra':
|
||||
E(f'goto {a[0].replace("$", "")};'); return
|
||||
if base == 'ret':
|
||||
if getattr(self, 'stamps', None):
|
||||
E(f'D4R_STAMP({len(self.stamps) + 1});')
|
||||
E('return;'); return
|
||||
if base == 'bar':
|
||||
E('bar_sync();'); return
|
||||
if base == 'mov':
|
||||
dst, src = a
|
||||
if dst.startswith('{'):
|
||||
names = [x.strip() for x in dst[1:-1].split(',')]
|
||||
srcv = V(src, 'u32')
|
||||
A(names[0], f'{srcv} & 0xffffu', 'u16')
|
||||
A(names[1], f'{srcv} >> 16', 'u16')
|
||||
return
|
||||
if src.startswith('{'):
|
||||
names = [x.strip() for x in src[1:-1].split(',')]
|
||||
if 'b64' in mods:
|
||||
A(dst, f'(uint64_t){V(names[0], "u32")} | ((uint64_t){V(names[1], "u32")} << 32)', 'u64'); return
|
||||
A(dst, f'pack16({V(names[0], "u16")}, {V(names[1], "u16")})', 'u32'); return
|
||||
if src == self.param_name:
|
||||
A(dst, 'KARGU', 'u64'); return
|
||||
if 'f32' in mods:
|
||||
A(dst, V(src, 'f32'), 'float'); return
|
||||
if mods & {'u64', 'b64', 's64'}:
|
||||
A(dst, V(src, 'u64'), 'u64'); return
|
||||
if mods & {'u16', 'b16', 's16'}:
|
||||
A(dst, V(src, 'u16'), 'u16'); return
|
||||
A(dst, V(src, 'u32'), 'u32'); return
|
||||
|
||||
ft = ftype()
|
||||
# ---------------------------------------------------- float arithmetic
|
||||
if ft == 'f32' and base in ('add', 'sub', 'mul'):
|
||||
if 'rz' in mods:
|
||||
raise Err('add.rz outside the roundf idiom')
|
||||
sym = {'add': '+', 'sub': '-', 'mul': '*'}[base]
|
||||
A(a[0], f'{V(a[1], "f32")} {sym} {V(a[2], "f32")}', 'float'); return
|
||||
if ft == 'f32' and base == 'fma':
|
||||
A(a[0], f'__builtin_fmaf({V(a[1], "f32")}, {V(a[2], "f32")}, {V(a[3], "f32")})', 'float'); return
|
||||
if ft == 'f32' and base in ('max', 'min'):
|
||||
if 'NaN' in mods:
|
||||
raise Err('NaN min/max')
|
||||
A(a[0], f'{"fmaxn" if base == "max" else "fminn"}({V(a[1], "f32")}, {V(a[2], "f32")})', 'float'); return
|
||||
if ft == 'f32' and base == 'neg':
|
||||
A(a[0], f'neg_ftz({V(a[1], "f32")})' if 'ftz' in mods else f'-{V(a[1], "f32")}', 'float'); return
|
||||
if ft == 'f32' and base == 'abs':
|
||||
A(a[0], f'abs_ftz({V(a[1], "f32")})' if 'ftz' in mods else f'__builtin_fabsf({V(a[1], "f32")})', 'float'); return
|
||||
if ft == 'f32' and base in ('rcp', 'ex2', 'lg2', 'rsqrt', 'sqrt'):
|
||||
if 'approx' not in mods:
|
||||
raise Err(op)
|
||||
fn = {'rcp': 'rcp_a', 'ex2': 'ex2_a', 'lg2': 'lg2_a', 'rsqrt': 'rsq_a', 'sqrt': '__builtin_amdgcn_sqrtf'}[base]
|
||||
A(a[0], f'{fn}({V(a[1], "f32")})', 'float'); return
|
||||
if ft == 'f32' and base == 'div':
|
||||
if 'approx' not in mods:
|
||||
raise Err(op)
|
||||
A(a[0], f'div_a({V(a[1], "f32")}, {V(a[2], "f32")})', 'float'); return
|
||||
# ---------------------------------------------------- f16 / f16x2
|
||||
if ft == 'f16x2' and base in ('add', 'sub', 'mul', 'fma'):
|
||||
x = [V(t, 'f16x2') for t in a[1:]]
|
||||
e = f'h2fma({x[0]}, {x[1]}, {x[2]})' if base == 'fma' else f'({x[0]} {"+-*"[["add", "sub", "mul"].index(base)]} {x[1]})'
|
||||
if 'sat' in mods:
|
||||
e = f'h2sat({e})'
|
||||
A(a[0], e, 'f16x2'); return
|
||||
if ft == 'f16x2' and base == 'abs':
|
||||
A(a[0], f'h2abs({V(a[1], "f16x2")})', 'f16x2'); return
|
||||
if ft == 'f16x2' and base == 'ex2':
|
||||
A(a[0], f'h2ex2({V(a[1], "f16x2")})', 'f16x2'); return
|
||||
if ft in ('f16', 'f16x2') and base in ('min', 'max'):
|
||||
if 'NaN' in mods:
|
||||
raise Err('NaN-propagating half min/max is not supported')
|
||||
fn = 'hminn' if base == 'min' else 'hmaxn'
|
||||
if ft == 'f16':
|
||||
A(a[0], f'{fn}({V(a[1], "f16")}, {V(a[2], "f16")})', 'f16'); return
|
||||
x, y = V(a[1], 'f16x2'), V(a[2], 'f16x2')
|
||||
A(a[0], f'(h2){{{fn}({x}.x, {y}.x), {fn}({x}.y, {y}.y)}}', 'f16x2'); return
|
||||
if ft == 'f16' and base in ('add', 'sub', 'mul', 'fma'):
|
||||
x = [V(t, 'f16') for t in a[1:]]
|
||||
e = f'hfma({x[0]}, {x[1]}, {x[2]})' if base == 'fma' else f'(h1)({x[0]} {"+-*"[["add", "sub", "mul"].index(base)]} {x[1]})'
|
||||
if 'sat' in mods:
|
||||
e = f'hsat({e})'
|
||||
A(a[0], e, 'f16'); return
|
||||
if op == 'tanh.approx.f16':
|
||||
A(a[0], f'__ocml_tanh_f16({V(a[1], "f16")})', 'f16'); return
|
||||
if base == 'set' and 'f16x2' in mods and 'u32' in mods:
|
||||
cmp = self.cmp_op(mods)
|
||||
x, y = V(a[1], 'f16x2'), V(a[2], 'f16x2')
|
||||
A(a[0], f'pack16(({cmp.format(f"{x}.x", f"{y}.x")}) ? 0xffffu : 0u, ({cmp.format(f"{x}.y", f"{y}.y")}) ? 0xffffu : 0u)', 'u32'); return
|
||||
# ---------------------------------------------------- compares / selects
|
||||
if base == 'setp':
|
||||
cmp = self.cmp_op(mods)
|
||||
if mods & {'f32'}:
|
||||
x, y = V(a[1], 'f32'), V(a[2], 'f32')
|
||||
elif 'f16' in mods:
|
||||
x, y = V(a[1], 'f16'), V(a[2], 'f16')
|
||||
elif mods & {'s32'}:
|
||||
x, y = V(a[1], 's32'), V(a[2], 's32')
|
||||
elif mods & {'u32', 'b32'}:
|
||||
x, y = V(a[1], 'u32'), V(a[2], 'u32')
|
||||
elif mods & {'s16'}:
|
||||
x, y = V(a[1], 's16'), V(a[2], 's16')
|
||||
elif mods & {'u16', 'b16'}:
|
||||
x, y = V(a[1], 'u16'), V(a[2], 'u16')
|
||||
elif mods & {'s64'}:
|
||||
x, y = V(a[1], 's64'), V(a[2], 's64')
|
||||
elif mods & {'u64'}:
|
||||
x, y = V(a[1], 'u64'), V(a[2], 'u64')
|
||||
else:
|
||||
raise Err(op)
|
||||
if len(a) != 3 or '|' in a[0]:
|
||||
raise Err('setp form ' + op)
|
||||
A(a[0], cmp.format(x, y), 'bool'); return
|
||||
if base == 'selp':
|
||||
t = 'f32' if 'f32' in mods else 'u16' if mods & {'u16', 'b16', 's16'} else 'u64' if mods & {'u64', 'b64', 's64'} else 'u32'
|
||||
ct = {'f32': 'float', 'u16': 'u16', 'u64': 'u64', 'u32': 'u32'}[t]
|
||||
A(a[0], f'{V(a[3], "pred")} ? {V(a[1], t)} : {V(a[2], t)}', ct); return
|
||||
# ---------------------------------------------------- predicates / integer
|
||||
if 'pred' in mods and base in ('and', 'or', 'xor', 'not'):
|
||||
if base == 'not':
|
||||
A(a[0], f'!{V(a[1], "pred")}', 'bool'); return
|
||||
sym = {'and': '&&', 'or': '||', 'xor': '!='}[base]
|
||||
A(a[0], f'{V(a[1], "pred")} {sym} {V(a[2], "pred")}', 'bool'); return
|
||||
if base in ('and', 'or', 'xor', 'not', 'shl', 'shr', 'add', 'sub', 'mul', 'mad', 'min', 'max', 'div', 'rem', 'neg', 'abs'):
|
||||
if ft is not None:
|
||||
raise Err('unhandled floating operation: ' + op)
|
||||
width = 64 if mods & {'u64', 's64', 'b64'} else 16 if mods & {'u16', 's16', 'b16'} else 32
|
||||
signed = bool(mods & {'s32', 's64', 's16'})
|
||||
ut = {64: 'u64', 32: 'u32', 16: 'u16'}[width]
|
||||
st = {64: 's64', 32: 's32', 16: 's16'}[width]
|
||||
ct = {64: 'u64', 32: 'u32', 16: 'u16'}[width]
|
||||
if base == 'mul' and 'hi' in mods and width == 32:
|
||||
wide = 'int64_t' if signed else 'uint64_t'
|
||||
typ = 's32' if signed else 'u32'
|
||||
A(a[0], f'(({wide}){V(a[1], typ)} * ({wide}){V(a[2], typ)}) >> 32', 'u32'); return
|
||||
if base == 'mul' and 'wide' in mods:
|
||||
if mods & {'s32'}:
|
||||
A(a[0], f'(int64_t){V(a[1], "s32")} * (int64_t){V(a[2], "s32")}', 'u64'); return
|
||||
if mods & {'u32'}:
|
||||
A(a[0], f'(uint64_t){V(a[1], "u32")} * (uint64_t){V(a[2], "u32")}', 'u64'); return
|
||||
if mods & {'u16'}:
|
||||
A(a[0], f'(uint32_t){V(a[1], "u16")} * (uint32_t){V(a[2], "u16")}', 'u32'); return
|
||||
raise Err(op)
|
||||
if base == 'not':
|
||||
A(a[0], f'~{V(a[1], ut)}', ct); return
|
||||
if base in ('neg', 'abs'):
|
||||
if base == 'abs' and signed:
|
||||
A(a[0], f'{V(a[1], st)} < 0 ? (uint{width}_t)(0 - {V(a[1], ut)}) : {V(a[1], ut)}', ct); return
|
||||
if base == 'neg':
|
||||
A(a[0], f'(uint{width}_t)(0 - {V(a[1], ut)})', ct); return
|
||||
raise Err(op)
|
||||
x = V(a[1], st if signed else ut)
|
||||
if base == 'shl':
|
||||
A(a[0], f'{V(a[1], ut)} << ({V(a[2], "u32")} & {width - 1})', ct); return
|
||||
if base == 'shr':
|
||||
if signed:
|
||||
A(a[0], f'{V(a[1], st)} >> ({V(a[2], "u32")} & {width - 1})', ct); return
|
||||
A(a[0], f'{V(a[1], ut)} >> ({V(a[2], "u32")} & {width - 1})', ct); return
|
||||
y = V(a[2], st if signed else ut)
|
||||
if base in ('and', 'or', 'xor'):
|
||||
sym = {'and': '&', 'or': '|', 'xor': '^'}[base]
|
||||
A(a[0], f'{V(a[1], ut)} {sym} {V(a[2], ut)}', ct); return
|
||||
if base in ('add', 'sub'):
|
||||
A(a[0], f'{V(a[1], ut)} {"+" if base == "add" else "-"} {V(a[2], ut)}', ct); return
|
||||
if base == 'mul':
|
||||
if 'lo' not in mods:
|
||||
raise Err(op)
|
||||
A(a[0], f'{V(a[1], ut)} * {V(a[2], ut)}', ct); return
|
||||
if base == 'mad':
|
||||
if 'lo' not in mods:
|
||||
raise Err(op)
|
||||
A(a[0], f'{V(a[1], ut)} * {V(a[2], ut)} + {V(a[3], ut)}', ct); return
|
||||
if base in ('min', 'max'):
|
||||
fn = 'min' if base == 'min' else 'max'
|
||||
A(a[0], f'__builtin_elementwise_{fn}({x}, {y})', ct); return
|
||||
if base == 'div':
|
||||
A(a[0], f'{x} / {y}', ct); return
|
||||
if base == 'rem':
|
||||
A(a[0], f'{x} % {y}', ct); return
|
||||
if base == 'bfi':
|
||||
A(a[0], f'bfi32({V(a[1], "u32")}, {V(a[2], "u32")}, {V(a[3], "u32")}, {V(a[4], "u32")})', 'u32'); return
|
||||
if op == 'prmt.b32':
|
||||
A(a[0], f'prmt32({V(a[1], "u32")}, {V(a[2], "u32")}, {V(a[3], "u32")})', 'u32'); return
|
||||
# ---------------------------------------------------- conversions
|
||||
if base == 'cvt':
|
||||
return self.cvt(op, o[1:], a)
|
||||
if base == 'cvta':
|
||||
A(a[0], V(a[1], 'u64'), 'u64'); return
|
||||
# ---------------------------------------------------- memory
|
||||
if base == 'ld':
|
||||
return self.ld(o[1:], a)
|
||||
if base == 'st':
|
||||
return self.st(o[1:], a)
|
||||
if base == 'tex':
|
||||
dst = [x.strip() for x in a[0][1:-1].split(',')]
|
||||
m = re.match(r'\[\s*(%rd\d+)\s*,\s*\{([^}]*)\}\s*\]', a[1])
|
||||
h = V(m.group(1), 'u64')
|
||||
c = [x.strip() for x in m.group(2).split(',')]
|
||||
if 'base' in mods and mods >= {'2d', 'v4', 's32'} and mods & {'f32', 'f16'}:
|
||||
E(f'{{ ::f4 t_ = tex_fetch({h}, (int32_t){V(c[0], "u32")}, (int32_t){V(c[1], "u32")});')
|
||||
elif mods >= {'level', '2d', 'v4', 'f32'} and op.endswith('f32.f32'):
|
||||
E(f'{{ ::f4 t_ = tex_level({h}, {V(c[0], "f32")}, {V(c[1], "f32")}, {V(a[2], "f32")});')
|
||||
else:
|
||||
raise Err(op)
|
||||
for i, d in enumerate(dst):
|
||||
A(d, f'(h1)t_[{i}]', 'f16') if 'f16' in mods else A(d, f't_[{i}]', 'float')
|
||||
E('}')
|
||||
return
|
||||
if base == 'tld4':
|
||||
if not (mods >= {'r', '2d', 'v4', 'f32'}):
|
||||
raise Err(op)
|
||||
dst = [x.strip() for x in a[0][1:-1].split(',')]
|
||||
m = re.match(r'\[\s*(%rd\d+)\s*,\s*\{([^}]*)\}\s*\]', a[1])
|
||||
c = [x.strip() for x in m.group(2).split(',')]
|
||||
E(f'{{ ::f4 t_ = tex_gather_r({V(m.group(1), "u64")}, {V(c[0], "f32")}, {V(c[1], "f32")});')
|
||||
for i, d in enumerate(dst):
|
||||
A(d, f't_[{i}]', 'float')
|
||||
E('}')
|
||||
return
|
||||
if base == 'sust':
|
||||
m = re.match(r'\[\s*(%rd\d+)\s*,\s*\{([^}]*)\}\s*\]', a[0])
|
||||
h = V(m.group(1), 'u64')
|
||||
c = [V(x.strip(), 'u32') for x in m.group(2).split(',')]
|
||||
v = [V(x.strip(), 'u32') for x in a[1][1:-1].split(',')]
|
||||
if op == 'sust.p.2d.v4.b32.zero':
|
||||
E(f'd4r_sust_p_v4b32({h}, (int32_t){c[0]}, (int32_t){c[1]}, {v[0]}, {v[1]}, {v[2]}, {v[3]});'); return
|
||||
if op == 'sust.b.2d.b32.zero':
|
||||
E(f'd4r_sust_b32({h}, (int32_t){c[0]}, (int32_t){c[1]}, {v[0]});'); return
|
||||
if op == 'sust.b.2d.v2.b16.zero':
|
||||
v = [V(x.strip(), 'u16') for x in a[1][1:-1].split(',')]
|
||||
E(f'd4r_sust_v2b16({h}, (int32_t){c[0]}, (int32_t){c[1]}, {v[0]}, {v[1]});'); return
|
||||
raise Err(op)
|
||||
if base == 'shfl':
|
||||
if op == 'shfl.sync.idx.b32':
|
||||
d = a[0].split('|')
|
||||
pred = f'&{self.reg(d[1])}' if len(d) > 1 else 'nullptr'
|
||||
A(d[0], f'shfl_idx({V(a[1], "u32")}, {V(a[2], "u32")}, {V(a[3], "u32")}, {pred})', 'u32'); return
|
||||
if op != 'shfl.sync.bfly.b32':
|
||||
raise Err(op)
|
||||
d = a[0].split('|')
|
||||
p = f'&{self.reg(d[1])}' if len(d) > 1 else 'nullptr'
|
||||
A(d[0], f'shfl_bfly({V(a[1], "u32")}, (int32_t){V(a[2], "u32")}, {V(a[3], "u32")}, {p})', 'u32'); return
|
||||
if op == 'vote.sync.ballot.b32':
|
||||
A(a[0], f'(uint32_t)(__builtin_amdgcn_ballot_w64({V(a[1], "pred")}) >> (wave_lane() & 32u))', 'u32'); return
|
||||
raise Err('unhandled ' + op)
|
||||
|
||||
def cmp_op(self, mods):
|
||||
table = {'lt': '{} < {}', 'gt': '{} > {}', 'le': '{} <= {}', 'ge': '{} >= {}', 'eq': '{} == {}',
|
||||
'ne': '{} != {}',
|
||||
# unordered: true if either is NaN
|
||||
'ltu': '!({} >= {})', 'gtu': '!({} <= {})', 'leu': '!({} > {})', 'geu': '!({} < {})',
|
||||
'equ': '!({} != {})', 'neu': '!({} == {})'}
|
||||
for k in table:
|
||||
if k in mods:
|
||||
if k in ('ne', 'equ') and mods & {'f32', 'f16', 'f16x2', 'f64'}:
|
||||
# PTX setp.ne.f32 is ordered (false on NaN) and equ is unordered (true on NaN), while C's
|
||||
# != is unordered: spell both out. Integer ne is plain.
|
||||
return '({0} < {1} || {0} > {1})' if k == 'ne' else '!({0} < {1} || {0} > {1})'
|
||||
return table[k]
|
||||
raise Err('compare ' + str(mods))
|
||||
|
||||
def cvt(self, op, o, a):
|
||||
A, V = self.assign, self.val
|
||||
mods = set(o)
|
||||
if op == 'cvt.rn.satfinite.e4m3x2.f16x2':
|
||||
A(a[0], f'e4m3x2({V(a[1], "u32")})', 'u16'); return
|
||||
if op == 'cvt.rn.f16x2.e4m3x2':
|
||||
A(a[0], f'd4r_decode_e4m3x2({V(a[1], "u16")})', 'u32'); return
|
||||
types = [t for t in o if re.match(r'^(f|s|u|b)\d+(x2)?$', t)]
|
||||
dt, stp = types[0], types[1]
|
||||
if dt == 'f32' and stp == 'f32':
|
||||
x = V(a[1], 'f32')
|
||||
if 'sat' in mods and 'ftz' in mods and not (mods & {'rzi', 'rmi', 'rni', 'rpi'}):
|
||||
A(a[0], f'sat_ftz({x})', 'float'); return
|
||||
if mods & {'sat'}:
|
||||
raise Err(op)
|
||||
if 'rzi' in mods:
|
||||
A(a[0], f'__builtin_truncf({x})', 'float'); return
|
||||
if 'rmi' in mods:
|
||||
A(a[0], f'__builtin_floorf({x})', 'float'); return
|
||||
if 'rni' in mods:
|
||||
A(a[0], f'__builtin_rintf({x})', 'float'); return
|
||||
if 'rpi' in mods:
|
||||
A(a[0], f'__builtin_ceilf({x})', 'float'); return
|
||||
if 'ftz' in mods:
|
||||
A(a[0], f'canon({x})', 'float'); return
|
||||
A(a[0], x, 'float'); return
|
||||
if dt == 's32' and stp == 'f32':
|
||||
x = V(a[1], 'f32')
|
||||
if 'rzi' in mods:
|
||||
A(a[0], f'(uint32_t)f2i_rz({x})', 'u32'); return
|
||||
if 'rmi' in mods:
|
||||
A(a[0], f'(uint32_t)f2i_rz(__builtin_floorf({x}))', 'u32'); return
|
||||
if 'rni' in mods:
|
||||
A(a[0], f'(uint32_t)f2i_rz(__builtin_rintf({x}))', 'u32'); return
|
||||
raise Err(op)
|
||||
if dt == 'u32' and stp == 'f32' and 'rzi' in mods:
|
||||
A(a[0], f'f2u_rz({V(a[1], "f32")})', 'u32'); return
|
||||
if dt == 'f32' and stp == 's32':
|
||||
A(a[0], f'(float)(int32_t){V(a[1], "u32")}', 'float'); return
|
||||
if dt == 'f32' and stp == 'u32':
|
||||
A(a[0], f'(float){V(a[1], "u32")}', 'float'); return
|
||||
if dt == 'f32' and stp == 'f16':
|
||||
A(a[0], f'(float){V(a[1], "f16")}', 'float'); return
|
||||
if dt == 'f16' and stp == 'f32':
|
||||
if 'rn' not in mods:
|
||||
raise Err(op)
|
||||
A(a[0], f'(h1){V(a[1], "f32")}', 'f16'); return
|
||||
if dt == 'f16x2' and stp == 'f32':
|
||||
# d = {cvt(a) in the high half, cvt(b) in the low half}
|
||||
A(a[0], f'pack16(ashu((h1){V(a[2], "f32")}), ashu((h1){V(a[1], "f32")}))', 'u32'); return
|
||||
if dt == 'f16' and stp == 'f16':
|
||||
x = V(a[1], 'f16')
|
||||
if 'rni' in mods:
|
||||
A(a[0], f'h_roundeven({x})', 'f16'); return
|
||||
if 'rmi' in mods:
|
||||
A(a[0], f'h_floor({x})', 'f16'); return
|
||||
raise Err(op)
|
||||
if dt == 's32' and stp == 'f16':
|
||||
x = V(a[1], 'f16')
|
||||
if 'rni' in mods:
|
||||
A(a[0], f'(uint32_t)h2i_sat(h_roundeven({x}))', 'u32'); return
|
||||
if 'rmi' in mods:
|
||||
A(a[0], f'(uint32_t)h2i_sat(h_floor({x}))', 'u32'); return
|
||||
raise Err(op)
|
||||
if dt == 'f16' and stp == 's32':
|
||||
A(a[0], f'(h1)(int32_t){V(a[1], "u32")}', 'f16'); return
|
||||
if dt == 's64' and stp == 's32':
|
||||
A(a[0], f'(uint64_t)(int64_t)(int32_t){V(a[1], "u32")}', 'u64'); return
|
||||
if dt == 'u64' and stp == 'u32':
|
||||
A(a[0], f'(uint64_t){V(a[1], "u32")}', 'u64'); return
|
||||
if dt in ('u16', 's16') and stp in ('u32', 's32'):
|
||||
A(a[0], f'(uint16_t){V(a[1], "u32")}', 'u16'); return
|
||||
if dt in ('u32', 's32') and stp in ('u16',):
|
||||
A(a[0], f'(uint32_t){V(a[1], "u16")}', 'u32'); return
|
||||
if dt in ('u32', 's32') and stp in ('s16',):
|
||||
A(a[0], f'(uint32_t)(int32_t)(int16_t){V(a[1], "u16")}', 'u32'); return
|
||||
if dt == 'u32' and stp == 'u64':
|
||||
A(a[0], f'(uint32_t){V(a[1], "u64")}', 'u32'); return
|
||||
raise Err('cvt ' + op)
|
||||
|
||||
def mem_type(self, o):
|
||||
for t in ('u8', 's8', 'u16', 's16', 'b16', 'u32', 's32', 'b32', 'f32', 'u64', 's64', 'b64', 'f16'):
|
||||
if t in o:
|
||||
return t
|
||||
raise Err('mem type ' + str(o))
|
||||
|
||||
def ctype_of(self, t):
|
||||
return {'u8': 'uint8_t', 's8': 'int8_t', 'u16': 'uint16_t', 's16': 'int16_t', 'b16': 'uint16_t',
|
||||
'u32': 'uint32_t', 's32': 'int32_t', 'b32': 'uint32_t', 'f32': 'float', 'u64': 'uint64_t',
|
||||
's64': 'int64_t', 'b64': 'uint64_t'}[t]
|
||||
|
||||
def ptr(self, space, base, off, ct):
|
||||
if base == 'KARG':
|
||||
return f'*(const AS4 {ct}*)(KARGP + {off})'
|
||||
if space == 'param':
|
||||
return f'*(const AS4 {ct}*)(uintptr_t)({base} + {off})'
|
||||
if space == 'shared':
|
||||
return f'*(AS3 {ct}*)(uintptr_t)({base} + {off}u)'
|
||||
if space == 'local':
|
||||
return f'*(AS5 {ct}*)(uintptr_t)(uint32_t)({base} + {off})'
|
||||
if space == 'global' and getattr(self, 'as1', False):
|
||||
# --as1: ld/st.global through address-space-1 pointers (global_load / global_store instead of flat)
|
||||
return f'*(AS1 {ct}*)(uintptr_t)({base} + {off})'
|
||||
if space in ('global', 'local'):
|
||||
return f'*({ct}*)(uintptr_t)({base} + {off})'
|
||||
raise Err('space ' + space)
|
||||
|
||||
def ld(self, o, a):
|
||||
space = next(s for s in ('param', 'global', 'shared', 'local') if s in o)
|
||||
t = self.mem_type(o)
|
||||
ct = self.ctype_of(t)
|
||||
base, off = self.addr(a[1])
|
||||
dsts = [x.strip() for x in a[0][1:-1].split(',')] if a[0].startswith('{') else [a[0]]
|
||||
width = {'uint8_t': 1, 'int8_t': 1, 'uint16_t': 2, 'int16_t': 2, 'uint32_t': 4, 'int32_t': 4, 'float': 4,
|
||||
'uint64_t': 8, 'int64_t': 8}[ct]
|
||||
if len(dsts) in (2, 4) and width == 4 and space in ('global', 'shared') and base != 'KARG':
|
||||
# ld.v2/v4 (naturally aligned in PTX) as one vector load instead of per-element loads
|
||||
vt = 'u4' if len(dsts) == 4 else 'u2v'
|
||||
asq = 'AS3 ' if space == 'shared' else ('AS1 ' if getattr(self, 'as1', False) else '')
|
||||
ptr = f'(uintptr_t)({base} + {off}u)' if space == 'shared' else f'(uintptr_t)({base} + {off})'
|
||||
self.emit(f'{{ const {vt} v_ = *({asq}const {vt}*){ptr};')
|
||||
for i, d in enumerate(dsts):
|
||||
self.assign(d, f'v_[{i}]', 'u32')
|
||||
self.emit('}')
|
||||
return
|
||||
for i, d in enumerate(dsts):
|
||||
e = self.ptr(space, base, off + i * width, ct)
|
||||
if ct == 'float':
|
||||
self.assign(d, e, 'float')
|
||||
elif width == 8:
|
||||
self.assign(d, e, 'u64')
|
||||
else:
|
||||
name = self.reg(d)
|
||||
if self.rtype(name) == 'uint16_t':
|
||||
self.assign(d, e, 'u16')
|
||||
else:
|
||||
self.assign(d, f'(uint32_t){e}' if ct != 'int8_t' and ct != 'int16_t' else f'(uint32_t)(int32_t){e}', 'u32')
|
||||
|
||||
def st(self, o, a):
|
||||
space = next(s for s in ('param', 'global', 'shared', 'local') if s in o)
|
||||
t = self.mem_type(o)
|
||||
ct = self.ctype_of(t)
|
||||
base, off = self.addr(a[0])
|
||||
srcs = [x.strip() for x in a[1][1:-1].split(',')] if a[1].startswith('{') else [a[1]]
|
||||
width = {'uint8_t': 1, 'uint16_t': 2, 'uint32_t': 4, 'int32_t': 4, 'float': 4, 'uint64_t': 8}[ct]
|
||||
for i, s in enumerate(srcs):
|
||||
v = self.val(s, 'f32' if ct == 'float' else 'u64' if width == 8 else 'u16' if width == 2 else 'u32')
|
||||
self.emit(f'{self.ptr(space, base, off + i * width, ct)} = ({ct})({v});')
|
||||
|
||||
# ------------------------------------------------------------ roundf idiom
|
||||
def rewrite_rz(self):
|
||||
"""add.rz.ftz.f32 t, x, h; cvt.rzi.f32.f32 y, t (h = copysign(0.5, x)) -> y = roundf(x)"""
|
||||
text = self.body
|
||||
pat = re.compile(r'add\.rz\.ftz\.f32\s+(%f\d+),\s*(%f\d+),\s*(%f\d+);\s*(?://[^\n]*\s*)*cvt\.rzi\.f32\.f32\s+(%f\d+),\s*(%f\d+);')
|
||||
count = 0
|
||||
|
||||
def repl(m):
|
||||
nonlocal count
|
||||
t, x, h, y, t2 = m.groups()
|
||||
if t != t2:
|
||||
raise Err('rz idiom mismatch')
|
||||
uses = len(re.findall(re.escape(t) + r'\b', text))
|
||||
if uses != 2:
|
||||
raise Err(f'rz temp {t} used {uses} times')
|
||||
# check h = copysign(0.5, x): mov.b32 rA, x; and.b32 rB, rA, -2147483648; or.b32 rC, rB, 1056964608; mov.b32 h, rC
|
||||
count += 1
|
||||
return f'd4r.roundf {y}, {x};'
|
||||
self.body = pat.sub(repl, text)
|
||||
return count
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith('--')]
|
||||
flags = {a for a in sys.argv[1:] if a.startswith('--')}
|
||||
if len(args) != 2:
|
||||
sys.exit(__doc__)
|
||||
ptx, kernel = open(args[0]).read(), args[1]
|
||||
mpart = None
|
||||
if '--m-tail' in flags:
|
||||
lines = ptx.replace('\r\n', '\n').split('\n')
|
||||
if kernel == 'rrlite_enc0_4x4_mvhi_hdr_folded':
|
||||
if lines[2547].strip() != 'bar.warp.sync -1;':
|
||||
raise Err('enc0 cut changed')
|
||||
lines = lines[:2548] + ['d4r.enc0 %r259, %rd106, %rd78, %r1066, %r1067, %r1230, %r1233;', 'ret;', '}']
|
||||
mpart = 'enc0_tail'
|
||||
elif kernel == 'rrlite_dec0_4x4_folded':
|
||||
if lines[138].strip() != 'mov.u32 %r17, %laneid;' or lines[586].strip() != 'st.shared.v2.u16 [%r570+1168], {%rs63, %rs64};':
|
||||
raise Err('dec0 cut changed')
|
||||
lines[138:587] = ['mov.u32 %r560, _ZZ22rrlite_dec0_4x4_foldedN6RRLite18Decoder0ParametersEE4smem;',
|
||||
'd4r.dec0 %r560, %rd22, %rd24, %r475, %r476, %r3, %r4;']
|
||||
mpart = 'dec0_head'
|
||||
else:
|
||||
raise Err('--m-tail requires folded M enc0 or dec0')
|
||||
ptx = '\n'.join(lines)
|
||||
tr = Translator(ptx, kernel)
|
||||
n_rz = tr.rewrite_rz()
|
||||
if '--ifconv' in flags:
|
||||
import ifconv, ctile as _ct
|
||||
tr.body, n_if = ifconv.apply(tr.body, tokenize, _ct.parse, split_operands)
|
||||
sys.stderr.write(f'ifconv: {n_if} diamonds converted\n')
|
||||
stamps = [f.split('=', 1)[1].split(',') for f in flags if f.startswith('--stamps=')]
|
||||
tr.stamps = stamps[0] if stamps else []
|
||||
tr.bbcount = '--bbcount' in flags
|
||||
tr.as1 = '--as1' in flags
|
||||
tr.bblabels = []
|
||||
ct = None
|
||||
if '--ctile' in flags:
|
||||
import ctile
|
||||
ct = ctile.emit(tr, tokenize, Translator)
|
||||
if '--sink' in flags:
|
||||
import ifconv, ctile as _ct
|
||||
tr.body, n_sink = ifconv.sink_stores(tr.body, tokenize, _ct.parse)
|
||||
sys.stderr.write(f'sink: {n_sink} stores moved\n')
|
||||
orig_op = tr.op
|
||||
|
||||
def op_with_round(op, a):
|
||||
if op in ('d4r.hmin2', 'd4r.hmax2'):
|
||||
fn = 'm_min2' if op == 'd4r.hmin2' else 'm_max2'
|
||||
left = ', '.join(tr.val(v, 'f16') for v in a[2:4])
|
||||
right = ', '.join(tr.val(v, 'f16') for v in a[4:6])
|
||||
tr.emit(f'{{ uint32_t packed_ = ash2u({fn}((h2){{{left}}}, (h2){{{right}}}));')
|
||||
tr.assign(a[0], 'packed_ & 0xffffu', 'u16')
|
||||
tr.assign(a[1], 'packed_ >> 16', 'u16')
|
||||
tr.emit('}')
|
||||
return
|
||||
if op in ('d4r.enc0', 'd4r.dec0'):
|
||||
vals = [tr.val(v, t) for v, t in zip(a, ('u32', 'u64', 'u64', 'u32', 'u32', 's32', 's32'))]
|
||||
tr.emit(f'd4r_{"enc0_tail" if op == "d4r.enc0" else "dec0_head"}({", ".join(vals)});')
|
||||
return
|
||||
if op == 'd4r.roundf':
|
||||
tr.assign(a[0], f'__builtin_roundf({tr.val(a[1], "f32")})', 'float')
|
||||
return
|
||||
orig_op(op, a)
|
||||
tr.op = op_with_round
|
||||
tr.translate()
|
||||
out = []
|
||||
out.append(f'// Generated by kernels/native/ptx2hip.py from NVIDIA PTX ({kernel}); {n_rz} roundf idioms.')
|
||||
out.append('#include "native_rt.h"')
|
||||
if kernel.startswith('rrlite_'):
|
||||
out.append('#include "m_ptx_rt.h"')
|
||||
if 'cvt.rn.f16x2.e4m3x2' in ptx:
|
||||
out.append('RT uint32_t d4r_decode_e4m3x2(uint16_t v) { h2 h; for (int i = 0; i < 2; ++i) { '
|
||||
'uint32_t b = (v >> (8*i)) & 255u; '
|
||||
'h[i] = (b & 127u) == 127u ? ash((uint16_t)(0x7fffu | ((b & 128u) << 8))) : '
|
||||
'(h1)(ash((uint16_t)(((b & 127u) << 7) | ((b & 128u) << 8))) * (h1)256.0f); } return ash2u(h); }')
|
||||
if mpart:
|
||||
out.append(f'#include "../tex/{mpart}.hip"')
|
||||
if ct is None:
|
||||
for name, size, align in tr.shared:
|
||||
out.append(f'__attribute__((shared)) __attribute__((aligned({max(align, 16)}))) static uint8_t {name}[{size}];')
|
||||
else:
|
||||
# one LDS block: the kernel's own arrays (path B) and the color tile (path A) are never live together
|
||||
off = 0
|
||||
for name, size, align in tr.shared:
|
||||
off = (off + 15) & ~15
|
||||
out.append(f'#define {name} (d4r_lds + {off})')
|
||||
off += size
|
||||
prelude, fill, info = ct
|
||||
out.insert(2, f'__attribute__((shared)) __attribute__((aligned(16))) static uint8_t d4r_lds[{max(off, info["lds_bytes"])}];')
|
||||
out.insert(2, f'// --ctile: {info["taps"]} color taps through an LDS tile, {len(info["groups"])} conversion shapes')
|
||||
out.extend(prelude)
|
||||
out.extend(fill)
|
||||
out.append(f'struct __attribute__((aligned({tr.param_align}))) {kernel}_params {{ uint8_t b[{tr.param_size}]; }};')
|
||||
maxntid = re.search(r'\.maxntid\s+(\d+)', tr.attrs)
|
||||
lb = f'__attribute__((amdgpu_flat_work_group_size(1, {maxntid.group(1)})))' if maxntid else ''
|
||||
bounds = [f.split('=', 1)[1] for f in flags if f.startswith('--max-block=')]
|
||||
if bounds:
|
||||
n = int(bounds[0])
|
||||
if not 1 <= n <= 1024:
|
||||
raise Err('invalid --max-block')
|
||||
lb = f'__attribute__((amdgpu_flat_work_group_size(1, {n})))'
|
||||
out.append(f'extern "C" __attribute__((global)) {lb} void {kernel}({kernel}_params)')
|
||||
out.append('{')
|
||||
out.append(' const AS4 uint8_t* KARGP = (const AS4 uint8_t*)__builtin_amdgcn_kernarg_segment_ptr();')
|
||||
out.append(' const uint64_t KARGU = (uint64_t)(uintptr_t)KARGP;')
|
||||
for name, size, align in tr.locals:
|
||||
out.append(f' __attribute__((aligned({align}))) uint8_t {name}[{size}];')
|
||||
if ct is not None:
|
||||
out.append(' int32_t ct_x0 = 0, ct_y0 = 0, ct_w = 0, ct_h = 0;')
|
||||
if tr.bbcount:
|
||||
# diagnostic build: d4r_bbc[k] = number of waves that entered basic block k (label list in a comment)
|
||||
out.insert(2, '__attribute__((device)) unsigned long long d4r_bbc[512];\n'
|
||||
'#define D4R_BBC(k) do { if (wave_lane() == (uint32_t)__builtin_ctzll(__builtin_amdgcn_read_exec())) '
|
||||
'__hip_atomic_fetch_add(&d4r_bbc[k], 1ull, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } while (0)\n'
|
||||
'// d4r_bbc labels: ' + ' '.join(tr.bblabels))
|
||||
if tr.stamps:
|
||||
# diagnostic build: per-region wave cycles (20-bit SHADER_CYCLES deltas) summed into d4r_dbg[k],
|
||||
# wave counts into d4r_dbg[32 + k]; region k ends at stamp label k (k = len + 1: kernel return)
|
||||
out.insert(2, '__attribute__((device)) unsigned long long d4r_dbg[64];\n'
|
||||
'#define D4R_STAMP(k) do { uint32_t n_ = (uint32_t)__builtin_readcyclecounter(); '
|
||||
'if (wave_lane() == (uint32_t)__builtin_ctzll(__builtin_amdgcn_read_exec())) { '
|
||||
'__hip_atomic_fetch_add(&d4r_dbg[k], (unsigned long long)((n_ - st_last) & 0xfffffu), __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); '
|
||||
'__hip_atomic_fetch_add(&d4r_dbg[32 + (k)], 1ull, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT); } '
|
||||
'st_last = n_; } while (0)')
|
||||
out.append(' uint32_t st_last = (uint32_t)__builtin_readcyclecounter();')
|
||||
by_type = {}
|
||||
for name, t in sorted(tr.regs.items()):
|
||||
by_type.setdefault(t, []).append(name)
|
||||
for t, names in by_type.items():
|
||||
for i in range(0, len(names), 16):
|
||||
out.append(f' {t} {", ".join(names[i:i + 16])};')
|
||||
out.extend(tr.lines)
|
||||
out.append('}')
|
||||
print('\n'.join(out))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,166 @@
|
||||
// Native runtime for PTX kernels translated to HIP by ptx2hip.py (no ZLUDA compile involved).
|
||||
// Every helper reproduces ZLUDA's lowering of the PTX instruction (ptx/src/pass/llvm/emit.rs and
|
||||
// ptx/lib/zluda_ptx_impl.cpp, patches/zluda), so a translated kernel computes the same values as
|
||||
// ZLUDA's own compile of NVIDIA's PTX. Texture and surface objects are ZLUDA handles: the address of
|
||||
// a HIP image descriptor (8 dwords), its sampler (4 dwords at +48 bytes) and the CUDA format (+80).
|
||||
// Compile flags (see build_native.sh): -ffp-contract=off, f32 denormals flushed (ZLUDA's
|
||||
// "denormal-fp-math-f32"="preserve-sign"), f16 denormals kept.
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
|
||||
#pragma clang fp contract(off)
|
||||
|
||||
#define RT static __attribute__((device, always_inline)) inline
|
||||
#define AS4 __attribute__((address_space(4)))
|
||||
#define AS3 __attribute__((address_space(3)))
|
||||
#define AS1 __attribute__((address_space(1)))
|
||||
#define AS5 __attribute__((address_space(5)))
|
||||
|
||||
typedef _Float16 h1;
|
||||
typedef _Float16 h2 __attribute__((ext_vector_type(2)));
|
||||
typedef float f4 __attribute__((ext_vector_type(4)));
|
||||
typedef float f2 __attribute__((ext_vector_type(2)));
|
||||
typedef int i2 __attribute__((ext_vector_type(2)));
|
||||
typedef int i8v __attribute__((ext_vector_type(8)));
|
||||
typedef int i4v __attribute__((ext_vector_type(4)));
|
||||
typedef uint32_t u4 __attribute__((ext_vector_type(4)));
|
||||
typedef uint32_t u2v __attribute__((ext_vector_type(2)));
|
||||
|
||||
// ---------------------------------------------------------------- bit casts
|
||||
RT float asf(uint32_t u) { return __builtin_bit_cast(float, u); }
|
||||
RT uint32_t asu(float f) { return __builtin_bit_cast(uint32_t, f); }
|
||||
RT h1 ash(uint16_t u) { return __builtin_bit_cast(h1, u); }
|
||||
RT uint16_t ashu(h1 h) { return __builtin_bit_cast(uint16_t, h); }
|
||||
RT h2 ash2(uint32_t u) { return __builtin_bit_cast(h2, u); }
|
||||
RT uint32_t ash2u(h2 h) { return __builtin_bit_cast(uint32_t, h); }
|
||||
RT uint32_t pack16(uint16_t lo, uint16_t hi) { return (uint32_t)lo | ((uint32_t)hi << 16); }
|
||||
|
||||
// ---------------------------------------------------------------- f32
|
||||
RT float canon(float x) { return __builtin_canonicalizef(x); }
|
||||
RT float fmaxn(float a, float b) { return __builtin_fmaximum_numf(a, b); }
|
||||
RT float fminn(float a, float b) { return __builtin_fminimum_numf(a, b); }
|
||||
RT float sat(float x) { return fminn(fmaxn(x, 0.0f), 1.0f); }
|
||||
RT float sat_ftz(float x) { return sat(canon(x)); } // cvt.ftz.sat.f32.f32
|
||||
RT float abs_ftz(float x) { return __builtin_fabsf(canon(x)); }
|
||||
RT float neg_ftz(float x) { return -canon(x); }
|
||||
RT float rcp_a(float x) { return __builtin_amdgcn_rcpf(x); } // rcp.approx(.ftz).f32
|
||||
RT float div_a(float a, float b) { return a * __builtin_amdgcn_rcpf(b); } // div.approx.ftz.f32 (fdiv arcp afn)
|
||||
RT float ex2_a(float x) { return __builtin_amdgcn_exp2f(x); }
|
||||
RT float lg2_a(float x) { return __builtin_amdgcn_logf(x); }
|
||||
RT float rsq_a(float x) { return __builtin_amdgcn_rsqf(x); }
|
||||
// cvt.rzi.s32.f32: saturating, NaN -> 0 (llvm.fptosi.sat, v_cvt_i32_f32); spelled out, a plain C cast is UB out of range
|
||||
// Branch-free: the operand is clamped into int range first (fmaxf(NaN, lo) = lo), so the cast is always defined and
|
||||
// the compiler can use selects (the guarded form compiled to a branch per conversion, splitting basic blocks).
|
||||
RT int32_t f2i_rz(float x)
|
||||
{
|
||||
#ifdef D4R_NATIVE_F2I
|
||||
// AMD V_CVT_I32_F32 truncates, saturates out-of-range inputs and maps NaN to zero.
|
||||
// Explicit instruction avoids redundant clamps/selects emitted for a C saturating cast.
|
||||
int32_t r;
|
||||
__asm__("v_cvt_i32_f32 %0, %1" : "=v"(r) : "v"(x));
|
||||
return r;
|
||||
#else
|
||||
const float c = __builtin_fminf(__builtin_fmaxf(x, -2147483648.0f), 2147483520.0f);
|
||||
int32_t r = (int32_t)c;
|
||||
r = x >= 2147483648.0f ? 2147483647 : r;
|
||||
return x != x ? 0 : r;
|
||||
#endif
|
||||
}
|
||||
|
||||
// cvt.rzi.u32.f32: ZLUDA emits a plain fptoui (frozen) and relies on V_CVT_U32_F32 (truncates; negative and NaN -> 0,
|
||||
// too large -> 0xffffffff), so the native helper is that instruction
|
||||
RT uint32_t f2u_rz(float x)
|
||||
{
|
||||
uint32_t r;
|
||||
__asm__("v_cvt_u32_f32 %0, %1" : "=v"(r) : "v"(x));
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- f16 (denormals kept)
|
||||
RT h1 hfma(h1 a, h1 b, h1 c) { return __builtin_fmaf16(a, b, c); }
|
||||
RT h2 h2fma(h2 a, h2 b, h2 c) { return __builtin_elementwise_fma(a, b, c); }
|
||||
RT h1 hmaxn(h1 a, h1 b) { return __builtin_fmaximum_numf16(a, b); }
|
||||
RT h1 hminn(h1 a, h1 b) { return __builtin_fminimum_numf16(a, b); }
|
||||
RT h1 hsat(h1 x) { return hminn(hmaxn(x, (h1)0.0f), (h1)1.0f); }
|
||||
RT h2 h2sat(h2 x) { return (h2){hsat(x.x), hsat(x.y)}; }
|
||||
RT h2 h2abs(h2 x) { return __builtin_elementwise_abs(x); }
|
||||
// ex2.approx.f16x2: v_exp_f32 per half (zluda_ptx_impl.cpp)
|
||||
RT h2 h2ex2(h2 a) { return (h2){(h1)__builtin_amdgcn_exp2f((float)a.x), (h1)__builtin_amdgcn_exp2f((float)a.y)}; }
|
||||
// cvt.rni/rmi.s32.f16: llvm.fptosi.sat.i32.f16 after the rounding (|f16| < 2^31, NaN -> 0)
|
||||
RT int32_t h2i_sat(h1 h)
|
||||
{
|
||||
#ifdef D4R_NATIVE_H2I
|
||||
return f2i_rz((float)h);
|
||||
#else
|
||||
return h != h ? 0 : (int32_t)(float)h;
|
||||
#endif
|
||||
}
|
||||
RT h1 h_roundeven(h1 h) { return __builtin_elementwise_roundeven(h); }
|
||||
RT h1 h_floor(h1 h) { return __builtin_elementwise_floor(h); }
|
||||
|
||||
// ---------------------------------------------------------------- threads (wave64: two CUDA warps per wave)
|
||||
RT uint32_t wave_lane() { return __builtin_amdgcn_mbcnt_hi(~0u, __builtin_amdgcn_mbcnt_lo(~0u, 0u)); }
|
||||
RT uint32_t warp_base()
|
||||
{
|
||||
#if __AMDGCN_WAVEFRONT_SIZE == 64
|
||||
return wave_lane() & 32u;
|
||||
#else
|
||||
return 0u;
|
||||
#endif
|
||||
}
|
||||
RT uint32_t laneid() { return wave_lane() & 31u; }
|
||||
// shfl.sync.bfly.b32 d|p, a, delta, opts, mask (ZLUDA's SHFL_SYNC_IMPL)
|
||||
RT uint32_t shfl_bfly(uint32_t input, int32_t delta, uint32_t opts, bool* in_bounds)
|
||||
{
|
||||
int32_t section_mask = (opts >> 8) & 31, warp_end = opts & 31;
|
||||
delta &= 31;
|
||||
int32_t self = (int32_t)laneid();
|
||||
int32_t subsection = section_mask & self;
|
||||
int32_t subsection_end = subsection | (~section_mask & warp_end);
|
||||
int32_t idx = self ^ delta;
|
||||
bool oob = idx > subsection_end;
|
||||
if (oob)
|
||||
idx = self;
|
||||
if (in_bounds)
|
||||
*in_bounds = !oob;
|
||||
return (uint32_t)__builtin_amdgcn_ds_bpermute((int32_t)(warp_base() | (uint32_t)idx) << 2, (int32_t)input);
|
||||
}
|
||||
RT uint32_t bfi32(uint32_t insert, uint32_t base, uint32_t pos_32, uint32_t len_32)
|
||||
{
|
||||
uint32_t pos = pos_32 & 0xffu, len = len_32 & 0xffu;
|
||||
if (pos >= 32)
|
||||
return base;
|
||||
uint32_t mask = len >= 32 ? (0xffffffffu << pos) : (((len >= 32 ? 0u : (1u << len)) - 1u) << pos);
|
||||
return (~mask & base) | (mask & (insert << pos));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- textures (ZLUDA handles)
|
||||
__attribute__((device)) static uint32_t d4r_null_texture[32];
|
||||
RT uint64_t tex_or_null(uint64_t t) { return t != 0 ? t : (uint64_t)(uintptr_t)d4r_null_texture; }
|
||||
RT i8v tdesc(uint64_t t) { return *(const AS4 i8v*)(uintptr_t)tex_or_null(t); }
|
||||
RT i4v sdesc(uint64_t t) { return *(const AS4 i4v*)(uintptr_t)(tex_or_null(t) + 48); }
|
||||
|
||||
__attribute__((device)) f4 llvm_image_load_mip_2d(uint32_t, int32_t, int32_t, int32_t, i8v, int, int)
|
||||
__asm("llvm.amdgcn.image.load.mip.2d.v4f32.i32");
|
||||
__attribute__((device)) f4 llvm_image_sample_l_2d(uint32_t, float, float, float, i8v, i4v, bool, int, int)
|
||||
__asm("llvm.amdgcn.image.sample.l.2d.v4f32.f32");
|
||||
extern "C" __attribute__((device)) f4 __ockl_image_gather4r_2D(const AS4 uint32_t* image, const AS4 uint32_t* sampler, f2 coord);
|
||||
|
||||
// tex.base.2d.v4.f32.s32: texel load, mip 0
|
||||
RT f4 tex_fetch(uint64_t t, int32_t x, int32_t y) { return llvm_image_load_mip_2d(0xf, x, y, 0, tdesc(t), 0, 0); }
|
||||
// tex.level.2d.v4.f32.f32
|
||||
RT f4 tex_level(uint64_t t, float x, float y, float lod) { return llvm_image_sample_l_2d(0xf, x, y, lod, tdesc(t), sdesc(t), false, 0, 0); }
|
||||
// tld4.r.2d.v4.f32.f32
|
||||
RT f4 tex_gather_r(uint64_t t, float x, float y)
|
||||
{
|
||||
const AS4 uint32_t* image = (const AS4 uint32_t*)(uintptr_t)tex_or_null(t);
|
||||
return __ockl_image_gather4r_2D(image, image + 12, (f2){x, y});
|
||||
}
|
||||
|
||||
// bar.sync 0 (HIP's __syncthreads)
|
||||
RT void bar_sync()
|
||||
{
|
||||
__builtin_amdgcn_fence(__ATOMIC_RELEASE, "workgroup");
|
||||
__builtin_amdgcn_s_barrier();
|
||||
__builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");
|
||||
}
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the staged output rewrite from the exact translated kernel.
|
||||
|
||||
Replaces path B's cooperative tile fill with output_tile.h. All other phases and
|
||||
the DLAA path remain intact. Refuses to generate if the region's register contract
|
||||
changes, so new translator output cannot silently reuse an incompatible mapping.
|
||||
"""
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
REGISTER = re.compile(r"\b(?:rd|rs|r|f|fd|p)\d+\b")
|
||||
ASSIGN = re.compile(r"^\s*((?:rd|rs|r|f|fd|p)\d+)\s*=")
|
||||
LABEL = re.compile(r"^(L__BB0_\d+):;$")
|
||||
|
||||
|
||||
def rewrite(source):
|
||||
head, main = source.split('extern "C" __attribute__((global))', 1)
|
||||
lines = main.splitlines(keepends=True)
|
||||
labels = [(i, LABEL.match(line.strip()).group(1)) for i, line in enumerate(lines) if LABEL.match(line.strip())]
|
||||
blocks = {name: (start, labels[k + 1][0] if k + 1 < len(labels) else len(lines))
|
||||
for k, (start, name) in enumerate(labels)}
|
||||
pending, region = ['L__BB0_124'], set()
|
||||
while pending:
|
||||
name = pending.pop()
|
||||
if name in region or name == 'L__BB0_138':
|
||||
continue
|
||||
region.add(name)
|
||||
start, end = blocks[name]
|
||||
body = ''.join(lines[start + 1:end])
|
||||
pending += re.findall(r'goto (L__BB0_\d+);', body)
|
||||
if not re.search(r'^\s*goto L__BB0_\d+;\s*$', lines[end - 1]):
|
||||
# This region's fallthroughs are only its internal branch joins.
|
||||
pending += [n for i, n in labels if i == end]
|
||||
expected = {'L__BB0_124'} | {f'L__BB0_{k}' for k in range(127, 138)}
|
||||
if region != expected:
|
||||
raise ValueError(f'tile CFG changed: {sorted(region)}')
|
||||
removed = {i for name in region for i in range(*blocks[name])}
|
||||
definitions = {m.group(1) for i in removed if (m := ASSIGN.match(lines[i]))}
|
||||
# Definitions declared at function scope count as uses. Ignore declaration-only lines.
|
||||
uses_outside = set(REGISTER.findall(''.join(line for i, line in enumerate(lines)
|
||||
if i not in removed and not re.match(r'^\s*(?:uint\d+_t|float|double|bool)\s+(?:rd|rs|r|f|fd|p)\d+(?:,|;)', line))))
|
||||
live_out = definitions & uses_outside
|
||||
if live_out != {'r215', 'r217'}:
|
||||
raise ValueError(f'tile register contract changed: {sorted(live_out)}')
|
||||
replacement = '''L__BB0_124:;
|
||||
output_tile_fill(KARGP, (int32_t)r3, (int32_t)r4, (int32_t)r7, (int32_t)(r8 + 1u),
|
||||
f1, (AS3 uint8_t*)sharedColor, (AS3 uint8_t*)sharedDepth);
|
||||
r215 = (uint32_t)__builtin_amdgcn_workitem_id_y();
|
||||
r217 = (uint32_t)__builtin_amdgcn_workitem_id_x();
|
||||
goto L__BB0_138;
|
||||
'''
|
||||
result = []
|
||||
for i, line in enumerate(lines):
|
||||
if i == blocks['L__BB0_124'][0]:
|
||||
result.append(replacement)
|
||||
if i not in removed:
|
||||
result.append(line)
|
||||
return head + '#include "output_tile.h"\n' + 'extern "C" __attribute__((global))' + ''.join(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('source', type=Path)
|
||||
args = parser.parse_args()
|
||||
print(rewrite(args.source.read_text()), end='')
|
||||
Reference in new issue
Block a user