ayo
22026ede12
Preset K, byte-identical to the previous output in all captured buffers and the harness image, measured on an RX 7700 XT: - LDS row padding so WMMA operand loads hit distinct banks - WGP mode for enc1/enc2/dec1/dec2, ds_bpermute half-wave exchange for enc0/dec0/dec3, more waves per window in enc4/dec4/dec5 - cheaper operand plumbing in common/wmma_layout.h (packed f16 conversions, no dynamic vector indexing), raw buffer weight loads, accumulator-init vectors prepared once - enc1 as a single-buffer recompute layer, dec1 Q/K/V chains split - dec5 as four dependent phase kernels
186 lines
7.8 KiB
C++
186 lines
7.8 KiB
C++
// Runtime for natively compiled DLSS texture kernels: PTX semantics (ptx_rt.h) plus surface stores
|
|
// (tex/tex_common.h, with ZLUDA's generic raw-store fallback ported here, since no ZLUDA bitcode is linked).
|
|
#pragma once
|
|
#include "ptx_rt.h"
|
|
#include "../tex/tex_common.h"
|
|
|
|
extern "C" __attribute__((device)) v4f __ockl_image_load_2D(tsharp_t* image, v2i coord);
|
|
|
|
namespace zsurf {
|
|
// zluda_ptx_impl.cpp surface_* helpers (raw sust.b stores), verbatim semantics
|
|
RT int channel_bytes(int dt)
|
|
{
|
|
switch (dt)
|
|
{
|
|
case 0: case 2: case 8: case 11: return 1;
|
|
case 1: case 3: case 9: case 12: case 14: return 2;
|
|
case 10: case 13: case 15: return 4;
|
|
default: return 0;
|
|
}
|
|
}
|
|
RT int channel_count(int order)
|
|
{
|
|
switch (order)
|
|
{
|
|
case 3: case 4: case 5: return 2;
|
|
case 6: case 12: return 3;
|
|
case 7: case 8: case 9: case 10: case 11: case 13: case 14: case 15: return 4;
|
|
default: return 1;
|
|
}
|
|
}
|
|
RT int memory_channel_lane(int order, int channel)
|
|
{
|
|
switch (order)
|
|
{
|
|
case 0: return 3;
|
|
case 5: return channel == 0 ? 0 : 3;
|
|
case 9: case 15: return channel == 0 ? 2 : channel == 2 ? 0 : channel;
|
|
case 10: return channel == 0 ? 3 : channel - 1;
|
|
case 11: return 3 - channel;
|
|
default: return channel;
|
|
}
|
|
}
|
|
RT uint32_t float_to_half_bits(float value)
|
|
{
|
|
uint32_t bits = __builtin_bit_cast(uint32_t, value);
|
|
uint32_t sign = (bits >> 16) & 0x8000u, exponent = (bits >> 23) & 0xffu, mantissa = bits & 0x7fffffu;
|
|
if (exponent == 0xff)
|
|
return sign | 0x7c00u | (mantissa != 0 ? 0x200u | (mantissa >> 13) : 0u);
|
|
int half_exponent = int(exponent) - 112;
|
|
if (half_exponent >= 31)
|
|
return sign | 0x7c00u;
|
|
uint32_t shift = 13;
|
|
if (half_exponent <= 0)
|
|
{
|
|
if (half_exponent < -10)
|
|
return sign;
|
|
mantissa |= 0x800000u;
|
|
shift = uint32_t(14 - half_exponent);
|
|
half_exponent = 0;
|
|
}
|
|
uint32_t half_mantissa = mantissa >> shift, remainder = mantissa & ((1u << shift) - 1u), halfway = 1u << (shift - 1u);
|
|
uint32_t result = sign | (uint32_t(half_exponent) << 10);
|
|
result += half_mantissa & (half_exponent == 0 ? 0x7ffu : 0x3ffu);
|
|
if (remainder > halfway || (remainder == halfway && (half_mantissa & 1u)))
|
|
result++;
|
|
return result;
|
|
}
|
|
RT int32_t clampi(int32_t v, int32_t lo, int32_t hi) { return v < lo ? lo : (v > hi ? hi : v); }
|
|
RT float decode(uint32_t bits, int dt)
|
|
{
|
|
switch (dt)
|
|
{
|
|
case 0: return float(clampi(int32_t(int8_t(bits)), -127, 127)) / 127.0f;
|
|
case 1: return float(clampi(int32_t(int16_t(bits)), -32767, 32767)) / 32767.0f;
|
|
case 2: return float(int32_t(bits & 0xffu)) / 255.0f;
|
|
case 3: return float(int32_t(bits & 0xffffu)) / 65535.0f;
|
|
case 8: return __builtin_bit_cast(float, int32_t(int8_t(bits)));
|
|
case 9: return __builtin_bit_cast(float, int32_t(int16_t(bits)));
|
|
case 14: return half_bits_to_float(bits);
|
|
default: return __builtin_bit_cast(float, bits);
|
|
}
|
|
}
|
|
RT uint32_t encode(float value, int dt)
|
|
{
|
|
switch (dt)
|
|
{
|
|
case 0: return uint32_t(clampi(int32_t(__builtin_floorf(value * 127.0f + 0.5f)), -127, 127)) & 0xffu;
|
|
case 1: return uint32_t(clampi(int32_t(__builtin_floorf(value * 32767.0f + 0.5f)), -32767, 32767)) & 0xffffu;
|
|
case 2: return uint32_t(clampi(int32_t(__builtin_floorf(value * 255.0f + 0.5f)), 0, 255));
|
|
case 3: return uint32_t(clampi(int32_t(__builtin_floorf(value * 65535.0f + 0.5f)), 0, 65535));
|
|
case 14: return float_to_half_bits(value);
|
|
default: return __builtin_bit_cast(uint32_t, value);
|
|
}
|
|
}
|
|
RT int lane_channel(int order, int lane, int channels)
|
|
{
|
|
switch (order)
|
|
{
|
|
case 0: return lane == 3 ? 0 : -1;
|
|
case 5: return lane == 0 ? 0 : (lane == 3 ? 1 : -1);
|
|
case 9: case 15: return lane == 0 ? 2 : (lane == 2 ? 0 : lane);
|
|
case 10: return lane == 3 ? 0 : lane + 1;
|
|
case 11: return 3 - lane;
|
|
default: return lane < channels ? lane : -1;
|
|
}
|
|
}
|
|
RT uint32_t sel_channel(uint32_t c0, uint32_t c1, uint32_t c2, uint32_t c3, int ch)
|
|
{
|
|
return ch == 0 ? c0 : (ch == 1 ? c1 : (ch == 2 ? c2 : c3));
|
|
}
|
|
RT float sel_lane(v4f v, int lane) { return lane == 0 ? v.x : (lane == 1 ? v.y : (lane == 2 ? v.z : v.w)); }
|
|
RT uint32_t merge_channel(uint32_t bits, int channel, int cb, uint32_t data, int byte_offset, int byte_count)
|
|
{
|
|
for (int b = 0; b < 4; b++)
|
|
{
|
|
int relative = channel * cb + b - byte_offset;
|
|
if (b < cb && relative >= 0 && relative < byte_count)
|
|
{
|
|
uint32_t byte = (data >> (8 * relative)) & 0xffu;
|
|
bits = (bits & ~(0xffu << (8 * b))) | (byte << (8 * b));
|
|
}
|
|
}
|
|
return bits;
|
|
}
|
|
RT float lane_value(uint32_t c0, uint32_t c1, uint32_t c2, uint32_t c3, int order, int lane, int channels, int dt)
|
|
{
|
|
int ch = lane_channel(order, lane, channels);
|
|
return ch < 0 ? 0.0f : decode(sel_channel(c0, c1, c2, c3, ch), dt);
|
|
}
|
|
__attribute__((device, noinline)) static void store_raw_2d(tsharp_t* image, int32_t cx, int32_t cy, uint32_t data, int byte_count)
|
|
{
|
|
int dt = __ockl_image_channel_data_type_2D(image), order = __ockl_image_channel_order_2D(image);
|
|
int cb = channel_bytes(dt);
|
|
if (cb == 0 || cx < 0)
|
|
return;
|
|
int channels = channel_count(order), pixel_bytes = cb * channels;
|
|
for (int consumed = 0; consumed < byte_count;)
|
|
{
|
|
int byte_x = cx + consumed, pixel_x = byte_x / pixel_bytes, byte_offset = byte_x - pixel_x * pixel_bytes;
|
|
int chunk = pixel_bytes - byte_offset;
|
|
if (chunk > byte_count - consumed)
|
|
chunk = byte_count - consumed;
|
|
uint32_t fragment = data >> (8 * consumed);
|
|
uint32_t c0 = 0, c1 = 0, c2 = 0, c3 = 0;
|
|
if (byte_offset != 0 || chunk < pixel_bytes)
|
|
{
|
|
v4f loaded = __ockl_image_load_2D(image, (v2i){pixel_x, cy});
|
|
c0 = encode(sel_lane(loaded, memory_channel_lane(order, 0)), dt);
|
|
c1 = encode(sel_lane(loaded, memory_channel_lane(order, 1)), dt);
|
|
c2 = encode(sel_lane(loaded, memory_channel_lane(order, 2)), dt);
|
|
c3 = encode(sel_lane(loaded, memory_channel_lane(order, 3)), dt);
|
|
}
|
|
c0 = merge_channel(c0, 0, cb, fragment, byte_offset, chunk);
|
|
c1 = merge_channel(c1, 1, cb, fragment, byte_offset, chunk);
|
|
c2 = merge_channel(c2, 2, cb, fragment, byte_offset, chunk);
|
|
c3 = merge_channel(c3, 3, cb, fragment, byte_offset, chunk);
|
|
__ockl_image_store_2D(image, (v2i){pixel_x, cy},
|
|
(v4f){lane_value(c0, c1, c2, c3, order, 0, channels, dt), lane_value(c0, c1, c2, c3, order, 1, channels, dt),
|
|
lane_value(c0, c1, c2, c3, order, 2, channels, dt), lane_value(c0, c1, c2, c3, order, 3, channels, dt)});
|
|
consumed += chunk;
|
|
}
|
|
}
|
|
} // namespace zsurf
|
|
|
|
extern "C" __attribute__((device)) void __zluda_ptx_impl_surfobj_b_2d_b32_zero(uint64_t surface, v2i coord, uint32_t value)
|
|
{
|
|
zsurf::store_raw_2d((tsharp_t*)(uintptr_t)surface, coord.x, coord.y, value, 4);
|
|
}
|
|
extern "C" __attribute__((device)) void __zluda_ptx_impl_surfobj_b_2d_v2_b16_zero(uint64_t surface, v2i coord, v2s value)
|
|
{
|
|
zsurf::store_raw_2d((tsharp_t*)(uintptr_t)surface, coord.x, coord.y, uint32_t(uint16_t(value.x)) | (uint32_t(uint16_t(value.y)) << 16), 4);
|
|
}
|
|
|
|
// A sample that every lane takes with the same handle, coordinates and lod (e.g. a 1x1 exposure texture at (0.5,
|
|
// 0.5)): only the first active lane samples and the result is broadcast with readfirstlane. Same values, but the
|
|
// texture unit processes one lane instead of the whole wave.
|
|
RT f4 tex_level_uniform(uint64_t t, float x, float y, float lod)
|
|
{
|
|
f4 r = (f4){0.0f, 0.0f, 0.0f, 0.0f};
|
|
if (wave_lane() == (uint32_t)__builtin_ctzll(__builtin_amdgcn_read_exec()))
|
|
r = tex_level(t, x, y, lod);
|
|
return (f4){asf(__builtin_amdgcn_readfirstlane(asu(r[0]))), asf(__builtin_amdgcn_readfirstlane(asu(r[1]))),
|
|
asf(__builtin_amdgcn_readfirstlane(asu(r[2]))), asf(__builtin_amdgcn_readfirstlane(asu(r[3])))};
|
|
}
|
|
|