wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Port K kernel tuning and M gfx1101 flags

Browse Source 
Preset K, byte-identical to the previous output in all captured buffers
and the harness image, measured on an RX 7700 XT:

- LDS row padding so WMMA operand loads hit distinct banks

- WGP mode for enc1/enc2/dec1/dec2, ds_bpermute half-wave exchange for
  enc0/dec0/dec3, more waves per window in enc4/dec4/dec5

- cheaper operand plumbing in common/wmma_layout.h (packed f16
  conversions, no dynamic vector indexing), raw buffer weight loads,
  accumulator-init vectors prepared once

- enc1 as a single-buffer recompute layer, dec1 Q/K/V chains split

- dec5 as four dependent phase kernels
This commit is contained in:
ayo committed 2026-10-05 14:48:37 -04:00
1 parent 8ce8618181
commit 22026ede12
45 files changed
+4514 -354
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

kernels/k/pwin_wide.h
+124 -102
View File
@@ -15,27 +15,71 @@ template <class L, int NG> struct WideDims
// NA: heads per attention round (their K / V images share R)
static constexpr int NA = NG < 4 ? NG : 4;
static constexpr int C = L::C, KT = L::KT, H = L::H, NTL = (KT + NG - 1) / NG, ROUNDS = (H + NA - 1) / NA;
static constexpr int X0_BYTES = 64 * C * 2, KV_BYTES = NA * 2 * 64 * 32 * 2;
// K rows (64 x 32), V^T rows (32 x 64), O rows (64 x 32), hidden rows (64 x 32): all padded (LDSR)
static constexpr int KL_BYTES = 64 * LDSR(32) * 2, VT_BYTES = 32 * LDSR(64) * 2;
static constexpr int X0_BYTES = 64 * LDSR(C) * 2, KV_BYTES = NA * (KL_BYTES + VT_BYTES);
// MLP hidden chunks: one per group and round, double-buffered while that fits
static constexpr int NB = NG <= 4 ? 2 : 1, MROUNDS = (L::NMLP + NG - 1) / NG, HID_BYTES = NB * NG * 64 * 32 * 2;
static constexpr int NB = NG <= 4 ? 2 : 1, MROUNDS = (L::NMLP + NG - 1) / NG, HID_BYTES = NB * NG * 64 * LDSR(32) * 2;
static constexpr int R_BYTES = X0_BYTES > KV_BYTES ? (X0_BYTES > HID_BYTES ? X0_BYTES : HID_BYTES)
: (KV_BYTES > HID_BYTES ? KV_BYTES : HID_BYTES);
static_assert(!L::POS, "position-only attention layers use pwin_pos.h");
};
// QKV chains u = U0 .. U0 + NU - 1 of head h for this wave's token tile (u = 2 j + nt; j: Q, K, V): Q stays in
// registers (qop), K and V^T go to LDS slot s. Each chain runs over kt in order (exact).
template <class L, int U0, int NU>
__device__ __forceinline__ void wide_qkv(const u8v* __restrict__ img, const half_t* hrow, int h, int s, int tok,
op_t (&qop)[2], half_t (*kl)[64][LDSR(32)], half_t (*vt)[32][LDSR(64)])
{
constexpr int KT = L::KT;
const uint32_t m = lane_id() & 15;
f8v dq[NU];
#pragma unroll
for (int uu = 0; uu < NU; ++uu)
dq[uu] = splat8(0.0f);
for (int kt = 0; kt < KT; ++kt)
{
const op_t row = op_lds(hrow + 16 * kt);
#pragma unroll
for (int uu = 0; uu < NU; ++uu)
{
const int u = U0 + uu;
dq[uu] = mma16(op_imgt(img, L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)), row, dq[uu]);
}
}
#pragma unroll
for (int uu = 0; uu < NU; ++uu)
{
const int u = U0 + uu, j = u >> 1, nt = u & 1;
const f8v d = dq[uu];
if (j == 0)
qop[nt] = operand_from_f8(d);
else if (j == 1)
op_store(&kl[s][tok][16 * nt], operand_from_f8(d));
else
{
#pragma unroll
for (int i = 0; i < 8; ++i)
vt[s][16 * nt + wm_acc_row(i)][tok] = (half_t)d[i];
}
}
}
// Core for one window. On entry R holds x0 (64 x C); on return hb holds the block output y.
template <class L, int NG>
__device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, const u4v* __restrict__ bias,
half_t (*hb)[L::C], uint8_t* R)
half_t (*hb)[LDSR(L::C)], uint8_t* R)
{
using D = WideDims<L, NG>;
constexpr int C = L::C, KT = L::KT, H = L::H, NTL = D::NTL;
const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
const int wv = threadIdx.z, t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const int wv = __builtin_amdgcn_readfirstlane(threadIdx.z), t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const uint8_t* w = p.w;
half_t(*act)[C] = (half_t(*)[C])R;
half_t(*act)[LDSR(C)] = (half_t(*)[LDSR(C)])R;
const half_t* arow = act[tok];
half_t* hrow = hb[tok];
PROF_INIT;
PROF(1); // entry (x0 load + barrier)
// ---- L2 norm: h1 = x0 * (rsqrt(sum x0^2) * gamma1) -> hb (all groups of a token tile compute the sum)
float ss = 0.0f;
@@ -49,7 +93,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
ss += (float)(half_t)(v * v);
}
}
const half_t r16 = (half_t)__builtin_amdgcn_rsqf((float)(half_t)ss);
const half_t r16 = half_rsqrt((half_t)ss);
for (int kt = 2 * g + (int)hf; kt < KT; kt += 2 * NG)
{
const u8v xv = lds_row16(arow + 16 * kt);
@@ -76,6 +120,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
}
}
block_sync(); // hb complete, R free
PROF(2); // norm
// ---- attention: NG heads per round; the output projection accumulates head by head
f8v acc[NTL];
@@ -84,65 +129,57 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
{
const int nt = g + NG * j;
if (nt < KT)
{
half_t vv[8];
dvec8(w, L::BO + 32 * nt, vv);
#pragma unroll
for (int i = 0; i < 8; ++i)
acc[j][i] = (float)vv[i];
}
acc[j] = acc_vec<L>(img, w, L::BO + 32 * nt, L::V_BO + nt);
}
half_t(*kl)[64][32] = (half_t(*)[64][32])R; // [group][token][dim]
half_t(*kl)[64][LDSR(32)] = (half_t(*)[64][LDSR(32)])R; // [group][token][dim]
constexpr int NA = D::NA;
half_t(*vt)[32][64] = (half_t(*)[32][64])(R + NA * 64 * 32 * 2); // [group][dim][token]
half_t(*ob)[64][32] = (half_t(*)[64][32])R; // [group][token][dim] (after K / V)
half_t(*vt)[32][LDSR(64)] = (half_t(*)[32][LDSR(64)])(R + NA * D::KL_BYTES); // [group][dim][token]
half_t(*ob)[64][LDSR(32)] = (half_t(*)[64][LDSR(32)])R; // [group][token][dim] (after K / V)
// NG >= 2 NA: the groups beyond the NA heads of a round would idle during QKV and attention; instead group
// g + NA computes half of the six QKV chains of group g's head (K tile 1 and V), group g the other half
// (Q, kept in registers for its attention, and K tile 0)
constexpr bool SPLIT = NG >= 2 * NA;
const int hs = SPLIT ? g % NA : g, part = SPLIT ? g / NA : 0;
for (int r = 0; r < D::ROUNDS; ++r)
{
const int h = g < NA ? r * NA + g : H;
const int h = (SPLIT ? g < 2 * NA : g < NA) ? r * NA + hs : H;
const bool att = h < H && part == 0;
op_t qop[2], oop[2];
#ifdef PWIN_BIAS_PREFETCH
u4v bpre[4];
if (att)
{
#pragma unroll
for (int kt = 0; kt < 4; ++kt)
bpre[kt] = bias[((h * 4 + t) * 4 + kt) * 32 + l];
}
#endif
if (h < H)
{
// six independent chains (Q, K, V x 2 n-tiles), each over kt in order
f8v dq[6];
#pragma unroll
for (int u = 0; u < 6; ++u)
dq[u] = splat8(0.0f);
for (int kt = 0; kt < KT; ++kt)
if constexpr (SPLIT)
{
const op_t row = op_lds(hrow + 16 * kt);
#pragma unroll
for (int u = 0; u < 6; ++u)
dq[u] = mma16(op_img(img, (L::T_QKV + ((h * 3 + (u >> 1)) * KT + kt) * 2 + (u & 1)) * 16 + m), row, dq[u]);
if (part == 0)
wide_qkv<L, 0, 3>(img, hrow, h, hs, tok, qop, kl, vt);
else
wide_qkv<L, 3, 3>(img, hrow, h, hs, tok, qop, kl, vt);
}
#pragma unroll
for (int j = 0; j < 3; ++j)
#pragma unroll
for (int nt = 0; nt < 2; ++nt)
{
const f8v d = dq[2 * j + nt];
if (j == 0)
qop[nt] = operand_from_f8(d);
else if (j == 1)
{
op_store(&kl[g][tok][16 * nt], operand_from_f8(d));
}
else
{
#pragma unroll
for (int i = 0; i < 8; ++i)
vt[g][16 * nt + wm_acc_row(i)][tok] = (half_t)d[i];
}
}
else
wide_qkv<L, 0, 6>(img, hrow, h, hs, tok, qop, kl, vt);
}
block_sync();
if (h < H)
PROF(3); // QKV
if (att)
{
half_t e[4][8];
float rs = 0.0f;
#pragma unroll
for (int kt = 0; kt < 4; ++kt)
{
#ifdef PWIN_BIAS_PREFETCH
const u4v bi = bpre[kt];
#else
const u4v bi = bias[((h * 4 + t) * 4 + kt) * 32 + l];
#endif
f8v d;
#pragma unroll
for (int i = 0; i < 4; ++i)
@@ -152,7 +189,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
}
#pragma unroll
for (int ds = 0; ds < 2; ++ds)
d = mma16(op_lds(&kl[g][16 * kt + m][16 * ds]), qop[ds], d);
d = mma16(op_lds(&kl[hs][16 * kt + m][16 * ds]), qop[ds], d);
#pragma unroll
for (int i = 0; i < 8; ++i)
{
@@ -162,7 +199,7 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
}
}
rs = row_sum_total(rs, e);
const half_t rc = (half_t)__builtin_amdgcn_rcpf((float)(half_t)rs);
const half_t rc = half_reciprocal((half_t)rs);
op_t pop[4];
#pragma unroll
for (int kt = 0; kt < 4; ++kt)
@@ -179,15 +216,17 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
f8v d = splat8(0.0f);
#pragma unroll
for (int kt = 0; kt < 4; ++kt)
d = mma16(op_lds(&vt[g][16 * nt + m][16 * kt]), pop[kt], d);
d = mma16(op_lds(&vt[hs][16 * nt + m][16 * kt]), pop[kt], d);
oop[nt] = operand_from_f8(d);
}
}
PROF(4); // attention
block_sync(); // K / V of the round consumed
if (h < H)
PROF(5); // barrier after attention
if (att)
{
op_store(&ob[g][tok][0], oop[0]);
op_store(&ob[g][tok][16], oop[1]);
op_store(&ob[hs][tok][0], oop[0]);
op_store(&ob[hs][tok][16], oop[1]);
}
block_sync();
for (int jh = 0; jh < NA; ++jh)
@@ -202,13 +241,14 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
const int nt = g + NG * j;
if (nt < KT)
{
acc[j] = mma16(op_img(img, (L::T_WO + (2 * hh) * KT + nt) * 16 + m), o0, acc[j]);
acc[j] = mma16(op_img(img, (L::T_WO + (2 * hh + 1) * KT + nt) * 16 + m), o1, acc[j]);
acc[j] = mma16(op_imgt(img, L::T_WO + (2 * hh) * KT + nt), o0, acc[j]);
acc[j] = mma16(op_imgt(img, L::T_WO + (2 * hh + 1) * KT + nt), o1, acc[j]);
}
}
}
if (r + 1 < D::ROUNDS)
block_sync(); // ob is overwritten by the next round's K / V
PROF(6); // O exchange + Wo
}
// ---- residual; MLP init acc = x1 + b2, MLP input m = x1 * g2 (-> hb)
@@ -219,8 +259,8 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
if (nt < KT)
{
half_t mv[8], g2[8], b2[8];
dvec8(w, L::G2 + 32 * nt, g2);
dvec8(w, L::B2 + 32 * nt, b2);
residual_vector<L>(w, 0, nt, g2);
residual_vector<L>(w, 1, nt, b2);
#pragma unroll
for (int i = 0; i < 8; ++i)
{
@@ -232,9 +272,10 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
}
}
block_sync();
PROF(7); // residual
// ---- MLP: rounds of NG chunks of 32 hidden (double-buffered in R)
half_t(*hid)[NG][64][32] = (half_t(*)[NG][64][32])R;
half_t(*hid)[NG][64][LDSR(32)] = (half_t(*)[NG][64][LDSR(32)])R;
for (int rr = 0; rr < D::MROUNDS; ++rr)
{
const int c = rr * NG + g;
@@ -243,21 +284,14 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
{
if (c >= L::NMLP)
break;
f8v d;
{
half_t vv[8];
dvec8(w, L::B1 + 64 * c + 32 * hn, vv);
#pragma unroll
for (int i = 0; i < 8; ++i)
d[i] = (float)vv[i];
}
f8v d = acc_vec<L>(img, w, L::B1 + 64 * c + 32 * hn, L::V_B1 + 2 * c + hn);
for (int kt = 0; kt < KT; ++kt)
d = mma16(op_img(img, (L::T_W1 + (c * KT + kt) * 2 + hn) * 16 + m), op_lds(hrow + 16 * kt), d);
half_t gg[8];
gelu8(d, gg);
op_store(&hid[rr % D::NB][g][tok][16 * hn], operand_from_dt(gg));
d = mma16(op_imgt(img, L::T_W1 + (c * KT + kt) * 2 + hn), op_lds(hrow + 16 * kt), d);
op_store(&hid[rr % D::NB][g][tok][16 * hn], gelu_op(d));
}
PROF(8); // MLP W1 + gelu
block_sync();
PROF(9); // MLP barrier
for (int jc = 0; jc < NG; ++jc)
{
const int cc = rr * NG + jc;
@@ -270,13 +304,14 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
const int nt = g + NG * j;
if (nt < KT)
{
acc[j] = mma16(op_img(img, (L::T_W2 + (cc * 2) * KT + nt) * 16 + m), h0, acc[j]);
acc[j] = mma16(op_img(img, (L::T_W2 + (cc * 2 + 1) * KT + nt) * 16 + m), h1, acc[j]);
acc[j] = mma16(op_imgt(img, L::T_W2 + (cc * 2) * KT + nt), h0, acc[j]);
acc[j] = mma16(op_imgt(img, L::T_W2 + (cc * 2 + 1) * KT + nt), h1, acc[j]);
}
}
}
if (D::NB == 1 && rr + 1 < D::MROUNDS)
block_sync();
PROF(10); // MLP W2
}
// y -> hb (the last reads of hb were the W1 operands before the final barrier)
#pragma unroll
@@ -289,15 +324,16 @@ __device__ void wide_core(const PwinParams& p, const u8v* __restrict__ img, cons
}
}
block_sync();
PROF(11); // y store + barrier
}
// x0 rows of a window (mirror padded) -> R
template <class L, int NG> __device__ void wide_load_x0(const PwinParams& p, uint8_t* R)
{
constexpr int C = L::C;
half_t(*act)[C] = (half_t(*)[C])R;
half_t(*act)[LDSR(C)] = (half_t(*)[LDSR(C)])R;
const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
const int wv = threadIdx.z, t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const int wv = __builtin_amdgcn_readfirstlane(threadIdx.z), t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const int X = mirror(8 * (int)blockIdx.x - p.sx + (tok & 7), p.W), Y = mirror(8 * (int)blockIdx.y - p.sy + (tok >> 3), p.H);
const half_t* xin = (const half_t*)p.in + (size_t)(Y * p.W + X) * C;
for (int kt = 2 * g + (int)hf; kt < L::KT; kt += 2 * NG)
@@ -305,10 +341,10 @@ template <class L, int NG> __device__ void wide_load_x0(const PwinParams& p, uin
}
// y rows of the window's own tokens -> dst (C per token)
template <class L, int NG> __device__ void wide_store_y(const PwinParams& p, half_t (*hb)[L::C], uint8_t* dst)
template <class L, int NG> __device__ void wide_store_y(const PwinParams& p, half_t (*hb)[LDSR(L::C)], uint8_t* dst)
{
const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
const int wv = threadIdx.z, t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const int wv = __builtin_amdgcn_readfirstlane(threadIdx.z), t = wv & 3, g = wv >> 2, tok = 16 * t + m;
const int Y0 = 8 * (int)blockIdx.y - p.sy + (tok >> 3), X0 = 8 * (int)blockIdx.x - p.sx + (tok & 7);
if (Y0 >= 0 && Y0 < p.H && X0 >= 0 && X0 < p.W)
{
@@ -322,7 +358,7 @@ template <class L, int NG>
__device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, const u4v* __restrict__ bias)
{
constexpr int C = L::C, COUT = L::COUT;
__shared__ __attribute__((aligned(16))) half_t hb[64][C];
__shared__ __attribute__((aligned(16))) half_t hb[64][LDSR(C)];
__shared__ __attribute__((aligned(16))) uint8_t R[WideDims<L, NG>::R_BYTES];
wide_load_x0<L, NG>(p, R);
block_sync();
@@ -331,7 +367,7 @@ __device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, c
// patch merge: 16 merged tokens, K = 4 sub-tokens x C, the allocated n-tiles over the waves
const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
const int wv = threadIdx.z;
const int wv = __builtin_amdgcn_readfirstlane(threadIdx.z);
const int my = m >> 2, mx = m & 3;
const int Ym = (8 * (int)blockIdx.y - p.sy) / 2 + my, Xm = (8 * (int)blockIdx.x - p.sx) / 2 + mx;
const bool minb = Ym >= 0 && Ym < p.H / 2 && Xm >= 0 && Xm < p.W / 2;
@@ -339,19 +375,12 @@ __device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, c
const uint8_t* w = p.w;
for (int vt = wv; vt < L::PMT; vt += 4 * NG)
{
f8v d;
{
half_t vv[8];
dvec8(w, L::PMB + 32 * vt, vv);
#pragma unroll
for (int i = 0; i < 8; ++i)
d[i] = (float)vv[i];
}
f8v d = acc_vec<L>(img, w, L::PMB + 32 * vt, L::V_PM + vt);
for (int ks = 0; ks < C / 4; ++ks)
{
const int sub = ks / L::KT, dy = sub >> 1, dx = sub & 1;
const int src = 8 * (2 * my + dy) + 2 * mx + dx;
d = mma16(op_img(img, (L::T_PM + ks * L::PMT + vt) * 16 + m), op_lds(&hb[src][16 * (ks % L::KT)]), d);
d = mma16(op_imgt(img, L::T_PM + ks * L::PMT + vt), op_lds(&hb[src][16 * (ks % L::KT)]), d);
}
const int gw = vt / (L::NPA / 16), u = vt % (L::NPA / 16), valid = L::NPW - 16 * u;
op_gstore(mout + L::NPW * gw + 16 * u, operand_from_f8(d), minb, valid);
@@ -362,7 +391,7 @@ __device__ void wide_encoder(const PwinParams& p, const u8v* __restrict__ img, c
template <class L, int NG>
__device__ void wide_plain(const PwinParams& p, const u8v* __restrict__ img, const u4v* __restrict__ bias)
{
__shared__ __attribute__((aligned(16))) half_t hb[64][L::C];
__shared__ __attribute__((aligned(16))) half_t hb[64][LDSR(L::C)];
__shared__ __attribute__((aligned(16))) uint8_t R[WideDims<L, NG>::R_BYTES];
wide_load_x0<L, NG>(p, R);
block_sync();
@@ -374,11 +403,11 @@ template <class L, int NG>
__device__ void wide_decoder(const PwinParams& p, const u8v* __restrict__ img, const u4v* __restrict__ bias)
{
constexpr int C = L::C;
__shared__ __attribute__((aligned(16))) half_t hb[64][C];
__shared__ __attribute__((aligned(16))) half_t hb[64][LDSR(C)];
__shared__ __attribute__((aligned(16))) uint8_t R[WideDims<L, NG>::R_BYTES];
half_t(*act)[C] = (half_t(*)[C])R;
half_t(*act)[LDSR(C)] = (half_t(*)[LDSR(C)])R;
const uint32_t l = lane_id(), m = l & 15, hf = l >> 4;
const int wv = threadIdx.z, q = wv & 3, g = wv >> 2, bx = blockIdx.x, by = blockIdx.y;
const int wv = __builtin_amdgcn_readfirstlane(threadIdx.z), q = wv & 3, g = wv >> 2, bx = blockIdx.x, by = blockIdx.y;
// ---- patch expand: waves of sub-token q = (dy, dx) produce its tiles nt = g + NG j for the 16 low-res tokens
{
@@ -392,16 +421,9 @@ __device__ void wide_decoder(const PwinParams& p, const u8v* __restrict__ img, c
for (int nt = g; nt < C / 16; nt += NG)
{
const int gnt = q * (C / 16) + nt;
f8v d;
{
half_t vv[8];
dvec8(p.w, L::EXPB + 32 * gnt, vv);
#pragma unroll
for (int i = 0; i < 8; ++i)
d[i] = (float)vv[i];
}
f8v d = acc_vec<L>(img, p.w, L::EXPB + 32 * gnt, L::V_EXP + gnt);
for (int ks = 0; ks < L::KL; ++ks)
d = mma16(op_img(img, (L::T_EXP + ks * L::EXPN + gnt) * 16 + m), op_gload(xl + 16 * ks), d);
d = mma16(op_imgt(img, L::T_EXP + ks * L::EXPN + gnt), op_gload(xl + 16 * ks), d);
const op_t sv = op_gload(sk + 16 * nt);
half_t xo[8];
#pragma unroll
@@ -420,7 +442,7 @@ __device__ void wide_decoder(const PwinParams& p, const u8v* __restrict__ img, c
#define WIDE_KERNEL(NAME, PREP, BODY, NG, ...) \
using NAME##_L = __VA_ARGS__; \
__device__ u8v g_img[NAME##_L::T_END * 16]; \
__device__ u8v g_img[NAME##_L::IMG_SLOTS]; \
__device__ u4v g_bias[NAME##_L::NBIAS]; \
extern "C" __device__ __attribute__((used)) uint32_t d4r_prep_blocks = (NAME##_L::PREP_ITEMS + 127) / 128; \
extern "C" __device__ __attribute__((used)) uint32_t d4r_block_z = 4 * (NG); \
@@ -435,5 +457,5 @@ __device__ void wide_decoder(const PwinParams& p, const u8v* __restrict__ img, c
}
#define WIDE_ENCODER(NAME, H, C, COUT, NG) WIDE_KERNEL(NAME, pwin_prep, wide_encoder, NG, PwinLayout<H, C, COUT>)
#define WIDE_PLAIN(NAME, H, C, NG) WIDE_KERNEL(NAME, pwin_prep, wide_plain, NG, PwinLayout<H, C, 16>)
#define WIDE_PLAIN(NAME, H, C, NG) WIDE_KERNEL(NAME, pwin_prep_nomerge, wide_plain, NG, PwinLayout<H, C, 16>)
#define WIDE_DECODER(NAME, H, C, CL, NG) WIDE_KERNEL(NAME, pwin_dec_prep, wide_decoder, NG, PwinDecLayout<H, C, CL>)