feat: add d4r inference engines and Windows parity tooling

Add native K/M/RR inference engines, HIP kernels, replay and validation
harnesses, and Windows DLSS setup and runtime support. Include GPU
profiling tools, build and packaging scripts, platform integration
patches, and implementation documentation.

kernels/common/hip_device_minimal.h

@@ -0,0 +1,43 @@
+#pragma once
+
+// Device-only Windows HIP build shim. The diagnostic/kernel code objects are
+// compiled with -nogpuinc/-nogpulib; no host HIP or C++ standard library is
+// needed. Definitions below mirror public HIP qualifiers and LLVM AMDGPU
+// work-item/grid builtins. The normal Linux HIP include path is untouched.
+#define __device__ __attribute__((device))
+#define __host__ __attribute__((host))
+#define __global__ __attribute__((global))
+#define __shared__ __attribute__((shared))
+#define __constant__ __attribute__((constant))
+#define __forceinline__ __attribute__((always_inline)) inline
+#define __launch_bounds__(n) __attribute__((amdgpu_flat_work_group_size(1, n)))
+
+struct d4r_device_dim3 { unsigned x, y, z; };
+// Public HIP vector storage ABI used by M's packed global/LDS accesses.
+struct alignas(8) uint2 { unsigned x, y; };
+struct alignas(16) uint4 { unsigned x, y, z, w; };
+static_assert(sizeof(uint2) == 8 && alignof(uint2) == 8, "HIP uint2 ABI");
+static_assert(sizeof(uint4) == 16 && alignof(uint4) == 16, "HIP uint4 ABI");
+__device__ __forceinline__ d4r_device_dim3 d4r_thread_idx() {
+    return {__builtin_amdgcn_workitem_id_x(), __builtin_amdgcn_workitem_id_y(),
+        __builtin_amdgcn_workitem_id_z()};
+}
+__device__ __forceinline__ d4r_device_dim3 d4r_block_idx() {
+    return {__builtin_amdgcn_workgroup_id_x(), __builtin_amdgcn_workgroup_id_y(),
+        __builtin_amdgcn_workgroup_id_z()};
+}
+__device__ __forceinline__ d4r_device_dim3 d4r_grid_dim() {
+    return {__builtin_amdgcn_grid_size_x() / __builtin_amdgcn_workgroup_size_x(),
+        __builtin_amdgcn_grid_size_y() / __builtin_amdgcn_workgroup_size_y(),
+        __builtin_amdgcn_grid_size_z() / __builtin_amdgcn_workgroup_size_z()};
+}
+#define threadIdx d4r_thread_idx()
+#define blockIdx d4r_block_idx()
+#define gridDim d4r_grid_dim()
+__device__ __attribute__((convergent, always_inline)) inline void __syncthreads() {
+    // Match HIP __work_group_barrier: execution rendezvous alone does not
+    // publish LDS stores or prevent loads moving across the barrier.
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "workgroup");
+    __builtin_amdgcn_s_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "workgroup");
+}

kernels/common/wave32_exchange.h

@@ -0,0 +1,31 @@
+#pragma once
+#include <stdint.h>
+
+// Windows gfx11 portability workaround informed by realdody/d4r 1897b4a.
+// The reported permlanex16 failure duplicates the even K value into the odd
+// slot. Retain the native gfx11 layout and exchange through per-wave LDS.
+// Other platforms and gfx12 keep their original instruction sequence.
+__attribute__((device, always_inline)) static inline uint32_t d4r_wave32_other_half(uint32_t value)
+{
+#if defined(D4R_WINDOWS_GFX11_LANE_EXCHANGE) && defined(__GFX11__)
+    __attribute__((shared)) volatile uint32_t scratch[512];
+    const uint32_t tid = __builtin_amdgcn_workitem_id_x() + __builtin_amdgcn_workgroup_size_x() *
+        (__builtin_amdgcn_workitem_id_y() + __builtin_amdgcn_workgroup_size_y() * __builtin_amdgcn_workitem_id_z());
+    // All native K/M launch bounds are <=512. Do not silently corrupt LDS if
+    // this helper is later reused by a larger workgroup.
+    if (tid >= 512) __builtin_trap();
+    scratch[tid] = value;
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+    __builtin_amdgcn_wave_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+    const uint32_t result = scratch[tid ^ 16u];
+    // Protect scratch reuse by the next exchange in this wave. No block-wide
+    // rendezvous: different waves can reach GEMMs in a different order.
+    __builtin_amdgcn_fence(__ATOMIC_RELEASE, "wavefront");
+    __builtin_amdgcn_wave_barrier();
+    __builtin_amdgcn_fence(__ATOMIC_ACQUIRE, "wavefront");
+    return result;
+#else
+    return __builtin_amdgcn_permlanex16(value, value, 0x76543210u, 0xfedcba98u, false, false);
+#endif
+}

kernels/common/wmma_backend.h

@@ -0,0 +1,57 @@
+#pragma once
+#include <stdint.h>
+#include "wave32_exchange.h"
+
+// d4r's gfx11 interface: each lane supplies 16 FP16 K operands. The 8 FP32
+// outputs have logical row 2*i + lane/16 and logical column lane%16.
+// gfx12 consumes only 8 FP16 operands per lane and emits 8 consecutive rows
+// per lane group. Keep the old interface for K/M/texture call sites until their
+// surrounding fragments are ported and validated individually.
+using d4r_wmma_h16 = _Float16 __attribute__((ext_vector_type(16)));
+using d4r_wmma_h8 = _Float16 __attribute__((ext_vector_type(8)));
+using d4r_wmma_f8 = float __attribute__((ext_vector_type(8)));
+
+__attribute__((device, always_inline)) static inline float d4r_wmma_other_half(float value)
+{
+    const uint32_t bits = __builtin_bit_cast(uint32_t, value);
+    const uint32_t swapped = d4r_wave32_other_half(bits);
+    return __builtin_bit_cast(float, swapped);
+}
+
+__attribute__((device, always_inline)) static inline d4r_wmma_f8
+d4r_wmma_legacy_layout(d4r_wmma_h16 a, d4r_wmma_h16 b, d4r_wmma_f8 c)
+{
+#if defined(__gfx1200__) || defined(__gfx1201__)
+    const uint32_t group = __builtin_amdgcn_mbcnt_lo(~0u, 0u) >> 4;
+    d4r_wmma_h8 a12, b12;
+    d4r_wmma_f8 c12;
+    d4r_wmma_f8 remote_c;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        remote_c[i] = d4r_wmma_other_half(c[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        a12[i] = a[8u * group + i];
+        b12[i] = b[8u * group + i];
+        const uint32_t row = 8u * group + i;
+        const uint32_t old_index = row >> 1;
+        c12[i] = (row & 1u) == group ? c[old_index] : remote_c[old_index];
+    }
+    const d4r_wmma_f8 d12 = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a12, b12, c12);
+    d4r_wmma_f8 result, remote_d;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        remote_d[i] = d4r_wmma_other_half(d12[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i) {
+        const uint32_t row = 2u * i + group;
+        const uint32_t new_index = row & 7u;
+        result[i] = (row >> 3) == group ? d12[new_index] : remote_d[new_index];
+    }
+    return result;
+#elif defined(__GFX11__)
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, c);
+#else
+#error "d4r WMMA backend requires a validated gfx11 or gfx12 target"
+#endif
+}

kernels/common/wmma_layout.h

@@ -0,0 +1,346 @@
+// WMMA register layouts of RDNA3 (gfx11) and RDNA4 (gfx12) for the native kernels (wave32, 16x16x16 f16 -> f32).
+// Every layout fact the kernels depend on is here; the kernel sources are written against these helpers.
+//
+// Notation: lane l, half h = l >> 4, r = l & 15. A operand lane l = row r of A, B operand lane l = column r
+// of B, D lane l = column r of D.
+//   gfx11: an operand holds all 16 K values of its row/column (8 dwords, the same data in both halves);
+//          D VGPR i is row 2i + h.
+//   gfx12: an operand holds 8 of the 16 K values (4 dwords), half h the K values 8h .. 8h+7 in order;
+//          D VGPR i is row i + 8h, which is the gfx12 B-operand layout of the next GEMM (no exchange).
+// A and B only have to agree on which K each slot holds; "half h = K 8h .. 8h+7" makes a gfx12 operand the
+// lower or upper 16 bytes of the gfx11 one, so row images and weight images serve both layouts.
+//
+// D4R_WMMA_LAYOUT selects the layout (default: the target's). D4R_WMMA_LAYOUT=12 on a gfx11 target is the
+// layout shim: the gfx12 register layout with every WMMA done by the gfx11 instruction on operands and
+// accumulators converted around it, in the gfx11 K order. Its results are bit-identical to the gfx11 build,
+// so the gfx12 indexing can be validated on RDNA3 hardware.
+//
+// Only clang builtins are used (the texture tails compile with -nogpuinc -nogpulib).
+#pragma once
+#include <stdint.h>
+#include "wave32_exchange.h"
+
+#ifndef D4R_WMMA_LAYOUT
+#if defined(__GFX12__)
+#define D4R_WMMA_LAYOUT 12
+#else
+#define D4R_WMMA_LAYOUT 11
+#endif
+#endif
+#if D4R_WMMA_LAYOUT == 12 && !defined(__GFX12__)
+#define D4R_WMMA_SHIM 1
+#endif
+#if D4R_WMMA_LAYOUT == 11 && defined(__GFX12__)
+#error "the gfx11 WMMA layout does not exist on gfx12"
+#endif
+
+#define WM_FN __attribute__((device, always_inline)) static inline
+
+typedef _Float16 wm_f16;
+typedef _Float16 wm_hv2 __attribute__((ext_vector_type(2)));
+typedef _Float16 wm_h8 __attribute__((ext_vector_type(8)));
+typedef _Float16 wm_h16 __attribute__((ext_vector_type(16)));
+typedef float wm_f8v __attribute__((ext_vector_type(8)));
+typedef uint32_t wm_u8v __attribute__((ext_vector_type(8)));
+typedef uint32_t wm_u4v __attribute__((ext_vector_type(4)));
+typedef int wm_i2v __attribute__((ext_vector_type(2)));
+
+// ---------------------------------------------------------------- lanes
+WM_FN uint32_t wm_lane()
+{
+    return __builtin_amdgcn_mbcnt_lo(~0u, 0u);
+}
+WM_FN uint32_t wm_half()
+{
+    return wm_lane() >> 4;
+}
+// value of the lane with the same index in the other half of the wave
+WM_FN uint32_t wm_other_half(uint32_t v)
+{
+    return d4r_wave32_other_half(v);
+}
+WM_FN float wm_other_half_f(float v)
+{
+    return __builtin_bit_cast(float, wm_other_half(__builtin_bit_cast(uint32_t, v)));
+}
+WM_FN uint32_t wm_pack2(wm_f16 lo, wm_f16 hi)
+{
+    return __builtin_bit_cast(uint32_t, (wm_hv2){lo, hi});
+}
+WM_FN wm_f16 wm_lo16(uint32_t v)
+{
+    return __builtin_bit_cast(wm_hv2, v)[0];
+}
+WM_FN wm_f16 wm_hi16(uint32_t v)
+{
+    return __builtin_bit_cast(wm_hv2, v)[1];
+}
+
+// ---------------------------------------------------------------- accumulator rows
+// row (of the 16-row tile) that accumulator VGPR i holds in a lane of half h (prep code: logical lanes)
+WM_FN int wm_acc_row_h(int i, int h)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return i + 8 * h;
+#else
+    return 2 * i + h;
+#endif
+}
+// ... in this lane
+WM_FN int wm_acc_row(int i)
+{
+    return wm_acc_row_h(i, (int)wm_half());
+}
+// rows are the K dimension of the next GEMM: VGPRs 2j, 2j + 1 are adjacent K values in gfx12 (and 16 bytes
+// of this lane's 8 rows are contiguous), rows 2i + h are interleaved between the halves in gfx11
+#if D4R_WMMA_LAYOUT == 12
+#define WM_ACC_CONTIGUOUS 1
+#else
+#define WM_ACC_CONTIGUOUS 0
+#endif
+
+// ---------------------------------------------------------------- f16 operands
+#if D4R_WMMA_LAYOUT == 12
+typedef wm_u4v wm_op; // this half's 8 K values
+#define WM_OP_DWORDS 4
+#else
+typedef wm_u8v wm_op; // all 16 K values
+#define WM_OP_DWORDS 8
+#endif
+
+// operand from 16 contiguous halves (K 0..15) in memory (16-byte aligned); gfx12 reads only this half's 8
+WM_FN wm_op wm_op_load(const wm_f16* p)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return *(const wm_u4v*)(p + 8 * wm_half());
+#else
+    const wm_u4v a = *(const wm_u4v*)p, b = *(const wm_u4v*)(p + 8);
+    return (wm_u8v){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+#endif
+}
+
+// operand slot `idx` of an image of 32-byte slots (16 halves each, K in order): gfx12 reads its 16-byte half
+WM_FN wm_op wm_op_image(const wm_u8v* __restrict__ img, int idx)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return ((const wm_u4v*)img)[2 * idx + (int)wm_half()];
+#else
+    return img[idx];
+#endif
+}
+
+// element (K value) of an operand that holds accumulator row wm_acc_row(i) of the same lane
+WM_FN wm_f16 wm_op_acc_elem(const wm_op& v, int i)
+{
+#if D4R_WMMA_LAYOUT == 12
+    return (i & 1) ? wm_hi16(v[i >> 1]) : wm_lo16(v[i >> 1]);
+#else
+    const int c = 2 * i + (int)wm_half();
+    return (c & 1) ? wm_hi16(v[c >> 1]) : wm_lo16(v[c >> 1]);
+#endif
+}
+
+// the f16 accumulator values own[i] (row wm_acc_row(i) of this lane's column) as an operand of the next GEMM
+// (the rows become K; gfx11 fetches the other parity from lane l ^ 16)
+WM_FN wm_op wm_op_from_acc(const wm_f16 own[8])
+{
+#if D4R_WMMA_LAYOUT == 12
+    return (wm_u4v){wm_pack2(own[0], own[1]), wm_pack2(own[2], own[3]), wm_pack2(own[4], own[5]), wm_pack2(own[6], own[7])};
+#else
+    const uint32_t hf = wm_half();
+    uint32_t mine[4], theirs[4];
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+    {
+        mine[j] = wm_pack2(own[2 * j], own[2 * j + 1]);
+        theirs[j] = wm_other_half(mine[j]);
+    }
+#ifndef PWIN_NO_PERM
+    // v_perm_b32(theirs, mine): bytes 0-3 = mine, 4-7 = theirs. Even j: the low halves, odd j: the high
+    // halves, in the order (mine, theirs) for hf = 0 and (theirs, mine) for hf = 1.
+    const uint32_t sel_lo = hf ? 0x01000504u : 0x05040100u, sel_hi = hf ? 0x03020706u : 0x07060302u;
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 8; ++j)
+        r[j] = __builtin_amdgcn_perm(theirs[j >> 1], mine[j >> 1], (j & 1) ? sel_hi : sel_lo);
+    return r;
+#else
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 8; ++j)
+    {
+        const wm_f16 a = (j & 1) ? wm_hi16(mine[j >> 1]) : wm_lo16(mine[j >> 1]);
+        const wm_f16 b = (j & 1) ? wm_hi16(theirs[j >> 1]) : wm_lo16(theirs[j >> 1]);
+        r[j] = hf ? wm_pack2(b, a) : wm_pack2(a, b);
+    }
+    return r;
+#endif
+#endif
+}
+
+WM_FN wm_op wm_op_from_f8(wm_f8v d)
+{
+    wm_f16 h[8];
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        h[i] = (wm_f16)d[i];
+    return wm_op_from_acc(h);
+}
+
+// Stores an operand back as a row of 16 halves (K 0..15). In gfx11 both halves hold the whole row and the
+// lanes of half 0 store it; in gfx12 each half stores its 8 values. `count` (8 or 16) limits the row to its
+// first `count` halves; `ok` is the lane's own condition.
+WM_FN void wm_op_store(wm_f16* p, const wm_op& v, bool ok = true, int count = 16)
+{
+#if D4R_WMMA_LAYOUT == 12
+    if (ok && (wm_half() == 0 || count > 8))
+        *(wm_u4v*)(p + 8 * wm_half()) = v;
+#else
+    if (ok && wm_half() == 0)
+    {
+        *(wm_u4v*)p = (wm_u4v){v[0], v[1], v[2], v[3]};
+        if (count > 8)
+            *(wm_u4v*)(p + 8) = (wm_u4v){v[4], v[5], v[6], v[7]};
+    }
+#endif
+}
+
+// 8 halves of an f16 vector in accumulator row order: out[i] = v[wm_acc_row(i)], v at p (16 halves, 16-byte
+// aligned)
+WM_FN void wm_acc_vec_load(const wm_f16* p, wm_f16 out[8])
+{
+#if D4R_WMMA_LAYOUT == 12
+    const wm_u4v a = *(const wm_u4v*)(p + 8 * wm_half());
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        out[i] = (i & 1) ? wm_hi16(a[i >> 1]) : wm_lo16(a[i >> 1]);
+#else
+    const uint32_t hf = wm_half();
+    const wm_u4v a = *(const wm_u4v*)p, b = *(const wm_u4v*)(p + 8);
+    const wm_u8v v = (wm_u8v){a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]};
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        out[i] = hf ? wm_hi16(v[i]) : wm_lo16(v[i]);
+#endif
+}
+
+// ---------------------------------------------------------------- the WMMA itself
+#if D4R_WMMA_LAYOUT == 12 && defined(__GFX12__)
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(__builtin_bit_cast(wm_h8, a), __builtin_bit_cast(wm_h8, b), c);
+}
+#ifdef D4R_K_F16_WMMA
+// Rejected K experiment: direct packed F16 changes the arithmetic on gfx1201.
+// The public WMMA reproducer and enc1 replay are in tools/windows and docs.
+// Experimental gfx12 packed FP16 C/D. K's strict baseline enters each K16
+// step with an already FP16-rounded accumulator. Keep the existing logical
+// eight-row contract; only this instruction/register representation changes.
+WM_FN wm_f8v wm_mma_f16_step(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    const wm_h8 hc = __builtin_convertvector(c, wm_h8);
+    const wm_h8 hd = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(
+        __builtin_bit_cast(wm_h8, a), __builtin_bit_cast(wm_h8, b), hc);
+    return __builtin_convertvector(hd, wm_f8v);
+}
+#endif
+#else
+WM_FN wm_f8v wm_mma11(const wm_u8v& a, const wm_u8v& b, wm_f8v c)
+{
+    return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(__builtin_bit_cast(wm_h16, a), __builtin_bit_cast(wm_h16, b), c);
+}
+#if D4R_WMMA_LAYOUT == 12
+// layout shim: gfx12 operand halves -> one gfx11 operand in K order (half 0's 8 K, then half 1's)
+WM_FN wm_u8v wm_shim_operand(const wm_u4v& v)
+{
+    const bool hf = wm_half() != 0;
+    wm_u8v r;
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+    {
+        const uint32_t o = wm_other_half(v[j]);
+        r[j] = hf ? o : v[j];
+        r[4 + j] = hf ? v[j] : o;
+    }
+    return r;
+}
+// gfx12 accumulator (VGPR i = row i + 8h) <-> gfx11 accumulator (VGPR i = row 2i + h)
+WM_FN wm_f8v wm_shim_acc_to11(const wm_f8v& c)
+{
+    const bool hf = wm_half() != 0;
+    wm_f8v o, r;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        o[i] = wm_other_half_f(c[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+    {
+        // row 2i + h lives in half (2i + h) >> 3, VGPR (2i + h) & 7
+        const float h0 = i < 4 ? c[2 * i] : o[2 * i - 8];
+        const float h1 = i < 4 ? o[2 * i + 1] : c[2 * i - 7];
+        r[i] = hf ? h1 : h0;
+    }
+    return r;
+}
+WM_FN wm_f8v wm_shim_acc_to12(const wm_f8v& d)
+{
+    const bool hf = wm_half() != 0;
+    wm_f8v o, r;
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+        o[i] = wm_other_half_f(d[i]);
+#pragma unroll
+    for (int i = 0; i < 8; ++i)
+    {
+        // row i + 8h lives in half i & 1, VGPR (i >> 1) + 4h
+        const float h0 = (i & 1) ? o[i >> 1] : d[i >> 1];
+        const float h1 = (i & 1) ? d[(i >> 1) + 4] : o[(i >> 1) + 4];
+        r[i] = hf ? h1 : h0;
+    }
+    return r;
+}
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_shim_acc_to12(wm_mma11(wm_shim_operand(a), wm_shim_operand(b), wm_shim_acc_to11(c)));
+}
+// ZLUDA's K order: its gfx11 helpers put K pair p (K 2p, 2p + 1) in operand dword p, its gfx12 helpers put
+// in half h, dword j the pair (j & 1) * 4 + 2h + (j >> 1) (see zluda_ptx_impl.cpp)
+WM_FN wm_u8v wm_shim_operand_zluda(const wm_u4v& v)
+{
+    const bool hf = wm_half() != 0;
+    wm_u4v o;
+#pragma unroll
+    for (int j = 0; j < 4; ++j)
+        o[j] = wm_other_half(v[j]);
+    wm_u8v r;
+#pragma unroll
+    for (int p = 0; p < 8; ++p)
+    {
+        const int hp = (p & 3) >> 1, jp = 2 * (p & 1) + (p >> 2);
+        r[p] = (hp != 0) == hf ? v[jp] : o[jp];
+    }
+    return r;
+}
+WM_FN wm_f8v wm_mma_zluda(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_shim_acc_to12(wm_mma11(wm_shim_operand_zluda(a), wm_shim_operand_zluda(b), wm_shim_acc_to11(c)));
+}
+#else
+WM_FN wm_f8v wm_mma(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_mma11(a, b, c);
+}
+#endif
+#endif
+#if D4R_WMMA_LAYOUT == 12 && defined(__GFX12__)
+// operands whose K slots follow ZLUDA's gfx12 order: the same instruction (the order only matters to the shim)
+WM_FN wm_f8v wm_mma_zluda(const wm_op& a, const wm_op& b, wm_f8v c)
+{
+    return wm_mma(a, b, c);
+}
+#endif
+
+WM_FN wm_f8v wm_splat8(float v)
+{
+    return (wm_f8v){v, v, v, v, v, v, v, v};
+}