wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
8eb7a94533eb8dd7b61770e23c28a11d6742a96f
d4r/kernels/rr/descriptor_enc4.py
T
woutermans 0fe3a28cb7 kernels/rr: staged-image Swin layer engine prototypes 
Per-layer staged-image engine for the eleven translated denoiser Swin
layers: descriptors (verified offsets per phase), cut recipes that splice an
isolated native phase into NVIDIA's PTX, wave templates, epilogue notes and
GPU probes. The isolated phase-8 splice produces a byte-identical final
4K frame but is slower than the translated body (14.90 vs 6.59 ms) - a
correctness experiment, not a shipping improvement; docs/rr-engine.md
records the design and the next direct-converter win (B operands built
from transposed accumulators).
2026-10-12 12:47:53 +02:00

2316 lines
103 KiB
Python
Raw Blame History

"""Descriptor for `cuda_dldn_engine_swin_enc4_kernel` (Ray Reconstruction denoiser, encoder 4).
Machine-readable form of the layer's 62-phase mma region, for the parameterised WMMA engine.
`layer()` carries the launch geometry and argument map; `phases()` carries one entry per
phase, in mma-statement order, with the GEMM shape, where A and C come from, how the 32-deep k
splits into k16 WMMA steps, and what the epilogue does.
Provenance of every number:
* shapes, weight-image byte ranges, C-source byte offsets, mma and epilogue statement
ranges: `python3 kernels/rr/rr_layer_spec.py enc4`, which derives them from
`~/.cache/d4r-rr-corpus/0017-PREPASS_ENTRYPOINT_NAME.ptx`.
* A/C semantics and epilogue contents: `kernels/rr/rrswin_enc4_epilogues.md`.
* phases 1-11 of the arithmetic were additionally verified bit-exact against debug-oracle
stages e4a, e4p1..e4p12 (see kernels/rr/swin_enc4_recipe.py); the rest is transcription of
the epilogue document and is *not* arithmetic-verified.
"""
ENTRY = "cuda_dldn_engine_swin_enc4_kernel"
MODULE = "0017-PREPASS_ENTRYPOINT_NAME.ptx"
WEIGHT_IMAGE_BYTES = 533844
WEIGHT_IMAGE_SLOTS = 2522
def layer():
"""Launch geometry and argument map, as measured from the corpus PTX and the launch dump.
The block is one 8x8 token patch with four waves (tid.z bounds to 0..3); the four waves read
different weight slices, so tid.z is part of every per-wave address.
"""
return {
"entry": ENTRY,
"module": MODULE,
"grid": [11, 7, 1],
"block": [32, 1, 4],
"waves_per_block": 4,
"token_grid": [80, 46],
"tokens_per_block": [8, 8],
"param_block_bytes": 144,
"arguments": {
"arg008": {"param_offset": 8, "kind": "input_plane",
"block_bytes": 65945600, "arena_offset_in_block": 39567360,
"note": "the plane arena / debug region; reached as param_0+80-72"},
"arg040": {"param_offset": 40, "kind": "weight_image",
"block_bytes": 533856, "slots": 2522},
"arg088": {"param_offset": 88, "kind": "texture", "shape": [80, 46, 4]},
"arg112": {"param_offset": 112, "kind": "surface", "shape": [80, 46, 4]},
"arg120": {"param_offset": 120, "kind": "surface", "shape": [80, 46, 4]},
"arg128": {"param_offset": 128, "kind": "surface", "shape": [40, 23, 4]},
},
"indirect_params": {
"param+0": "extents (ex, ey), ld.param.v2.u32",
"param+48": "plane arena, reached as param_0+80-32 (s19817); not a dumped argument",
"param+56": "small output buffer, reached as param_0+80-24 (s21037)",
"param+80": "two u16 block-origin offsets, ld.param.v2.u16 (s1033)",
},
"shared_arena_bytes": 16384,
"unverified": {
"grid_block": ("taken from the layer's own recipe comment; the launch dump records"
" resources but not the launch dimensions, so 11x7x4 was not"
" re-measured here"),
},
}
def phases():
"""One entry per mma phase, in order."""
return [
{
"phase": 1,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [320, 844],
"c_source_bytes": [],
"mma_stmt_range": [7554, 7577],
"epilogue_statements": 7,
"epilogue_stmt_range": [7578, 7584],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 0,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'zero',
'note': '%r11676, set to 0 at PTX line 11871 (s7547)',
'evidence': 'rrswin_enc4_epilogues.md 3/E1..E9; recipe A/C operands',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7578, 7584],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 2,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [5440, 5964],
"c_source_bytes": [],
"mma_stmt_range": [7585, 7608],
"epilogue_statements": 7,
"epilogue_stmt_range": [7609, 7615],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 0,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'zero',
'note': '%r11676, set to 0 at PTX line 11871 (s7547)',
'evidence': 'rrswin_enc4_epilogues.md 3/E1..E9; recipe A/C operands',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7609, 7615],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 3,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [1344, 1868],
"c_source_bytes": [],
"mma_stmt_range": [7616, 7639],
"epilogue_statements": 7,
"epilogue_stmt_range": [7640, 7646],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 1,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 1,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7640, 7646],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 4,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [6464, 6988],
"c_source_bytes": [],
"mma_stmt_range": [7647, 7670],
"epilogue_statements": 7,
"epilogue_stmt_range": [7671, 7677],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 1,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 2,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7671, 7677],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 5,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [2368, 2892],
"c_source_bytes": [],
"mma_stmt_range": [7678, 7701],
"epilogue_statements": 7,
"epilogue_stmt_range": [7702, 7708],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 2,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 3,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7702, 7708],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 6,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [7488, 8012],
"c_source_bytes": [],
"mma_stmt_range": [7709, 7732],
"epilogue_statements": 7,
"epilogue_stmt_range": [7733, 7739],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 2,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 4,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7733, 7739],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 7,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [3392, 3916],
"c_source_bytes": [],
"mma_stmt_range": [7740, 7763],
"epilogue_statements": 7,
"epilogue_stmt_range": [7764, 7770],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 3,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 5,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7764, 7770],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 8,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [8512, 9036],
"c_source_bytes": [],
"mma_stmt_range": [7771, 7794],
"epilogue_statements": 7,
"epilogue_stmt_range": [7795, 7801],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 3,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 6,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7795, 7801],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 9,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [4416, 4940],
"c_source_bytes": [],
"mma_stmt_range": [7802, 7825],
"epilogue_statements": 7,
"epilogue_stmt_range": [7826, 7832],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 4,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 7,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['next_b_load'],
'statements': 7,
'stmt_range': [7826, 7832],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 10,
"gemm": {
"m_tiles": 6,
"n_tiles": 4,
"k": 32,
"k_chain_len": 1,
"mma": 24,
"chains": 24,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [9536, 10060],
"c_source_bytes": [],
"mma_stmt_range": [7833, 7856],
"epilogue_statements": 171,
"epilogue_stmt_range": [7857, 8027],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': 4,
'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800],
'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560',
'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]',
},
"c_operand": {
'kind': 'prev_chain_acc',
'from_phase': 8,
'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)",
'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10',
},
"epilogue": {
'ops': ['e4m3_requantise', 'pack_next_a', 'load_next_b', 'load_next_c'],
'statements': 171,
'stmt_range': [7857, 8027],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 11,
"gemm": {
"m_tiles": 4,
"n_tiles": 12,
"k": 32,
"k_chain_len": 1,
"mma": 48,
"chains": 48,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": None,
"c_source_bytes": [10560, 10564, 10568, 10572, 11072, 11076, 11080, 11084, 11584, 11588, 11592, 11596, 12096, 12100, 12104, 12108, 12608, 12612, 12616, 12620, 13120, 13124, 13128, 13132, 13632, 13636, 13640, 13644, 14144, 14148, 14152, 14156, 14656, 14660, 14664, 14668, 15168, 15172, 15176, 15180, 15680, 15684, 15688, 15692, 16192, 16196, 16200, 16204, 16704, 16708, 16712, 16716, 17216, 17220, 17224, 17228, 17728, 17732, 17736, 17740, 18240, 18244, 18248, 18252, 18752, 18756, 18760, 18764, 19264, 19268, 19272, 19276, 19776, 19780, 19784, 19788, 20288, 20292, 20296, 20300, 20800, 20804, 20808, 20812, 21312, 21316, 21320, 21324, 21824, 21828, 21832, 21836, 22336, 22340, 22344, 22348],
"mma_stmt_range": [8028, 8075],
"epilogue_statements": 3844,
"epilogue_stmt_range": [8076, 11919],
},
"a_source": {
'kind': 'requantise_prev_acc',
'from_phase': 9,
'how': "E10 packs D9's f16x2 to e4m3 (cvt.rn.satfinite.e4m3x2.f16x2, 80 packs)",
'evidence': "rrswin_enc4_epilogues.md 3/E10; stage classification 'after phase 10 e4m3-pack:80@s7908'",
},
"c_operand": {
'kind': 'bias_pair_f16',
'note': 'per (m-tile,row,n-tile) f16 pair from the wave slice at 10560 + 512*(a + 4*(nt>>1)) + 16*lane + 4*(2*(nt&1)+row)',
'evidence': 'rrswin_enc4_epilogues.md 3/E11; the 512-byte A-tile stride is confirmed at s8028-s8040',
},
"epilogue": {
'ops': ['e4m3_requantise', 'softmax', 'transpose_pack'],
'statements': 3844,
'stmt_range': [8076, 11919],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 12,
"gemm": {
"m_tiles": 12,
"n_tiles": 12,
"k": 96,
"k_chain_len": 3,
"mma": 48,
"chains": 16,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 3 mma chained along k',
"weight_bytes": None,
"c_source_bytes": [],
"mma_stmt_range": [11920, 11967],
"epilogue_statements": 274,
"epilogue_stmt_range": [11968, 12241],
},
"a_source": {
'kind': 'requantise_softmax',
'from': 'E11 score softmax output',
'how': "probabilities f16x2 -> e4m3, packed as phase 12's A fragments",
'evidence': 'rrswin_enc4_epilogues.md 3/E11, 3/E12(a)',
},
"c_operand": {
'kind': 'zero',
'evidence': "rrswin_enc4_epilogues.md 3 'P12 ... C = 0'",
},
"epilogue": {
'ops': ['e4m3_requantise', 'shared_stage', 'column_table_load', 'residual_add', 'next_b_load'],
'statements': 274,
'stmt_range': [11968, 12241],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 13,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [22848, 27468],
"c_source_bytes": [],
"mma_stmt_range": [12242, 12261],
"epilogue_statements": 27,
"epilogue_stmt_range": [12262, 12288],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 0',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12262, 12288],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 14,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [50496, 55116],
"c_source_bytes": [],
"mma_stmt_range": [12289, 12308],
"epilogue_statements": 27,
"epilogue_stmt_range": [12309, 12335],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 2048',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12309, 12335],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 15,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [78144, 82764],
"c_source_bytes": [],
"mma_stmt_range": [12336, 12355],
"epilogue_statements": 27,
"epilogue_stmt_range": [12356, 12382],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 4096',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12356, 12382],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 16,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [105792, 110412],
"c_source_bytes": [],
"mma_stmt_range": [12383, 12402],
"epilogue_statements": 27,
"epilogue_stmt_range": [12403, 12429],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 6144',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12403, 12429],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 17,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [133440, 138060],
"c_source_bytes": [],
"mma_stmt_range": [12430, 12449],
"epilogue_statements": 27,
"epilogue_stmt_range": [12450, 12476],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 8192',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12450, 12476],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 18,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [161088, 165708],
"c_source_bytes": [],
"mma_stmt_range": [12477, 12496],
"epilogue_statements": 27,
"epilogue_stmt_range": [12497, 12523],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 10240',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12497, 12523],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 19,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [188736, 193356],
"c_source_bytes": [],
"mma_stmt_range": [12524, 12543],
"epilogue_statements": 27,
"epilogue_stmt_range": [12544, 12570],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 12288',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'bias_plus_residual',
'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)',
},
"epilogue": {
'ops': ['shared_load_next_a', 'next_b_load'],
'statements': 27,
'stmt_range': [12544, 12570],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 20,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [216384, 221004],
"c_source_bytes": [],
"mma_stmt_range": [12571, 12590],
"epilogue_statements": 1029,
"epilogue_stmt_range": [12591, 13619],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}',
'read_at': 'arena + (tid.z<<9) + 16*lane + 14336',
'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19',
'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved",
},
"c_operand": {
'kind': 'prev_acc',
'from_phase': 19,
'evidence': "rrswin_enc4_epilogues.md 3 'P20 ... C = D19'",
},
"epilogue": {
'ops': ['rms_norm', 'e4m3_requantise', 'gain_load', 'next_b_load', 'next_c_load'],
'statements': 1029,
'stmt_range': [12591, 13619],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 21,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [222144, 226764],
"c_source_bytes": [227264, 227280, 227296, 227312],
"mma_stmt_range": [13620, 13639],
"epilogue_statements": 321,
"epilogue_stmt_range": [13640, 13960],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [227264, 227280, 227296, 227312],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 321,
'stmt_range': [13640, 13960],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 22,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [227328, 231948],
"c_source_bytes": [],
"mma_stmt_range": [13961, 13980],
"epilogue_statements": 92,
"epilogue_stmt_range": [13981, 14072],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 21,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 20,
'table_bytes': 232464,
'evidence': 'rrswin_enc4_epilogues.md 3/E21(b)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'column_table_load', 'residual_add', 'pack_next_a', 'next_b_load'],
'statements': 92,
'stmt_range': [13981, 14072],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 23,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [232768, 237388],
"c_source_bytes": [237888, 237904, 237920, 237936],
"mma_stmt_range": [14073, 14092],
"epilogue_statements": 172,
"epilogue_stmt_range": [14093, 14264],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [237888, 237904, 237920, 237936],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [14093, 14264],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 24,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [237952, 242572],
"c_source_bytes": [],
"mma_stmt_range": [14265, 14284],
"epilogue_statements": 92,
"epilogue_stmt_range": [14285, 14376],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 23,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 22,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [14285, 14376],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 25,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [243072, 247692],
"c_source_bytes": [248192, 248208, 248224, 248240],
"mma_stmt_range": [14377, 14396],
"epilogue_statements": 172,
"epilogue_stmt_range": [14397, 14568],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [248192, 248208, 248224, 248240],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [14397, 14568],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 26,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [248256, 252876],
"c_source_bytes": [],
"mma_stmt_range": [14569, 14588],
"epilogue_statements": 92,
"epilogue_stmt_range": [14589, 14680],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 25,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 24,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [14589, 14680],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 27,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [253376, 257996],
"c_source_bytes": [258496, 258512, 258528, 258544],
"mma_stmt_range": [14681, 14700],
"epilogue_statements": 172,
"epilogue_stmt_range": [14701, 14872],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [258496, 258512, 258528, 258544],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [14701, 14872],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 28,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [258560, 263180],
"c_source_bytes": [],
"mma_stmt_range": [14873, 14892],
"epilogue_statements": 92,
"epilogue_stmt_range": [14893, 14984],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 27,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 26,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [14893, 14984],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 29,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [263680, 268300],
"c_source_bytes": [268800, 268816, 268832, 268848],
"mma_stmt_range": [14985, 15004],
"epilogue_statements": 172,
"epilogue_stmt_range": [15005, 15176],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [268800, 268816, 268832, 268848],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [15005, 15176],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 30,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [268864, 273484],
"c_source_bytes": [],
"mma_stmt_range": [15177, 15196],
"epilogue_statements": 92,
"epilogue_stmt_range": [15197, 15288],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 29,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 28,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [15197, 15288],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 31,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [273984, 278604],
"c_source_bytes": [279104, 279120, 279136, 279152],
"mma_stmt_range": [15289, 15308],
"epilogue_statements": 172,
"epilogue_stmt_range": [15309, 15480],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [279104, 279120, 279136, 279152],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [15309, 15480],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 32,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [279168, 283788],
"c_source_bytes": [],
"mma_stmt_range": [15481, 15500],
"epilogue_statements": 92,
"epilogue_stmt_range": [15501, 15592],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 31,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 30,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [15501, 15592],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 33,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [284288, 288908],
"c_source_bytes": [289408, 289424, 289440, 289456],
"mma_stmt_range": [15593, 15612],
"epilogue_statements": 172,
"epilogue_stmt_range": [15613, 15784],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [289408, 289424, 289440, 289456],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [15613, 15784],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 34,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [289472, 294092],
"c_source_bytes": [],
"mma_stmt_range": [15785, 15804],
"epilogue_statements": 92,
"epilogue_stmt_range": [15805, 15896],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 33,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 32,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [15805, 15896],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 35,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [294592, 299212],
"c_source_bytes": [299712, 299728, 299744, 299760],
"mma_stmt_range": [15897, 15916],
"epilogue_statements": 172,
"epilogue_stmt_range": [15917, 16088],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [299712, 299728, 299744, 299760],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [15917, 16088],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 36,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [299776, 304396],
"c_source_bytes": [],
"mma_stmt_range": [16089, 16108],
"epilogue_statements": 92,
"epilogue_stmt_range": [16109, 16200],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 35,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 34,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [16109, 16200],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 37,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [304896, 309516],
"c_source_bytes": [310016, 310032, 310048, 310064],
"mma_stmt_range": [16201, 16220],
"epilogue_statements": 172,
"epilogue_stmt_range": [16221, 16392],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [310016, 310032, 310048, 310064],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [16221, 16392],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 38,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [310080, 314700],
"c_source_bytes": [],
"mma_stmt_range": [16393, 16412],
"epilogue_statements": 92,
"epilogue_stmt_range": [16413, 16504],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 37,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 36,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [16413, 16504],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 39,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [315200, 319820],
"c_source_bytes": [320320, 320336, 320352, 320368],
"mma_stmt_range": [16505, 16524],
"epilogue_statements": 172,
"epilogue_stmt_range": [16525, 16696],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [320320, 320336, 320352, 320368],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [16525, 16696],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 40,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [320384, 325004],
"c_source_bytes": [],
"mma_stmt_range": [16697, 16716],
"epilogue_statements": 92,
"epilogue_stmt_range": [16717, 16808],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 39,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 38,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [16717, 16808],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 41,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [325504, 330124],
"c_source_bytes": [330624, 330640, 330656, 330672],
"mma_stmt_range": [16809, 16828],
"epilogue_statements": 172,
"epilogue_stmt_range": [16829, 17000],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [330624, 330640, 330656, 330672],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [16829, 17000],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 42,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [330688, 335308],
"c_source_bytes": [],
"mma_stmt_range": [17001, 17020],
"epilogue_statements": 92,
"epilogue_stmt_range": [17021, 17112],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 41,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 40,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [17021, 17112],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 43,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [335808, 340428],
"c_source_bytes": [340928, 340944, 340960, 340976],
"mma_stmt_range": [17113, 17132],
"epilogue_statements": 172,
"epilogue_stmt_range": [17133, 17304],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [340928, 340944, 340960, 340976],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [17133, 17304],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 44,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [340992, 345612],
"c_source_bytes": [],
"mma_stmt_range": [17305, 17324],
"epilogue_statements": 92,
"epilogue_stmt_range": [17325, 17416],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 43,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 42,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [17325, 17416],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 45,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [346112, 350732],
"c_source_bytes": [351232, 351248, 351264, 351280],
"mma_stmt_range": [17417, 17436],
"epilogue_statements": 172,
"epilogue_stmt_range": [17437, 17608],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [351232, 351248, 351264, 351280],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [17437, 17608],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 46,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [351296, 355916],
"c_source_bytes": [],
"mma_stmt_range": [17609, 17628],
"epilogue_statements": 92,
"epilogue_stmt_range": [17629, 17720],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 45,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 44,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [17629, 17720],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 47,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [356416, 361036],
"c_source_bytes": [361536, 361552, 361568, 361584],
"mma_stmt_range": [17721, 17740],
"epilogue_statements": 172,
"epilogue_stmt_range": [17741, 17912],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [361536, 361552, 361568, 361584],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [17741, 17912],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 48,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [361600, 366220],
"c_source_bytes": [],
"mma_stmt_range": [17913, 17932],
"epilogue_statements": 92,
"epilogue_stmt_range": [17933, 18024],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 47,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 46,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [17933, 18024],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 49,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [366720, 371340],
"c_source_bytes": [371840, 371856, 371872, 371888],
"mma_stmt_range": [18025, 18044],
"epilogue_statements": 172,
"epilogue_stmt_range": [18045, 18216],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [371840, 371856, 371872, 371888],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [18045, 18216],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 50,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [371904, 376524],
"c_source_bytes": [],
"mma_stmt_range": [18217, 18236],
"epilogue_statements": 92,
"epilogue_stmt_range": [18237, 18328],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 49,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 48,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [18237, 18328],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 51,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [377024, 381644],
"c_source_bytes": [382144, 382160, 382176, 382192],
"mma_stmt_range": [18329, 18348],
"epilogue_statements": 172,
"epilogue_stmt_range": [18349, 18520],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [382144, 382160, 382176, 382192],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [18349, 18520],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 52,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [382208, 386828],
"c_source_bytes": [],
"mma_stmt_range": [18521, 18540],
"epilogue_statements": 92,
"epilogue_stmt_range": [18541, 18632],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 51,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 50,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [18541, 18632],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 53,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [387328, 391948],
"c_source_bytes": [392448, 392464, 392480, 392496],
"mma_stmt_range": [18633, 18652],
"epilogue_statements": 172,
"epilogue_stmt_range": [18653, 18824],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [392448, 392464, 392480, 392496],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [18653, 18824],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 54,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [392512, 397132],
"c_source_bytes": [],
"mma_stmt_range": [18825, 18844],
"epilogue_statements": 92,
"epilogue_stmt_range": [18845, 18936],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 53,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 52,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [18845, 18936],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 55,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [397632, 402252],
"c_source_bytes": [402752, 402768, 402784, 402800],
"mma_stmt_range": [18937, 18956],
"epilogue_statements": 172,
"epilogue_stmt_range": [18957, 19128],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [402752, 402768, 402784, 402800],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [18957, 19128],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 56,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [402816, 407436],
"c_source_bytes": [],
"mma_stmt_range": [19129, 19148],
"epilogue_statements": 92,
"epilogue_stmt_range": [19149, 19240],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 55,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 54,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [19149, 19240],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 57,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [407936, 412556],
"c_source_bytes": [413056, 413072, 413088, 413104],
"mma_stmt_range": [19241, 19260],
"epilogue_statements": 172,
"epilogue_stmt_range": [19261, 19432],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [413056, 413072, 413088, 413104],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [19261, 19432],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 58,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [413120, 417740],
"c_source_bytes": [],
"mma_stmt_range": [19433, 19452],
"epilogue_statements": 92,
"epilogue_stmt_range": [19453, 19544],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 57,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 56,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'],
'statements': 92,
'stmt_range': [19453, 19544],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 59,
"gemm": {
"m_tiles": 5,
"n_tiles": 20,
"k": 160,
"k_chain_len": 5,
"mma": 20,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [418240, 422860],
"c_source_bytes": [423360, 423376, 423392, 423408],
"mma_stmt_range": [19545, 19564],
"epilogue_statements": 172,
"epilogue_stmt_range": [19565, 19736],
},
"a_source": {
'kind': 'normalised_tile',
'from_phase': 20,
'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3',
'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [423360, 423376, 423392, 423408],
'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)',
},
"epilogue": {
'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'],
'statements': 172,
'stmt_range': [19565, 19736],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 60,
"gemm": {
"m_tiles": 1,
"n_tiles": 20,
"k": 32,
"k_chain_len": 1,
"mma": 20,
"chains": 20,
"banded": False,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma',
"weight_bytes": [423424, 428044],
"c_source_bytes": [],
"mma_stmt_range": [19737, 19756],
"epilogue_statements": 1381,
"epilogue_stmt_range": [19757, 21137],
},
"a_source": {
'kind': 'requantise_activation',
'from_phase': 59,
'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3",
'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)',
},
"c_operand": {
'kind': 'prev_acc_plus_table',
'from_phase': 58,
'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)',
'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX',
},
"epilogue": {
'ops': ['pack_next_a', 'plane_scatter', 'shared_stage', 'next_b_load', 'next_c_load'],
'statements': 1381,
'stmt_range': [19757, 21137],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 61,
"gemm": {
"m_tiles": 20,
"n_tiles": 80,
"k": 640,
"k_chain_len": 20,
"mma": 80,
"chains": 4,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 20 mma chained along k',
"weight_bytes": [428544, 448524],
"c_source_bytes": [530944, 530960, 530976, 530992],
"mma_stmt_range": [21138, 21217],
"epilogue_statements": 117,
"epilogue_stmt_range": [21218, 21334],
},
"a_source": {
'kind': 'shared_staged_e4m3',
'shift_group': None,
'written_by': 'E60 stages 20 u32 per lane into the shared arena (s20051-s20215)',
'read_at': 'arena + idx2<<2 + {0,1024,...,7168} (80 ld.shared.u32, s20242-s21030)',
'evidence': "rrswin_enc4_epilogues.md 3/E60(c,d); the shape summary 'P61 A = 20 shared slots'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [530944, 530960, 530976, 530992],
'note': 'one f16 pair per lane at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': "rrswin_enc4_epilogues.md 3 'P61 C = bias@530944', 3 'P62 C = bias@533824'",
},
"epilogue": {
'ops': ['pack_next_a', 'surface_scatter', 'next_b_load', 'next_c_load'],
'statements': 117,
'stmt_range': [21218, 21334],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
{
"phase": 62,
"gemm": {
"m_tiles": 5,
"n_tiles": 10,
"k": 160,
"k_chain_len": 5,
"mma": 10,
"chains": 2,
"banded": True,
"k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k',
"weight_bytes": [531264, 533324],
"c_source_bytes": [533824, 533840],
"mma_stmt_range": [21335, 21344],
"epilogue_statements": 323,
"epilogue_stmt_range": [21345, 21667],
},
"a_source": {
'kind': 'requantise_prev_acc',
'from_phase': 61,
'how': "E61 packs phase 61's D to e4m3",
'evidence': "rrswin_enc4_epilogues.md 3 'P62 A = pack(E61)'",
},
"c_operand": {
'kind': 'bias_word',
'bytes': [533824, 533840],
'note': 'one f16 pair per lane at W + ((laneid<<2)&12) + offset + 16*i',
'evidence': "rrswin_enc4_epilogues.md 3 'P61 C = bias@530944', 3 'P62 C = bias@533824'",
},
"epilogue": {
'ops': ['lane_merge', 'texture_read', 'surface_scatter'],
'statements': 323,
'stmt_range': [21345, 21667],
'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections',
},
},
]
Reference in new issue View Git Blame Copy Permalink