"""Descriptor for `cuda_dldn_engine_swin_enc4_kernel` (Ray Reconstruction denoiser, encoder 4). Machine-readable form of the layer's 62-phase mma region, for the parameterised WMMA engine. `layer()` carries the launch geometry and argument map; `phases()` carries one entry per phase, in mma-statement order, with the GEMM shape, where A and C come from, how the 32-deep k splits into k16 WMMA steps, and what the epilogue does. Provenance of every number: * shapes, weight-image byte ranges, C-source byte offsets, mma and epilogue statement ranges: `python3 kernels/rr/rr_layer_spec.py enc4`, which derives them from `~/.cache/d4r-rr-corpus/0017-PREPASS_ENTRYPOINT_NAME.ptx`. * A/C semantics and epilogue contents: `kernels/rr/rrswin_enc4_epilogues.md`. * phases 1-11 of the arithmetic were additionally verified bit-exact against debug-oracle stages e4a, e4p1..e4p12 (see kernels/rr/swin_enc4_recipe.py); the rest is transcription of the epilogue document and is *not* arithmetic-verified. """ ENTRY = "cuda_dldn_engine_swin_enc4_kernel" MODULE = "0017-PREPASS_ENTRYPOINT_NAME.ptx" WEIGHT_IMAGE_BYTES = 533844 WEIGHT_IMAGE_SLOTS = 2522 def layer(): """Launch geometry and argument map, as measured from the corpus PTX and the launch dump. The block is one 8x8 token patch with four waves (tid.z bounds to 0..3); the four waves read different weight slices, so tid.z is part of every per-wave address. """ return { "entry": ENTRY, "module": MODULE, "grid": [11, 7, 1], "block": [32, 1, 4], "waves_per_block": 4, "token_grid": [80, 46], "tokens_per_block": [8, 8], "param_block_bytes": 144, "arguments": { "arg008": {"param_offset": 8, "kind": "input_plane", "block_bytes": 65945600, "arena_offset_in_block": 39567360, "note": "the plane arena / debug region; reached as param_0+80-72"}, "arg040": {"param_offset": 40, "kind": "weight_image", "block_bytes": 533856, "slots": 2522}, "arg088": {"param_offset": 88, "kind": "texture", "shape": [80, 46, 4]}, "arg112": {"param_offset": 112, "kind": "surface", "shape": [80, 46, 4]}, "arg120": {"param_offset": 120, "kind": "surface", "shape": [80, 46, 4]}, "arg128": {"param_offset": 128, "kind": "surface", "shape": [40, 23, 4]}, }, "indirect_params": { "param+0": "extents (ex, ey), ld.param.v2.u32", "param+48": "plane arena, reached as param_0+80-32 (s19817); not a dumped argument", "param+56": "small output buffer, reached as param_0+80-24 (s21037)", "param+80": "two u16 block-origin offsets, ld.param.v2.u16 (s1033)", }, "shared_arena_bytes": 16384, "unverified": { "grid_block": ("taken from the layer's own recipe comment; the launch dump records" " resources but not the launch dimensions, so 11x7x4 was not" " re-measured here"), }, } def phases(): """One entry per mma phase, in order.""" return [ { "phase": 1, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [320, 844], "c_source_bytes": [], "mma_stmt_range": [7554, 7577], "epilogue_statements": 7, "epilogue_stmt_range": [7578, 7584], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 0, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'zero', 'note': '%r11676, set to 0 at PTX line 11871 (s7547)', 'evidence': 'rrswin_enc4_epilogues.md 3/E1..E9; recipe A/C operands', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7578, 7584], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 2, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [5440, 5964], "c_source_bytes": [], "mma_stmt_range": [7585, 7608], "epilogue_statements": 7, "epilogue_stmt_range": [7609, 7615], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 0, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'zero', 'note': '%r11676, set to 0 at PTX line 11871 (s7547)', 'evidence': 'rrswin_enc4_epilogues.md 3/E1..E9; recipe A/C operands', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7609, 7615], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 3, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [1344, 1868], "c_source_bytes": [], "mma_stmt_range": [7616, 7639], "epilogue_statements": 7, "epilogue_stmt_range": [7640, 7646], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 1, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 1, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7640, 7646], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 4, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [6464, 6988], "c_source_bytes": [], "mma_stmt_range": [7647, 7670], "epilogue_statements": 7, "epilogue_stmt_range": [7671, 7677], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 1, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 2, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7671, 7677], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 5, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [2368, 2892], "c_source_bytes": [], "mma_stmt_range": [7678, 7701], "epilogue_statements": 7, "epilogue_stmt_range": [7702, 7708], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 2, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 3, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7702, 7708], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 6, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [7488, 8012], "c_source_bytes": [], "mma_stmt_range": [7709, 7732], "epilogue_statements": 7, "epilogue_stmt_range": [7733, 7739], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 2, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 4, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7733, 7739], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 7, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [3392, 3916], "c_source_bytes": [], "mma_stmt_range": [7740, 7763], "epilogue_statements": 7, "epilogue_stmt_range": [7764, 7770], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 3, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 5, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7764, 7770], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 8, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [8512, 9036], "c_source_bytes": [], "mma_stmt_range": [7771, 7794], "epilogue_statements": 7, "epilogue_stmt_range": [7795, 7801], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 3, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 6, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7795, 7801], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 9, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [4416, 4940], "c_source_bytes": [], "mma_stmt_range": [7802, 7825], "epilogue_statements": 7, "epilogue_stmt_range": [7826, 7832], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 4, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 7, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['next_b_load'], 'statements': 7, 'stmt_range': [7826, 7832], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 10, "gemm": { "m_tiles": 6, "n_tiles": 4, "k": 32, "k_chain_len": 1, "mma": 24, "chains": 24, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [9536, 10060], "c_source_bytes": [], "mma_stmt_range": [7833, 7856], "epilogue_statements": 171, "epilogue_stmt_range": [7857, 8027], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': 4, 'arena_byte_offsets_per_mtile': [0, 2560, 5120, 7680, 10240, 12800], 'note': 'shift group 0 is handed across the cut as 24 A registers; groups 1-4 are read from the shared arena, which the prologue staged per wave at base tid.z*2560', 'evidence': 'rrswin_enc4_epilogues.md 1.4; mma A operands at PTX lines 11883-12044 name %r11754 %r11774 %r11794 %r11814 %r11834 %r11854, i.e. offsets [0, 2560, 5120, 7680, 10240, 12800]', }, "c_operand": { 'kind': 'prev_chain_acc', 'from_phase': 8, 'note': "phase p's mma i reads phase p-2's D pair as its C, same (m-tile,row,n-tile)", 'evidence': 'rrswin_enc4_epilogues.md 3/E13..E19 chain note; verified by debug oracle stages e4p3..e4p10', }, "epilogue": { 'ops': ['e4m3_requantise', 'pack_next_a', 'load_next_b', 'load_next_c'], 'statements': 171, 'stmt_range': [7857, 8027], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 11, "gemm": { "m_tiles": 4, "n_tiles": 12, "k": 32, "k_chain_len": 1, "mma": 48, "chains": 48, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": None, "c_source_bytes": [10560, 10564, 10568, 10572, 11072, 11076, 11080, 11084, 11584, 11588, 11592, 11596, 12096, 12100, 12104, 12108, 12608, 12612, 12616, 12620, 13120, 13124, 13128, 13132, 13632, 13636, 13640, 13644, 14144, 14148, 14152, 14156, 14656, 14660, 14664, 14668, 15168, 15172, 15176, 15180, 15680, 15684, 15688, 15692, 16192, 16196, 16200, 16204, 16704, 16708, 16712, 16716, 17216, 17220, 17224, 17228, 17728, 17732, 17736, 17740, 18240, 18244, 18248, 18252, 18752, 18756, 18760, 18764, 19264, 19268, 19272, 19276, 19776, 19780, 19784, 19788, 20288, 20292, 20296, 20300, 20800, 20804, 20808, 20812, 21312, 21316, 21320, 21324, 21824, 21828, 21832, 21836, 22336, 22340, 22344, 22348], "mma_stmt_range": [8028, 8075], "epilogue_statements": 3844, "epilogue_stmt_range": [8076, 11919], }, "a_source": { 'kind': 'requantise_prev_acc', 'from_phase': 9, 'how': "E10 packs D9's f16x2 to e4m3 (cvt.rn.satfinite.e4m3x2.f16x2, 80 packs)", 'evidence': "rrswin_enc4_epilogues.md 3/E10; stage classification 'after phase 10 e4m3-pack:80@s7908'", }, "c_operand": { 'kind': 'bias_pair_f16', 'note': 'per (m-tile,row,n-tile) f16 pair from the wave slice at 10560 + 512*(a + 4*(nt>>1)) + 16*lane + 4*(2*(nt&1)+row)', 'evidence': 'rrswin_enc4_epilogues.md 3/E11; the 512-byte A-tile stride is confirmed at s8028-s8040', }, "epilogue": { 'ops': ['e4m3_requantise', 'softmax', 'transpose_pack'], 'statements': 3844, 'stmt_range': [8076, 11919], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 12, "gemm": { "m_tiles": 12, "n_tiles": 12, "k": 96, "k_chain_len": 3, "mma": 48, "chains": 16, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 3 mma chained along k', "weight_bytes": None, "c_source_bytes": [], "mma_stmt_range": [11920, 11967], "epilogue_statements": 274, "epilogue_stmt_range": [11968, 12241], }, "a_source": { 'kind': 'requantise_softmax', 'from': 'E11 score softmax output', 'how': "probabilities f16x2 -> e4m3, packed as phase 12's A fragments", 'evidence': 'rrswin_enc4_epilogues.md 3/E11, 3/E12(a)', }, "c_operand": { 'kind': 'zero', 'evidence': "rrswin_enc4_epilogues.md 3 'P12 ... C = 0'", }, "epilogue": { 'ops': ['e4m3_requantise', 'shared_stage', 'column_table_load', 'residual_add', 'next_b_load'], 'statements': 274, 'stmt_range': [11968, 12241], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 13, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [22848, 27468], "c_source_bytes": [], "mma_stmt_range": [12242, 12261], "epilogue_statements": 27, "epilogue_stmt_range": [12262, 12288], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 0', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12262, 12288], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 14, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [50496, 55116], "c_source_bytes": [], "mma_stmt_range": [12289, 12308], "epilogue_statements": 27, "epilogue_stmt_range": [12309, 12335], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 2048', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12309, 12335], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 15, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [78144, 82764], "c_source_bytes": [], "mma_stmt_range": [12336, 12355], "epilogue_statements": 27, "epilogue_stmt_range": [12356, 12382], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 4096', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12356, 12382], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 16, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [105792, 110412], "c_source_bytes": [], "mma_stmt_range": [12383, 12402], "epilogue_statements": 27, "epilogue_stmt_range": [12403, 12429], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 6144', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12403, 12429], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 17, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [133440, 138060], "c_source_bytes": [], "mma_stmt_range": [12430, 12449], "epilogue_statements": 27, "epilogue_stmt_range": [12450, 12476], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 8192', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12450, 12476], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 18, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [161088, 165708], "c_source_bytes": [], "mma_stmt_range": [12477, 12496], "epilogue_statements": 27, "epilogue_stmt_range": [12497, 12523], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 10240', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12497, 12523], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 19, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [188736, 193356], "c_source_bytes": [], "mma_stmt_range": [12524, 12543], "epilogue_statements": 27, "epilogue_stmt_range": [12544, 12570], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 12288', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'bias_plus_residual', 'note': 'f16 column table at W + 221504 + 16*i added to the long-lived residual', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(e,f)', }, "epilogue": { 'ops': ['shared_load_next_a', 'next_b_load'], 'statements': 27, 'stmt_range': [12544, 12570], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 20, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [216384, 221004], "c_source_bytes": [], "mma_stmt_range": [12571, 12590], "epilogue_statements": 1029, "epilogue_stmt_range": [12591, 13619], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E12 at arena + (tid.z<<11) + 16*lane + {0,512,1024,1536}', 'read_at': 'arena + (tid.z<<9) + 16*lane + 14336', 'evidence': 'rrswin_enc4_epilogues.md 3/E12(a,c), 3/E13..E19', 'unverified': "E12's store side uses tid.z<<11 while its read-back uses tid.z<<9; the doc records both verbatim and the mismatch is not resolved", }, "c_operand": { 'kind': 'prev_acc', 'from_phase': 19, 'evidence': "rrswin_enc4_epilogues.md 3 'P20 ... C = D19'", }, "epilogue": { 'ops': ['rms_norm', 'e4m3_requantise', 'gain_load', 'next_b_load', 'next_c_load'], 'statements': 1029, 'stmt_range': [12591, 13619], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 21, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [222144, 226764], "c_source_bytes": [227264, 227280, 227296, 227312], "mma_stmt_range": [13620, 13639], "epilogue_statements": 321, "epilogue_stmt_range": [13640, 13960], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [227264, 227280, 227296, 227312], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 321, 'stmt_range': [13640, 13960], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 22, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [227328, 231948], "c_source_bytes": [], "mma_stmt_range": [13961, 13980], "epilogue_statements": 92, "epilogue_stmt_range": [13981, 14072], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 21, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 20, 'table_bytes': 232464, 'evidence': 'rrswin_enc4_epilogues.md 3/E21(b)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'column_table_load', 'residual_add', 'pack_next_a', 'next_b_load'], 'statements': 92, 'stmt_range': [13981, 14072], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 23, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [232768, 237388], "c_source_bytes": [237888, 237904, 237920, 237936], "mma_stmt_range": [14073, 14092], "epilogue_statements": 172, "epilogue_stmt_range": [14093, 14264], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [237888, 237904, 237920, 237936], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [14093, 14264], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 24, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [237952, 242572], "c_source_bytes": [], "mma_stmt_range": [14265, 14284], "epilogue_statements": 92, "epilogue_stmt_range": [14285, 14376], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 23, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 22, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [14285, 14376], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 25, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [243072, 247692], "c_source_bytes": [248192, 248208, 248224, 248240], "mma_stmt_range": [14377, 14396], "epilogue_statements": 172, "epilogue_stmt_range": [14397, 14568], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [248192, 248208, 248224, 248240], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [14397, 14568], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 26, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [248256, 252876], "c_source_bytes": [], "mma_stmt_range": [14569, 14588], "epilogue_statements": 92, "epilogue_stmt_range": [14589, 14680], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 25, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 24, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [14589, 14680], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 27, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [253376, 257996], "c_source_bytes": [258496, 258512, 258528, 258544], "mma_stmt_range": [14681, 14700], "epilogue_statements": 172, "epilogue_stmt_range": [14701, 14872], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [258496, 258512, 258528, 258544], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [14701, 14872], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 28, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [258560, 263180], "c_source_bytes": [], "mma_stmt_range": [14873, 14892], "epilogue_statements": 92, "epilogue_stmt_range": [14893, 14984], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 27, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 26, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [14893, 14984], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 29, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [263680, 268300], "c_source_bytes": [268800, 268816, 268832, 268848], "mma_stmt_range": [14985, 15004], "epilogue_statements": 172, "epilogue_stmt_range": [15005, 15176], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [268800, 268816, 268832, 268848], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [15005, 15176], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 30, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [268864, 273484], "c_source_bytes": [], "mma_stmt_range": [15177, 15196], "epilogue_statements": 92, "epilogue_stmt_range": [15197, 15288], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 29, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 28, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [15197, 15288], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 31, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [273984, 278604], "c_source_bytes": [279104, 279120, 279136, 279152], "mma_stmt_range": [15289, 15308], "epilogue_statements": 172, "epilogue_stmt_range": [15309, 15480], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [279104, 279120, 279136, 279152], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [15309, 15480], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 32, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [279168, 283788], "c_source_bytes": [], "mma_stmt_range": [15481, 15500], "epilogue_statements": 92, "epilogue_stmt_range": [15501, 15592], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 31, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 30, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [15501, 15592], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 33, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [284288, 288908], "c_source_bytes": [289408, 289424, 289440, 289456], "mma_stmt_range": [15593, 15612], "epilogue_statements": 172, "epilogue_stmt_range": [15613, 15784], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [289408, 289424, 289440, 289456], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [15613, 15784], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 34, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [289472, 294092], "c_source_bytes": [], "mma_stmt_range": [15785, 15804], "epilogue_statements": 92, "epilogue_stmt_range": [15805, 15896], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 33, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 32, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [15805, 15896], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 35, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [294592, 299212], "c_source_bytes": [299712, 299728, 299744, 299760], "mma_stmt_range": [15897, 15916], "epilogue_statements": 172, "epilogue_stmt_range": [15917, 16088], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [299712, 299728, 299744, 299760], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [15917, 16088], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 36, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [299776, 304396], "c_source_bytes": [], "mma_stmt_range": [16089, 16108], "epilogue_statements": 92, "epilogue_stmt_range": [16109, 16200], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 35, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 34, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [16109, 16200], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 37, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [304896, 309516], "c_source_bytes": [310016, 310032, 310048, 310064], "mma_stmt_range": [16201, 16220], "epilogue_statements": 172, "epilogue_stmt_range": [16221, 16392], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [310016, 310032, 310048, 310064], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [16221, 16392], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 38, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [310080, 314700], "c_source_bytes": [], "mma_stmt_range": [16393, 16412], "epilogue_statements": 92, "epilogue_stmt_range": [16413, 16504], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 37, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 36, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [16413, 16504], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 39, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [315200, 319820], "c_source_bytes": [320320, 320336, 320352, 320368], "mma_stmt_range": [16505, 16524], "epilogue_statements": 172, "epilogue_stmt_range": [16525, 16696], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [320320, 320336, 320352, 320368], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [16525, 16696], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 40, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [320384, 325004], "c_source_bytes": [], "mma_stmt_range": [16697, 16716], "epilogue_statements": 92, "epilogue_stmt_range": [16717, 16808], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 39, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 38, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [16717, 16808], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 41, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [325504, 330124], "c_source_bytes": [330624, 330640, 330656, 330672], "mma_stmt_range": [16809, 16828], "epilogue_statements": 172, "epilogue_stmt_range": [16829, 17000], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [330624, 330640, 330656, 330672], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [16829, 17000], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 42, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [330688, 335308], "c_source_bytes": [], "mma_stmt_range": [17001, 17020], "epilogue_statements": 92, "epilogue_stmt_range": [17021, 17112], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 41, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 40, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [17021, 17112], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 43, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [335808, 340428], "c_source_bytes": [340928, 340944, 340960, 340976], "mma_stmt_range": [17113, 17132], "epilogue_statements": 172, "epilogue_stmt_range": [17133, 17304], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [340928, 340944, 340960, 340976], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [17133, 17304], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 44, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [340992, 345612], "c_source_bytes": [], "mma_stmt_range": [17305, 17324], "epilogue_statements": 92, "epilogue_stmt_range": [17325, 17416], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 43, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 42, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [17325, 17416], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 45, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [346112, 350732], "c_source_bytes": [351232, 351248, 351264, 351280], "mma_stmt_range": [17417, 17436], "epilogue_statements": 172, "epilogue_stmt_range": [17437, 17608], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [351232, 351248, 351264, 351280], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [17437, 17608], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 46, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [351296, 355916], "c_source_bytes": [], "mma_stmt_range": [17609, 17628], "epilogue_statements": 92, "epilogue_stmt_range": [17629, 17720], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 45, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 44, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [17629, 17720], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 47, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [356416, 361036], "c_source_bytes": [361536, 361552, 361568, 361584], "mma_stmt_range": [17721, 17740], "epilogue_statements": 172, "epilogue_stmt_range": [17741, 17912], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [361536, 361552, 361568, 361584], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [17741, 17912], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 48, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [361600, 366220], "c_source_bytes": [], "mma_stmt_range": [17913, 17932], "epilogue_statements": 92, "epilogue_stmt_range": [17933, 18024], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 47, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 46, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [17933, 18024], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 49, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [366720, 371340], "c_source_bytes": [371840, 371856, 371872, 371888], "mma_stmt_range": [18025, 18044], "epilogue_statements": 172, "epilogue_stmt_range": [18045, 18216], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [371840, 371856, 371872, 371888], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [18045, 18216], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 50, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [371904, 376524], "c_source_bytes": [], "mma_stmt_range": [18217, 18236], "epilogue_statements": 92, "epilogue_stmt_range": [18237, 18328], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 49, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 48, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [18237, 18328], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 51, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [377024, 381644], "c_source_bytes": [382144, 382160, 382176, 382192], "mma_stmt_range": [18329, 18348], "epilogue_statements": 172, "epilogue_stmt_range": [18349, 18520], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [382144, 382160, 382176, 382192], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [18349, 18520], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 52, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [382208, 386828], "c_source_bytes": [], "mma_stmt_range": [18521, 18540], "epilogue_statements": 92, "epilogue_stmt_range": [18541, 18632], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 51, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 50, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [18541, 18632], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 53, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [387328, 391948], "c_source_bytes": [392448, 392464, 392480, 392496], "mma_stmt_range": [18633, 18652], "epilogue_statements": 172, "epilogue_stmt_range": [18653, 18824], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [392448, 392464, 392480, 392496], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [18653, 18824], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 54, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [392512, 397132], "c_source_bytes": [], "mma_stmt_range": [18825, 18844], "epilogue_statements": 92, "epilogue_stmt_range": [18845, 18936], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 53, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 52, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [18845, 18936], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 55, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [397632, 402252], "c_source_bytes": [402752, 402768, 402784, 402800], "mma_stmt_range": [18937, 18956], "epilogue_statements": 172, "epilogue_stmt_range": [18957, 19128], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [402752, 402768, 402784, 402800], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [18957, 19128], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 56, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [402816, 407436], "c_source_bytes": [], "mma_stmt_range": [19129, 19148], "epilogue_statements": 92, "epilogue_stmt_range": [19149, 19240], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 55, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 54, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [19149, 19240], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 57, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [407936, 412556], "c_source_bytes": [413056, 413072, 413088, 413104], "mma_stmt_range": [19241, 19260], "epilogue_statements": 172, "epilogue_stmt_range": [19261, 19432], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [413056, 413072, 413088, 413104], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [19261, 19432], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 58, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [413120, 417740], "c_source_bytes": [], "mma_stmt_range": [19433, 19452], "epilogue_statements": 92, "epilogue_stmt_range": [19453, 19544], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 57, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 56, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'pack_next_a', 'next_b_load', 'next_c_load'], 'statements': 92, 'stmt_range': [19453, 19544], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 59, "gemm": { "m_tiles": 5, "n_tiles": 20, "k": 160, "k_chain_len": 5, "mma": 20, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [418240, 422860], "c_source_bytes": [423360, 423376, 423392, 423408], "mma_stmt_range": [19545, 19564], "epilogue_statements": 172, "epilogue_stmt_range": [19565, 19736], }, "a_source": { 'kind': 'normalised_tile', 'from_phase': 20, 'how': 'E20 row-RMS-normalised tile (rsqrt.approx.ftz.f32, gain at 221824) -> e4m3', 'evidence': "rrswin_enc4_epilogues.md 3/E20, and the shape summary 'P21/P23 ... A = normed tile'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [423360, 423376, 423392, 423408], 'note': 'one f16 pair per lane, read at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': 'rrswin_enc4_epilogues.md 3/E20, 3/E22..E58 (the bias words of the next odd phase)', }, "epilogue": { 'ops': ['clamped_cubic_activation', 'e4m3_requantise', 'next_b_load'], 'statements': 172, 'stmt_range': [19565, 19736], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 60, "gemm": { "m_tiles": 1, "n_tiles": 20, "k": 32, "k_chain_len": 1, "mma": 20, "chains": 20, "banded": False, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma', "weight_bytes": [423424, 428044], "c_source_bytes": [], "mma_stmt_range": [19737, 19756], "epilogue_statements": 1381, "epilogue_stmt_range": [19757, 21137], }, "a_source": { 'kind': 'requantise_activation', 'from_phase': 59, 'how': "clamped cubic activation of the previous phase's D, f16 -> e4m3", 'evidence': 'rrswin_enc4_epilogues.md 3/E21, 3/E23..E59 (pack of the activation outputs)', }, "c_operand": { 'kind': 'prev_acc_plus_table', 'from_phase': 58, 'evidence': 'rrswin_enc4_epilogues.md 3/E23..E59 op mix (identical to E21 minus the table loads)', 'unverified': 'E23..E59 have no table load, so whether phases 24..60 add a column table to their seed, and at which offset, is not pinned by the PTX', }, "epilogue": { 'ops': ['pack_next_a', 'plane_scatter', 'shared_stage', 'next_b_load', 'next_c_load'], 'statements': 1381, 'stmt_range': [19757, 21137], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 61, "gemm": { "m_tiles": 20, "n_tiles": 80, "k": 640, "k_chain_len": 20, "mma": 80, "chains": 4, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 20 mma chained along k', "weight_bytes": [428544, 448524], "c_source_bytes": [530944, 530960, 530976, 530992], "mma_stmt_range": [21138, 21217], "epilogue_statements": 117, "epilogue_stmt_range": [21218, 21334], }, "a_source": { 'kind': 'shared_staged_e4m3', 'shift_group': None, 'written_by': 'E60 stages 20 u32 per lane into the shared arena (s20051-s20215)', 'read_at': 'arena + idx2<<2 + {0,1024,...,7168} (80 ld.shared.u32, s20242-s21030)', 'evidence': "rrswin_enc4_epilogues.md 3/E60(c,d); the shape summary 'P61 A = 20 shared slots'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [530944, 530960, 530976, 530992], 'note': 'one f16 pair per lane at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': "rrswin_enc4_epilogues.md 3 'P61 C = bias@530944', 3 'P62 C = bias@533824'", }, "epilogue": { 'ops': ['pack_next_a', 'surface_scatter', 'next_b_load', 'next_c_load'], 'statements': 117, 'stmt_range': [21218, 21334], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, { "phase": 62, "gemm": { "m_tiles": 5, "n_tiles": 10, "k": 160, "k_chain_len": 5, "mma": 10, "chains": 2, "banded": True, "k_tiling": '32-deep per mma, so 2 k16 WMMA steps per mma; 5 mma chained along k', "weight_bytes": [531264, 533324], "c_source_bytes": [533824, 533840], "mma_stmt_range": [21335, 21344], "epilogue_statements": 323, "epilogue_stmt_range": [21345, 21667], }, "a_source": { 'kind': 'requantise_prev_acc', 'from_phase': 61, 'how': "E61 packs phase 61's D to e4m3", 'evidence': "rrswin_enc4_epilogues.md 3 'P62 A = pack(E61)'", }, "c_operand": { 'kind': 'bias_word', 'bytes': [533824, 533840], 'note': 'one f16 pair per lane at W + ((laneid<<2)&12) + offset + 16*i', 'evidence': "rrswin_enc4_epilogues.md 3 'P61 C = bias@530944', 3 'P62 C = bias@533824'", }, "epilogue": { 'ops': ['lane_merge', 'texture_read', 'surface_scatter'], 'statements': 323, 'stmt_range': [21345, 21667], 'evidence': 'rrswin_enc4_epilogues.md 3, per-epilogue sections', }, }, ]