Validate Windows native M FP8 and reduce operand packing cost

tools/windows/diagnostic.h

@@ -128,6 +128,7 @@ struct Args {
     unsigned ngx_create_flags = 0;
     unsigned output_width = 512, output_height = 288;
     std::string fixture_dir;
+    std::string benchmark_module;
     std::string output_dir;
     std::string kernel_name = "enc1";
     unsigned iterations = 32;
@@ -184,6 +185,7 @@ struct Args {
                     throw std::runtime_error("preset must be 5 (E), 11 (K) or 13 (M)");
             }
             else if (key == "--fixture-dir") fixture_dir = value;
+            else if (key == "--benchmark-module") benchmark_module = value;
             else if (key == "--output-dir") output_dir = value;
             else if (key == "--kernel-name") kernel_name = value;
             else if (key == "--iterations") {

tools/windows/fp8_pack_probe.hip

@@ -0,0 +1,9 @@
+// Diagnostic only: all finite OCP e4m3 encodings, including signed zero.
+#include "../../kernels/m/swin_common.h"
+struct PackParams { const half_t* input; uint32_t* output; };
+extern "C" __global__ void fp8_pack_probe(PackParams p) {
+    const unsigned lane = blockIdx.x * 32 + threadIdx.x;
+    if (lane >= 32) return;
+    const auto packed = to_fp8(*(const sop*)(p.input + lane * 8));
+    p.output[lane * 2] = packed[0]; p.output[lane * 2 + 1] = packed[1];
+}

tools/windows/fp8_pack_validate.py

@@ -0,0 +1,46 @@
+"""Check gfx1201 hardware packing of every finite OCP e4m3 value and signed zero."""
+import argparse
+import hashlib
+import json
+import os
+import pathlib
+import struct
+import subprocess
+
+import numpy as np
+from m_replay_validate import me
+
+
+def main(args):
+    root = args.output_dir.resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    codes = np.arange(256, dtype=np.uint8)
+    codes[(codes & 127) == 127] = 0  # NaN inputs are outside this finite-value packing contract.
+    inputs = me.E4M3[codes].astype(np.float16).tobytes()
+    (root / 'alloc-0.bin').write_bytes(inputs)
+    (root / 'alloc-1.bin').write_bytes(bytes(256))
+    (root / 'args.bin').write_bytes(struct.pack('<QQ', 0x100000000, 0x200000000))
+    (root / 'manifest.txt').write_text('kernel fp8_pack_probe\nlaunch 1 1 1 32 1 1 0\nargs 16\n'
+        'alloc 0 0x100000000 512\npointer 0 0 0\nalloc 1 0x200000000 256\npointer 8 1 0\n')
+    with (root / 'gpu.stdout.log').open('wb') as stdout, (root / 'gpu.stderr.log').open('wb') as stderr:
+        result = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
+            '--module', str(args.module.resolve()), '--fixture-dir', str(root), '--output-dir', str(root / 'output'),
+            '--iterations', '1'], stdout=stdout, stderr=stderr, timeout=90,
+            env=dict(os.environ, D4R_DIAG_DIR=str(root)))
+    summary = dict(passed=False, architecture='gfx1201', finiteEncodings=254, signedZero=True,
+        moduleSha256=hashlib.sha256(args.module.read_bytes()).hexdigest(), exitCode=f'0x{result.returncode & 0xffffffff:08x}')
+    if result.returncode == 0:
+        actual = np.fromfile(root / 'output/alloc-1.bin', np.uint8)
+        summary['passed'] = actual.size == 256 and bool(np.array_equal(actual, codes))
+        if actual.size == 256:
+            summary['differingCodes'] = int(np.count_nonzero(actual != codes))
+    (root / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
+    print(('PASS' if summary['passed'] else 'FAIL') + ' FP8_HARDWARE_PACK finite_encodings=254 signed_zero=1 architecture=gfx1201', flush=True)
+    return 0 if summary['passed'] else 1
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ('probe', 'module', 'hip-root', 'output-dir'):
+        parser.add_argument('--' + name, type=pathlib.Path, required=True)
+    raise SystemExit(main(parser.parse_args()))

tools/windows/m_native_replay_compare.py

@@ -0,0 +1,109 @@
+"""Replay private M captures and require exact logical outputs against a validated baseline."""
+import argparse
+import hashlib
+import json
+import os
+import pathlib
+import subprocess
+import traceback
+
+import numpy as np
+from m_capture_compare import logical, me, LAYERS
+from m_replay_validate import metrics
+
+
+def sha(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def main(args):
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    summary = dict(passed=False, strict=True, architecture='gfx1201', launches=[],
+                   hipRoot=str(args.hip_root), modules={},
+                   baselineValidation=json.loads(args.baseline_validation.read_text(encoding='utf-8-sig')))
+    try:
+        validation = summary['baselineValidation']
+        if not (validation.get('passed') and validation.get('exact') and validation.get('captured_outputs')):
+            raise RuntimeError('An exact captured-output NumPy validation is required for the baseline')
+        audited = {row['capture'] for row in validation['launches']}
+        captures = []
+        for layer in ('enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1'):
+            paths = sorted(args.capture_dir.glob(f'replay-*-rrlite_{layer}_4x4'))
+            if not paths:
+                raise RuntimeError(f'Baseline has no {layer} captures')
+            captures.extend((layer, path) for path in paths)
+        for layer, capture in captures:
+            if capture.name not in audited:
+                raise RuntimeError(f'Capture was not included in exact baseline validation: {capture.name}')
+            module = args.module_dir / f'rrlite_{layer}_4x4.hsaco'
+            summary['modules'][module.name] = sha(module)
+            output = args.output_dir / capture.name
+            output.mkdir(exist_ok=True)
+            env = dict(os.environ, D4R_DIAG_DIR=str(output.resolve()))
+            with (output / 'gpu.stdout.log').open('wb') as stdout, (output / 'gpu.stderr.log').open('wb') as stderr:
+                process = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
+                    '--module', str(module.resolve()), '--fixture-dir', str(capture.resolve()),
+                    '--output-dir', str((output / 'gpu-output').resolve()), '--iterations', '1'],
+                    stdout=stdout, stderr=stderr, env=env, timeout=90)
+            row = dict(capture=capture.name, layer=layer, exitCode=f'0x{process.returncode & 0xffffffff:08x}', fields=[])
+            summary['launches'].append(row)
+            if process.returncode:
+                raise RuntimeError(f'GPU replay failed: {capture.name} ({row["exitCode"]})')
+            p = me.Params(str(capture))
+            channels, _, merged, _ = LAYERS[layer]
+            fields = [('output', p.out, p.W * p.H * channels)]
+            if merged:
+                fields.append(('merge', p.p48, (p.W // 2) * (p.H // 2) * merged))
+            for name, pointer, count in fields:
+                reference = logical(p, pointer, count, True, capture)
+                actual = logical(p, pointer, count, True, output)
+                if reference.size != count or actual.size != count:
+                    raise RuntimeError(f'Truncated logical {name}: {capture.name}')
+                different = int(np.count_nonzero(reference != actual))
+                x, y = me.E4M3[reference], me.E4M3[actual]
+                maximum, _, psnr, _ = metrics(x, y, f'M_NATIVE_COMPARE capture={capture.name} field={name}')
+                row['fields'].append(dict(name=name, elements=count, differingCodes=different,
+                    maxAbs=maximum, referenceSha256=hashlib.sha256(reference.tobytes()).hexdigest(),
+                    actualSha256=hashlib.sha256(actual.tobytes()).hexdigest()))
+                if different:
+                    raise RuntimeError(f'Exact baseline mismatch: {capture.name} {name}, codes={different}, PSNR={psnr}')
+            row['passed'] = True
+        if args.benchmark_control_dir:
+            summary['benchmarks'] = []
+            for layer in ('enc1', 'enc2', 'enc3_tube', 'dec2', 'dec1'):
+                capture = next(path for item, path in captures if item == layer)
+                output = args.output_dir / ('paired-' + layer)
+                output.mkdir(exist_ok=True)
+                control = args.benchmark_control_dir / f'rrlite_{layer}_4x4.hsaco'
+                candidate = args.module_dir / control.name
+                with (output / 'paired.stdout.log').open('wb') as stdout, (output / 'paired.stderr.log').open('wb') as stderr:
+                    process = subprocess.run([str(args.probe.resolve()), '--hip-root', str(args.hip_root.resolve()),
+                        '--module', str(control.resolve()), '--benchmark-module', str(candidate.resolve()),
+                        '--fixture-dir', str(capture.resolve()), '--output-dir', str((output / 'output').resolve()),
+                        '--iterations', '64'], stdout=stdout, stderr=stderr, timeout=90,
+                        env=dict(os.environ, D4R_DIAG_DIR=str(output.resolve())))
+                if process.returncode:
+                    raise RuntimeError(f'Paired benchmark failed: {layer}, exit={process.returncode:#x}')
+                from profile_report import report
+                measured = report(output)
+                if len(measured['replayPairs']) != 1 or measured['replayPairs'][0]['controlGpuMs']['samples'] != 64:
+                    raise RuntimeError(f'Incomplete paired measurements: {layer}')
+                summary['benchmarks'].extend(measured['replayPairs'])
+                print(f'PAIRED M_NATIVE_COMPARE layer={layer} samples=64 logs={output}', flush=True)
+        summary['passed'] = True
+        print(f'PASS M_NATIVE_COMPARE launches={len(captures)} exact=1 finite=1 architecture=gfx1201', flush=True)
+        return 0
+    except Exception as error:
+        summary['error'] = str(error)
+        traceback.print_exc()
+        return 1
+    finally:
+        (args.output_dir / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n', encoding='utf-8')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    for name in ('capture-dir', 'module-dir', 'probe', 'hip-root', 'baseline-validation', 'output-dir'):
+        parser.add_argument('--' + name, type=pathlib.Path, required=True)
+    parser.add_argument('--benchmark-control-dir', type=pathlib.Path)
+    raise SystemExit(main(parser.parse_args()))

tools/windows/native_replay_probe.cpp

@@ -14,8 +14,9 @@ struct Allocation {
     int id;
     size_t size;
     void* device = nullptr;
+    std::vector<uint8_t> initial;
     Allocation(d4r::diag::HipApi& api, int identifier, const std::vector<uint8_t>& bytes)
-        : hip(api), id(identifier), size(bytes.size()) {
+        : hip(api), id(identifier), size(bytes.size()), initial(bytes) {
         hip.check(hip.hipMalloc(&device, size), "hipMalloc(replay)");
         try { hip.check(hip.hipMemcpy(device, bytes.data(), size, hipMemcpyHostToDevice), "hipMemcpy(replay upload)"); }
         catch (...) { (void)hip.hipFree(device); device = nullptr; throw; }
@@ -90,14 +91,15 @@ int main(int argc, char** argv) {
             if (size != sizeof(coordinates)) throw std::runtime_error("Swin diagnostic coordinates ABI");
             hip.check(hip.hipMemcpy(address, coordinates, size, hipMemcpyHostToDevice), "Set Swin diagnostic block");
         }
-        auto scalar = [&](const char* name, unsigned fallback) {
+        auto module_scalar = [&](hipModule_t owner, const char* name, unsigned fallback) {
             hipDeviceptr_t address = nullptr; size_t size = 0;
-            if (hip.hipModuleGetGlobal(&address, &size, module, name) != hipSuccess) return fallback;
+            if (hip.hipModuleGetGlobal(&address, &size, owner, name) != hipSuccess) return fallback;
             if (size != 4) throw std::runtime_error(std::string("Invalid native metadata ") + name);
             unsigned value = 0;
             hip.check(hip.hipMemcpy(&value, address, 4, hipMemcpyDeviceToHost), name);
             return value ? value : fallback;
         };
+        auto scalar = [&](const char* name, unsigned fallback) { return module_scalar(module, name, fallback); };
         block[2] = scalar("d4r_block_z", block[2]);
         const unsigned persistentGrid = scalar("d4r_grid_x", 0);
         if (persistentGrid) { grid[0] = persistentGrid; grid[1] = grid[2] = 1; }
@@ -111,7 +113,65 @@ int main(int argc, char** argv) {
         }
         std::printf("REPLAY kernel=%s grid=%u,%u,%u block=%u,%u,%u allocations=%zu arguments=%zu\n",
             kernel.c_str(), grid[0], grid[1], grid[2], block[0], block[1], block[2], allocations.size(), parameters.size());
-        for (unsigned iteration = 0; iteration < args.iterations; ++iteration)
+        if (!args.benchmark_module.empty()) {
+            hipModule_t candidate = nullptr;
+            hip.check(hip.hipModuleLoad(&candidate, args.benchmark_module.c_str()), "Load paired benchmark candidate");
+            Cleanup candidate_cleanup{hip, candidate};
+            if (module_scalar(candidate, "d4r_block_z", block[2]) != block[2] ||
+                module_scalar(candidate, "d4r_grid_x", 0) != persistentGrid)
+                throw std::runtime_error("Paired modules require identical launch geometry");
+            hipFunction_t other = nullptr, other_prep = nullptr;
+            hip.check(hip.hipModuleGetFunction(&other, candidate, kernel.c_str()), "Find paired benchmark function");
+            if (hip.hipModuleGetFunction(&other_prep, candidate, (kernel + "_prep").c_str()) == hipSuccess) {
+                const auto blocks = module_scalar(candidate, "d4r_prep_blocks", 0);
+                if (!blocks) throw std::runtime_error("Paired candidate prep metadata is missing");
+                hip.check(hip.hipModuleLaunchKernel(other_prep, blocks, 1, 1, 128, 1, 1, 0, nullptr,
+                    launchArguments, nullptr), "Paired candidate prep");
+            }
+            auto create = hip.library.symbol<decltype(&::hipEventCreate)>("hipEventCreate");
+            auto destroy = hip.library.symbol<decltype(&::hipEventDestroy)>("hipEventDestroy");
+            auto record = hip.library.symbol<decltype(&::hipEventRecord)>("hipEventRecord");
+            auto synchronize = hip.library.symbol<decltype(&::hipEventSynchronize)>("hipEventSynchronize");
+            auto elapsed = hip.library.symbol<decltype(&::hipEventElapsedTime)>("hipEventElapsedTime");
+            struct Event {
+                decltype(destroy) release; hipEvent_t value = nullptr;
+                ~Event() { if (value) (void)release(value); }
+            } begin{destroy}, end{destroy};
+            hip.check(create(&begin.value), "Create paired start event");
+            hip.check(create(&end.value), "Create paired end event");
+            auto launch = [&](hipFunction_t fn) {
+                hip.check(hip.hipModuleLaunchKernel(fn, grid[0], grid[1], grid[2], block[0], block[1], block[2],
+                    shared, nullptr, launchArguments, nullptr), "Paired benchmark launch");
+            };
+            // Both modules and weight images stay resident on the same GPU.
+            // Warm continuous work first, then alternate AB/BA order. Restore
+            // captured inputs outside the event so neither module consumes
+            // modified temporal inputs from a preceding sample.
+            const auto verbose = hip.verbose; hip.verbose = false;
+            for (unsigned warm = 0; warm < 16; ++warm)
+                for (auto fn : {function, other}) for (unsigned batch = 0; batch < 16; ++batch) launch(fn);
+            hip.check(hip.hipDeviceSynchronize(), "Paired benchmark warmup");
+            for (unsigned pair = 0; pair < args.iterations; ++pair) for (unsigned order = 0; order < 2; ++order) {
+                const unsigned variant = order ^ (pair & 1u);
+                for (const auto& allocation : allocations)
+                    hip.check(hip.hipMemcpy(allocation->device, allocation->initial.data(), allocation->size,
+                        hipMemcpyHostToDevice), "Restore paired captured input");
+                hip.check(record(begin.value, nullptr), "Record paired start");
+                launch(variant ? other : function);
+                hip.check(record(end.value, nullptr), "Record paired end");
+                hip.check(synchronize(end.value), "Complete paired sample");
+                float ms = 0; hip.check(elapsed(&ms, begin.value, end.value), "Paired GPU elapsed time");
+                std::printf("D4R_REPLAY_PROFILE kernel=%s variant=%s pair=%u gpu_ms=%.6f warmup_launches=512 paired=1 input_restore_outside_event=1\n",
+                    kernel.c_str(), variant ? "candidate" : "control", pair, double(ms));
+            }
+            hip.verbose = verbose;
+            // Leave a single ordinary control replay for numerical checking.
+            for (const auto& allocation : allocations)
+                hip.check(hip.hipMemcpy(allocation->device, allocation->initial.data(), allocation->size,
+                    hipMemcpyHostToDevice), "Restore final reference input");
+            launch(function);
+        }
+        for (unsigned iteration = 0; args.benchmark_module.empty() && iteration < args.iterations; ++iteration)
             hip.check(hip.hipModuleLaunchKernel(function, grid[0], grid[1], grid[2], block[0], block[1], block[2], shared,
                 nullptr, launchArguments, nullptr), "hipModuleLaunchKernel(replay)");
         hip.check(hip.hipDeviceSynchronize(), "hipDeviceSynchronize(replay)");
@@ -132,7 +192,8 @@ int main(int argc, char** argv) {
             file.write(reinterpret_cast<const char*>(bytes.data()), std::streamsize(bytes.size()));
             if (!file) throw std::runtime_error("Cannot save replay allocation");
         }
-        std::printf("PASS NATIVE_REPLAY architecture=gfx1201 kernel=%s iterations=%u numerical_reference=pending\n", kernel.c_str(), args.iterations);
+        std::printf("PASS NATIVE_REPLAY architecture=gfx1201 kernel=%s iterations=%u benchmark_pairs=%u numerical_reference=pending\n",
+            kernel.c_str(), args.benchmark_module.empty() ? args.iterations : 1, args.benchmark_module.empty() ? 0 : args.iterations);
         return 0;
     } catch (const std::exception& error) {
         std::fprintf(stderr, "FAIL NATIVE_REPLAY %s\n", error.what()); loaded_modules(); return 4;

tools/windows/profile_report.py

@@ -42,8 +42,13 @@ def report(directory, metadata_directory=None):
     commands = collections.defaultdict(list)
     kernels = collections.defaultdict(list)
     hook_threads = collections.defaultdict(list)
+    replay = collections.defaultdict(lambda: collections.defaultdict(dict))
     for path in sorted(directory.glob('*.log')):
-        text = path.read_text(encoding='utf-8-sig', errors='replace')
+        data = path.read_bytes()
+        text = data.decode('utf-16' if data.startswith((b'\xff\xfe', b'\xfe\xff')) else 'utf-8-sig', errors='replace')
+        for name, variant, pair, value in re.findall(
+                r'D4R_REPLAY_PROFILE kernel=(\S+) variant=(control|candidate) pair=([0-9]+) gpu_ms=([0-9.]+)', text):
+            replay[name][int(pair)][variant] = float(value)
         for name, value in re.findall(r'D4R_STAGE name=(\S+) cpu_ms=([0-9.]+)', text):
             stages[name].append(float(value))
         for kind, fields in re.findall(r'(?m)^D4R_COMMAND_(RECORD|SUBMIT) ([^\r\n]+)', text):
@@ -64,6 +69,7 @@ def report(directory, metadata_directory=None):
     result = dict(cpuStages={name: stats(values) for name, values in stages.items()},
                   commandStages={name: stats(values) for name, values in commands.items()},
                   commandHookSamples=[],
+                  replayPairs=[],
                   kernels=[], notes=[
                       'CPU stage times include waits and host work; they are not isolated GPU timings.',
                       'Command record intervals are between NGX recording calls on one thread, not Present/FPS.',
@@ -72,6 +78,13 @@ def report(directory, metadata_directory=None):
                       'HIP-event profiling synchronizes every sampled launch and changes scheduling.',
                       'Occupancy is an API prediction, not a measured hardware counter.',
                       'Bandwidth and WMMA utilization require additional supported hardware tooling.'])
+    for name, pairs in replay.items():
+        complete = [row for row in pairs.values() if set(row) == {'control', 'candidate'}]
+        if complete:
+            result['replayPairs'].append(dict(kernel=name,
+                controlGpuMs=stats([row['control'] for row in complete]),
+                candidateGpuMs=stats([row['candidate'] for row in complete]),
+                candidateToControl=stats([row['candidate'] / row['control'] for row in complete if row['control'] > 0])))
     for thread, rows in hook_threads.items():
         sums = {name: sum(float(row[name]) for row in rows) for name in
                 ('period_ms', 'calls', 'samples', 'marked', 'access_sum_ms', 'capture_sum_ms', 'driver_sum_ms')}
@@ -105,7 +118,7 @@ def main():
     parser.add_argument('--metadata-directory', type=pathlib.Path, help='llvm-readobj --notes output for native code objects')
     args = parser.parse_args()
     result = report(args.directory, args.metadata_directory)
-    if not result['cpuStages'] and not result['kernels']:
+    if not any(result[key] for key in ('cpuStages', 'kernels', 'commandStages', 'commandHookSamples', 'replayPairs')):
         parser.error('no complete stage or HIP-event records found')
     if args.output:
         args.output.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
@@ -116,6 +129,9 @@ def main():
     for row in result['commandHookSamples']:
         print(f"HOOK thread={row['thread']} samples={int(row['samples'])} access={row['accessMeanUs']:.3f} us "
               f"capture={row['captureMeanUsPerSample']:.3f} us estimated_cpu={row['estimatedAccessCaptureMsPerSecond']:.3f} ms/s")
+    for row in result['replayPairs']:
+        print(f"PAIRED {row['kernel']}: n={row['controlGpuMs']['samples']} control_median={row['controlGpuMs']['median']:.6f} ms "
+              f"candidate_median={row['candidateGpuMs']['median']:.6f} ms ratio_median={row['candidateToControl']['median']:.6f}")
     for row in result['kernels']:
         times = row['gpuMs']
         print(f"GPU {row['kernel']} {row['backend']} {row['phase']}: n={times['samples']} mean={times['mean']:.6f} ms total={times['total']:.3f} ms")