diff --git a/docs/native-kernels.md b/docs/native-kernels.md index 77d1eab..3dea3f4 100644 --- a/docs/native-kernels.md +++ b/docs/native-kernels.md @@ -178,8 +178,10 @@ The release includes both fast and accuracy network sets. Full releases also inc ``` It runs the prep and main kernels, saves every buffer after the first launch, and times 30 launches. The replay tool refuses captures with texture or surface objects; use the full harness for those kernels. -3. **Compare against a reference.** `kernels/tools/pwin_model.py` (K) and `swin_model.py` (M) are numpy models of the layers, written stage by stage against `ptxsim.py`, a small vectorised PTX interpreter that runs NVIDIA's kernels on the CPU. A bit-exact variant should match the model within f16 rounding; after any change, compare the new build's buffers with the previous build's. -4. **Check the whole pipeline.** Replay captured frames through the harness with both kernel sets and compare the outputs with `kernels/tools/psnr.py`. `D4R_CUDA_KERNEL_PROFILE=1` makes the bridge time every kernel; `kernels/tools/kprof.py` summarises the log per frame. +3. **A per-layer oracle for kernels with texture and surface objects.** The replay capture above cannot describe those kernels (the replayers refuse them), but the bridge's other instrumentation can: with `D4R_CUDA_LAUNCH_STATS=1` and `D4R_CUDA_LAUNCH_DUMP_DIR=` every `cuLaunchKernel` is followed by a synchronize and a raw dump of every allocation, surface object and texture object named in its packed argument buffer, one file per launch and argument (`launch-NNN--argNNN-{buffer,surface,texture}--fmtN.bin`). Run the harness once with the translated kernel and once with the native one and compare that kernel's files byte for byte: the surface files are the layer's own output, and a mismatch localises to that layer rather than to the frame. `cuda_dldn_engine_swin_enc0_kernel` dumps its weight image at arg040, its activation buffer at arg048 and four surfaces (1280x736x4 and 640x368x4 RGBA16F) as its outputs. The dump is large - 3 frames cost 3.7 GB - so keep the frame count small. + +4. **Compare against a reference.** `kernels/tools/pwin_model.py` (K) and `swin_model.py` (M) are numpy models of the layers, written stage by stage against `ptxsim.py`, a small vectorised PTX interpreter that runs NVIDIA's kernels on the CPU. A bit-exact variant should match the model within f16 rounding; after any change, compare the new build's buffers with the previous build's. +5. **Check the whole pipeline.** Replay captured frames through the harness with both kernel sets and compare the outputs with `kernels/tools/psnr.py`. `D4R_CUDA_KERNEL_PROFILE=1` makes the bridge time every kernel; `kernels/tools/kprof.py` summarises the log per frame. ## Per-kernel cost diff --git a/kernels/tools/compare_launch_dump.py b/kernels/tools/compare_launch_dump.py new file mode 100755 index 0000000..5c33c88 --- /dev/null +++ b/kernels/tools/compare_launch_dump.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Compare two D4R_CUDA_LAUNCH_DUMP_DIR trees byte for byte. + +usage: compare_launch_dump.py A_DIR B_DIR [--kernel SUBSTR] [--quiet] + +The bridge writes one file per launch and per argument named +`launch-NNN--argNNN---fmtN.bin` (wine_nvcuda_bridge.c +summarize_resource). The launch number is a global sequence, so it is not a +stable key: a native kernel that runs its own prep shifts every later number. +Files are therefore keyed by (kernel, argument offset, kind, shape, format) and +by how many times that key has occurred so far, which survives a shift as long +as each kernel is launched the same number of times. + +Every dump is written after the launch returns, so a surface file is that +layer's own output and a read-only argument (a weight image, a texture) is its +input. Any difference in an output is the kernel's; identical read-only +arguments are the check that the two runs are aligned at all. +""" +import argparse +import os +import re +import sys + +NAME = re.compile(r'^launch-(\d+)-(.+)-arg(\d+)-(buffer|surface|texture)-(.+)-fmt(-?\d+)\.bin$') +# A buffer's shape string embeds the device pointer it was found at, which differs +# between runs. Keep the offset within the allocation, drop the base address. +BASE = re.compile(r'base0x[0-9a-f]+') + + +def index(directory): + """(kernel, arg, kind, shape, fmt, occurrence) -> (sequence, path).""" + entries = {} + counts = {} + for name in sorted(os.listdir(directory)): + match = NAME.match(name) + if match is None: + continue + sequence, kernel, argument, kind, shape, fmt = match.groups() + key = (kernel, int(argument), kind, BASE.sub('base', shape), int(fmt)) + occurrence = counts.get(key, 0) + counts[key] = occurrence + 1 + entries[key + (occurrence,)] = (int(sequence), os.path.join(directory, name)) + return entries + + +def first_difference(a, b): + """(offset, count) of the differing bytes, or None when equal.""" + if len(a) != len(b): + return None + count = 0 + first = -1 + for index_ in range(len(a)): + if a[index_] != b[index_]: + if first < 0: + first = index_ + count += 1 + return (first, count) if count else None + + +def main(argv): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('a') + parser.add_argument('b') + parser.add_argument('--kernel', default='', help='only keys whose kernel name contains this') + parser.add_argument('--quiet', action='store_true', help='report only keys that differ') + args = parser.parse_args(argv[1:]) + + left, right = index(args.a), index(args.b) + keys = sorted(set(left) | set(right)) + mismatched = missing = compared = 0 + for key in keys: + if args.kernel and args.kernel not in key[0]: + continue + label = '%s arg%03d %s %s fmt%d #%d' % key + if key not in left or key not in right: + print('MISSING %s: %s' % (label, 'only in A' if key in left else 'only in B')) + missing += 1 + continue + with open(left[key][1], 'rb') as handle: + a = handle.read() + with open(right[key][1], 'rb') as handle: + b = handle.read() + compared += 1 + difference = first_difference(a, b) + if difference is None: + if not args.quiet: + print('ok %s (%d bytes)' % (label, len(a))) + else: + offset, count = difference + print('DIFFERS %s: %d of %d bytes, first at %d (A %02x B %02x)' % + (label, count, len(a), offset, a[offset], b[offset])) + mismatched += 1 + + print('%d arguments compared, %d differ, %d missing' % (compared, mismatched, missing)) + return 1 if mismatched or missing else 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv))