feat: add d4r inference engines and Windows parity tooling

Add native K/M/RR inference engines, HIP kernels, replay and validation
harnesses, and Windows DLSS setup and runtime support. Include GPU
profiling tools, build and packaging scripts, platform integration
patches, and implementation documentation.
This commit is contained in:
ayo committed 2026-10-16 08:54:47 -04:00
1 parent 25493b3198
commit 7c5d3c9157
430 files changed
+76191

No files matched your search

+46
View File
@@ -0,0 +1,46 @@
// Round the components of a float32 cooperative accumulator to binary16 (round-to-nearest-even) and widen them
// back to f32. A register-only conversion, or a f32 -> f16 -> f32 component conversion, is folded away by RADV on
// gfx1201, and so is the packHalf2x16 round-trip (which is round-toward-zero anyway): the rounding only happens
// once the binary16 value lands in binary16 memory. Scalar f32 -> f16 -> f32 casts are folded the same way,
// notably for filtered recurrent features, hence the integer helper below.
#ifndef D4R_ROUND_HALF_GL
#define D4R_ROUND_HALF_GL
// Cooperative matrices are composites of implementation-dependent components: every invocation owns and may
// subscript a fixed number of them (length(), 8 for a 16x16 subgroup-scope accumulator on a 32-lane wave).
float roundHalfValue(float v) {
uint bits = floatBitsToUint(v), magnitude = bits & 0x7fffffffu;
if (magnitude < 0x38800000u) return roundEven(v * 16777216.0) * (1.0 / 16777216.0);
if (magnitude >= 0x477ff000u)
return magnitude > 0x7f800000u ? v : uintBitsToFloat((bits & 0x80000000u) | 0x7f800000u);
return uintBitsToFloat((bits + 4095u + ((bits >> 13u) & 1u)) & 0xffffe000u);
}
vec4 roundHalfValue(vec4 v) {
return vec4(roundHalfValue(v.x), roundHalfValue(v.y), roundHalfValue(v.z), roundHalfValue(v.w));
}
// The cooperative-matrix store/read-back through a wave-owned row-major region: the cheapest form where the region
// already exists for the stage's own staging store (dec0 reuses EO). storage/offset/stride are in storage's
// element units, as in the engine's LD16/ST16.
#define ROUND_HALF_STORE(m, storage, offset, stride) { \
coopMatStore(HACC(m), storage, offset, stride, RM); \
subgroupMemoryBarrierShared(); subgroupBarrier(); \
HACC rh_st_; coopMatLoad(rh_st_, storage, offset, stride, RM); \
m = ACC(rh_st_); \
subgroupMemoryBarrierShared(); subgroupBarrier(); \
}
// The hardware binary16 conversion through one private scratch slot per invocation, ~2x cheaper than a
// cooperative-matrix store/read-back to shared memory. storage must hold 8 binary16 values per invocation of the
// workgroup: shared float16_t RS[gl_WorkGroupSize.x * 8u]; (8 = components per invocation, 256 / subgroup size).
// The memory fences prevent compiler forwarding from removing the rounding. Slots are invocation-private,
// so no subgroup execution barrier is needed (unlike ROUND_HALF_STORE's cross-lane cooperative loads).
#define ROUND_HALF_SHARED(m, storage) { \
for (int rh_i_ = 0, rh_n_ = (m).length(); rh_i_ < rh_n_; ++rh_i_) \
storage[gl_LocalInvocationID.x * 8u + uint(rh_i_)] = float16_t((m)[rh_i_]); \
subgroupMemoryBarrierShared(); \
for (int rh_i_ = 0, rh_n_ = (m).length(); rh_i_ < rh_n_; ++rh_i_) \
(m)[rh_i_] = float(storage[gl_LocalInvocationID.x * 8u + uint(rh_i_)]); \
subgroupMemoryBarrierShared(); \
}
#endif