ayo
7c5d3c9157
Add native K/M/RR inference engines, HIP kernels, replay and validation harnesses, and Windows DLSS setup and runtime support. Include GPU profiling tools, build and packaging scripts, platform integration patches, and implementation documentation.
46 lines
2.8 KiB
GLSL
46 lines
2.8 KiB
GLSL
// Round the components of a float32 cooperative accumulator to binary16 (round-to-nearest-even) and widen them
|
|
// back to f32. A register-only conversion, or a f32 -> f16 -> f32 component conversion, is folded away by RADV on
|
|
// gfx1201, and so is the packHalf2x16 round-trip (which is round-toward-zero anyway): the rounding only happens
|
|
// once the binary16 value lands in binary16 memory. Scalar f32 -> f16 -> f32 casts are folded the same way,
|
|
// notably for filtered recurrent features, hence the integer helper below.
|
|
#ifndef D4R_ROUND_HALF_GL
|
|
#define D4R_ROUND_HALF_GL
|
|
// Cooperative matrices are composites of implementation-dependent components: every invocation owns and may
|
|
// subscript a fixed number of them (length(), 8 for a 16x16 subgroup-scope accumulator on a 32-lane wave).
|
|
float roundHalfValue(float v) {
|
|
uint bits = floatBitsToUint(v), magnitude = bits & 0x7fffffffu;
|
|
if (magnitude < 0x38800000u) return roundEven(v * 16777216.0) * (1.0 / 16777216.0);
|
|
if (magnitude >= 0x477ff000u)
|
|
return magnitude > 0x7f800000u ? v : uintBitsToFloat((bits & 0x80000000u) | 0x7f800000u);
|
|
return uintBitsToFloat((bits + 4095u + ((bits >> 13u) & 1u)) & 0xffffe000u);
|
|
}
|
|
vec4 roundHalfValue(vec4 v) {
|
|
return vec4(roundHalfValue(v.x), roundHalfValue(v.y), roundHalfValue(v.z), roundHalfValue(v.w));
|
|
}
|
|
|
|
// The cooperative-matrix store/read-back through a wave-owned row-major region: the cheapest form where the region
|
|
// already exists for the stage's own staging store (dec0 reuses EO). storage/offset/stride are in storage's
|
|
// element units, as in the engine's LD16/ST16.
|
|
#define ROUND_HALF_STORE(m, storage, offset, stride) { \
|
|
coopMatStore(HACC(m), storage, offset, stride, RM); \
|
|
subgroupMemoryBarrierShared(); subgroupBarrier(); \
|
|
HACC rh_st_; coopMatLoad(rh_st_, storage, offset, stride, RM); \
|
|
m = ACC(rh_st_); \
|
|
subgroupMemoryBarrierShared(); subgroupBarrier(); \
|
|
}
|
|
|
|
// The hardware binary16 conversion through one private scratch slot per invocation, ~2x cheaper than a
|
|
// cooperative-matrix store/read-back to shared memory. storage must hold 8 binary16 values per invocation of the
|
|
// workgroup: shared float16_t RS[gl_WorkGroupSize.x * 8u]; (8 = components per invocation, 256 / subgroup size).
|
|
// The memory fences prevent compiler forwarding from removing the rounding. Slots are invocation-private,
|
|
// so no subgroup execution barrier is needed (unlike ROUND_HALF_STORE's cross-lane cooperative loads).
|
|
#define ROUND_HALF_SHARED(m, storage) { \
|
|
for (int rh_i_ = 0, rh_n_ = (m).length(); rh_i_ < rh_n_; ++rh_i_) \
|
|
storage[gl_LocalInvocationID.x * 8u + uint(rh_i_)] = float16_t((m)[rh_i_]); \
|
|
subgroupMemoryBarrierShared(); \
|
|
for (int rh_i_ = 0, rh_n_ = (m).length(); rh_i_ < rh_n_; ++rh_i_) \
|
|
(m)[rh_i_] = float(storage[gl_LocalInvocationID.x * 8u + uint(rh_i_)]); \
|
|
subgroupMemoryBarrierShared(); \
|
|
}
|
|
|
|
#endif |