wouter/d4r
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Engine: native presets K and M in the D3D12 shim, wider game coverage

Browse Source 
A standalone Vulkan inference engine (engine/) for DLSS presets K and M that
runs from weights extracted from the user's own nvngx_dlss.dll, with d4r's
own input, network and reconstruction shaders or the native HIP network
layers, recorded into the game's command list by the shim. Opt-in
([Engine] Enabled / D4R_ENGINE=1); anything uncovered falls back to the
CUDA/ZLUDA path with the reason logged.

Byte-identical to NGX's CUDA backend under the phase-capable reference
(1706x960 -> 2560x1440, native HIP network, eleven frames, RX 9070 XT):
- K render- and display-resolution motion vectors, HDR and LDR
- K game-supplied exposure and automatic exposure with any pre-exposure
  and exposure scale (one residual: effective exposure exactly 2 or 4)
- K and M with the DLSS indicator's inverted-axis parameters
- M with any exposure settings (NGX ignores them for M)
K packages need the current compile_k.py (marker D4RO0003).

Also: HIP shared buffers are pooled across feature recreation (ROCm leaks
destroyed imports), hipnet.bin must match the preset, M-only games do not
load the K model, install_d4r_runtime.sh stages k, k-ldr, m, m-ldr with
hip/, the activation table has a generator (gen_activation_m.py), and
package_release.sh can bundle the engine's shaders (never its weights).

Signed-off-by: woutermans <[email protected]>
This commit is contained in:
woutermans committed 2026-10-10 20:54:01 +02:00
1 parent 16cfc1f063
commit e0d1549a21
82 files changed
+13335 -96
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

engine/output_k.comp
+397
View File
@@ -0,0 +1,397 @@
#version 460
// d4r engine, preset K: the output stage (temporal accumulation and upsampling), written from the algorithm
// described in OUTPUT_K_NOTES.md. One 16x16 workgroup shades a 16x16 block of output pixels.
// 1. The block's footprint of the render-resolution colour (up to 24x24 texels) is range-compressed, converted to
// YCoCg and staged in shared memory (with depth, unless the motion is display-resolution).
// 2. Motion comes from the nearest of the diagonal depth neighbours (display-resolution: the pixel's own vector); it locates the history.
// 3. The 2x-resolution luma history is resampled with a sharpened Catmull-Rom filter at the four sub-pixels.
// 4. The current luma is filtered at the four sub-pixels with twelve taps of an anisotropic Gaussian whose shape
// comes from a table indexed by two network outputs.
// 5. History and current are blended per sub-pixel with a confidence from the tap geometry; chroma follows luma.
// Not handled (the reference kernel has them, the captured frames do not use them): blocks whose footprint exceeds
// 24x24 texels, and the alternative output modes selected by three flag bytes.
#extension GL_EXT_shader_explicit_arithmetic_types : require
#extension GL_KHR_shader_subgroup_basic : require
#extension GL_KHR_shader_subgroup_shuffle : require
#extension GL_GOOGLE_include_directive : require
#include "output_store.glsl"
#include "round_half.glsl"
layout(local_size_x = 16, local_size_y = 16) in;
layout(set = 0, binding = 0, std430) readonly buffer Params
{
vec2 out2rnd, rndOffs; // render position = (pixel + .5) * out2rnd + rndOffs
vec2 near2rnd, sub2rnd; // the same scale for the nearest-texel lookup; for 2x sub-pixels
vec2 out2tok; // token column = int(pixel * out2tok) / 2
vec2 mvScale, mvOffs; // previous position = pixel + .5 + mvScale * (mv + mvOffs)
vec2 invOut, invOutValid; // 1 / output size, for the history uv and for the on-screen test
ivec2 colOrigin, colMax, depOrigin, depMax, mvOrigin, mvMax, outOrigin, outSize;
int tokW, tokH;
uint flags; // bit 0: reset; bit 1: regular depth (larger is farther), otherwise inverted;
// bit 2: display-resolution motion (one vector per output pixel, no depth);
// bits 8..11: which of the model's reconstruction tables this frame uses
float expo, expoRatio; // exposure scale of this frame; previous / current
float sharp, minAlpha, alphaThr;
float lutScale, kA, kB, c184, c186;
} p;
layout(set = 0, binding = 1) uniform sampler2D colorTex; // render resolution, linear RGB + alpha
layout(set = 0, binding = 2) uniform sampler2D histTex; // output resolution: compressed Y, Co, Cg, alpha (bilinear)
layout(set = 0, binding = 3) uniform sampler2D hiresTex; // 2x output resolution: compressed luma
layout(set = 0, binding = 4) uniform sampler2D motionTex;
layout(set = 0, binding = 5) uniform sampler2D depthTex;
layout(set = 0, binding = 6, rgba16f) uniform writeonly image2D histOut;
layout(set = 0, binding = 7, r16ui) uniform writeonly uimage2D hiresOut;
layout(set = 0, binding = 8, rgba16f) uniform writeonly image2D tokOut;
layout(set = 0, binding = 9, FINAL_OUTPUT_FORMAT) uniform writeonly image2D finalOut;
layout(set = 0, binding = 10, std430) readonly buffer Head { float16_t h[]; } head; // 40 values per token: 4 sub-blocks x 10
layout(set = 0, binding = 11, std430) readonly buffer Lut { uvec4 e[]; } lut;
shared f16vec4 tileC[576]; // compressed Y, Co, Cg and alpha of the footprint, 24 texels per row
shared float tileD[576];
// LDR (NGX's *_ldr_* kernels, the game did not set the HDR flag): display-referred colour, no range compression
#ifndef LDR
#define LDR 0
#endif
#if LDR
float compressK(float L) { return 1.0; }
float expandK(float v) { return 1.0; }
#else
// The reference's constants and association (output_color_pack): rcp.approx(y + 1), or above 1000
// rcp.approx(y) * (lg2.approx(fma(y, 9, 1)) * ln2 / 9 + 0xBC4FAE29). Its offset differs by 1.7e-5
// relative from 0.012676, and folding ln2/9 into one constant moves the result a further ulp.
float compressK(float L) // L * compressK(L) maps [0, inf) to [0, 1): Reinhard below 1000, logarithmic above
{
if (L < 1000.0)
return 1.0 / (L + 1.0);
precise const float t = (log2(fma(L, 9.0, 1.0)) * uintBitsToFloat(0x3F317218u)) / 9.0
+ uintBitsToFloat(0xBC4FAE29u);
return t / L;
}
float expandK(float v) // v * expandK(v) is the inverse
{
if (v < uintBitsToFloat(0x3F7FBE88u))
return 1.0 / (1.0 - v);
precise const float y = (v + uintBitsToFloat(0x3C4FAE29u)) * 9.0;
return (exp2(y * uintBitsToFloat(0x3FB8AA3Bu)) - 1.0) / (9.0 * v);
}
#endif
float roundHalfAway(float x) { const float t = trunc(x); return abs(x - t) >= 0.5 ? t + sign(x) : t; }
ivec2 clampOrigin(ivec2 v, ivec2 origin, ivec2 hi) // the reference's clamp: origin for v <= 0, else min(v + origin, hi)
{
return ivec2(v.x > 0 ? min(v.x + origin.x, hi.x) : origin.x, v.y > 0 ? min(v.y + origin.y, hi.y) : origin.y);
}
vec4 softIn(vec4 v) { return mix(0.99 + (v - 0.99) * 0.001, v, lessThan(v, vec4(0.99))); } // stored luma -> working luma
// Catmull-Rom weights of the four taps around a fraction t
void crWeights(float t, out float w0, out float w1, out float w2, out float w3)
{
const float t2 = t * t, t3 = t2 * t;
w0 = t2 - 0.5 * (t + t3);
w1 = 1.5 * t3 - 2.5 * t2 + 1.0;
w3 = 0.5 * (t3 - t2);
w2 = 1.0 - w0 - w1 - w3;
}
// The reference's six coefficients for one filter axis, computed once per pixel: the bilinear split
// mid = w1 / (w1 + w2) and the four sharpened weights, kept separate from the bilinear pair (folding
// sharp*(w1-mid) into mid changes the row result by up to a few ulp, which the half-encoded highlight
// range then amplifies).
struct CrAxis { float mid, split, s0, s1, s2, s3; };
CrAxis crAxis(float w0, float w1, float w2, float w3, float mid)
{
return CrAxis(mid, 1.0 - mid, p.sharp * w0, p.sharp * (w1 - mid), p.sharp * (w2 - (1.0 - mid)), p.sharp * w3);
}
// One pass along one axis: the two middle taps are bilinear, the four sharpened taps are saturated, and
// every term is the reference's own rounded product/fma.
float crPass(float t0, float t1, float t2, float t3, CrAxis c)
{
float t = c.split * t2;
t = fma(c.mid, t1, t);
t = fma(c.s0, clamp(t0, 0.0, 1.0), t);
t = fma(c.s1, clamp(t1, 0.0, 1.0), t);
t = fma(c.s2, clamp(t2, 0.0, 1.0), t);
t = fma(c.s3, clamp(t3, 0.0, 1.0), t);
return t;
}
void main()
{
const ivec2 pix = ivec2(gl_GlobalInvocationID.xy);
const vec2 pc = vec2(pix) + 0.5;
const bool displayMv = (p.flags & 4u) != 0u;
// ---------------------------------------------------------------- 1. footprint of the block
const vec2 b0 = (vec2(gl_WorkGroupID.xy * 16u) + 0.5) * p.out2rnd + p.rndOffs;
const vec2 b1 = (vec2(gl_WorkGroupID.xy * 16u + 15u) + 0.5) * p.out2rnd + p.rndOffs;
const ivec2 t0 = ivec2(vec2(roundHalfAway(b0.x), roundHalfAway(b0.y)) - 4.0);
const ivec2 t1 = ivec2(vec2(roundHalfAway(b1.x), roundHalfAway(b1.y)) + 4.0);
const int tw = t1.x - t0.x + 1, th = t1.y - t0.y + 1;
for (int i = int(gl_LocalInvocationIndex); i < tw * th; i += 256)
{
const ivec2 t = ivec2(i % tw, i / tw), g = t + t0;
const vec4 c = texelFetch(colorTex, clampOrigin(g, p.colOrigin, p.colMax), 0);
const vec3 rgb = max(c.rgb, 0.0);
const float Y = 0.25 * rgb.r + 0.5 * rgb.g + 0.25 * rgb.b;
const float L = p.expo * Y;
const float k = compressK(L);
tileC[t.y * 24 + t.x] = f16vec4(L * k, p.expo * (0.5 * rgb.r - 0.5 * rgb.b) * k, p.expo * (0.5 * rgb.g - 0.25 * rgb.r - 0.25 * rgb.b) * k, c.a);
// larger = nearer: regular depth is negated, which turns its "nearest" comparisons into the inverted ones
if (!displayMv)
{
const float depth = texelFetch(depthTex, clampOrigin(g, p.depOrigin, p.depMax), 0).r;
tileD[t.y * 24 + t.x] = (p.flags & 2u) != 0u ? -depth : depth;
}
}
memoryBarrierShared(); barrier();
// ---------------------------------------------------------------- network outputs of this pixel's 2x2 sub-block
const ivec2 tk = ivec2(vec2(pix) * p.out2tok);
const uint hbase = uint((tk.y / 2) * p.tokW + tk.x / 2) * 40u + uint((tk.y & 1) * 2 + (tk.x & 1)) * 10u;
// sigmoid of a head word (the reference's fma chain: y = x * (x^2 + 27) / fma(x^2, 9, 27), sat(0.5*y + 0.5))
const float x = 0.5 * float(head.h[hbase + 9u]);
precise const float x2 = x * x;
const float sig = clamp(x < -3.0 ? 0.0 : x > 3.0 ? 1.0
: fma((x * (x2 + 27.0)) / fma(x2, 9.0, 27.0), 0.5, 0.5), 0.0, 1.0);
// ---------------------------------------------------------------- 2. motion
// Render-resolution vectors come from the nearest diagonal depth neighbour; display-resolution ones belong to the pixel.
ivec2 mvAt = pix;
if (!displayMv)
{
const vec2 nr = pc * p.near2rnd + p.rndOffs;
const ivec2 ni = ivec2(roundHalfAway(nr.x), roundHalfAway(nr.y));
const int di = (ni.y - t0.y) * 24 + (ni.x - t0.x);
const float d0 = tileD[di], dmm = tileD[di - 25], dpp = tileD[di + 25], dpm = tileD[di - 23], dmp = tileD[di + 23];
ivec2 dsel = ivec2(0);
if (d0 < 0.5 * (dmm + dpp) - 0.001 * abs(d0) || d0 < 0.5 * (dpm + dmp) - 0.001 * abs(d0))
{
float m = d0;
if (dmm > m) { m = dmm; dsel = ivec2(-1, -1); }
if (dpm > m) { m = dpm; dsel = ivec2(1, -1); }
if (dmp > m) { m = dmp; dsel = ivec2(-1, 1); }
if (dpp > m) { dsel = ivec2(1, 1); }
}
mvAt = ni + dsel;
}
const vec2 mv = texelFetch(motionTex, clampOrigin(mvAt, p.mvOrigin, p.mvMax), 0).xy;
const vec2 q = pc + p.mvScale * (mv + p.mvOffs); // where this pixel was, in output pixels
const vec2 qv = q * p.invOutValid;
bool noHist = (p.flags & 1u) != 0u || qv.x < 0.0 || qv.x > 1.0 || qv.y < 0.0 || qv.y > 1.0;
const vec4 hist = textureLod(histTex, q * p.invOut, 0.0);
// ---------------------------------------------------------------- 3. luma history at the four sub-pixels
vec4 Hq;
{
const vec2 h = (q - 0.5) * 2.0 + 0.5;
const vec2 base = floor(h - 0.5) + 0.5;
const vec2 fr = clamp(h - base, 0.0, 1.0);
float x0, x1, x2, x3, y0, y1, y2, y3;
crWeights(fr.x, x0, x1, x2, x3); crWeights(fr.y, y0, y1, y2, y3);
const float mx = x1 / (x1 + x2), my = y1 / (y1 + y2);
const vec2 ts = p.invOut * 0.5;
// 5x5 texels T[col][row] from nine gathers (gather: x = (0,1), y = (1,1), z = (1,0), w = (0,0))
vec4 g00 = textureGather(hiresTex, ts * (base + vec2(-0.5, -0.5)), 0);
vec4 g10 = textureGather(hiresTex, ts * (base + vec2(1.5, -0.5)), 0);
vec4 g20 = textureGather(hiresTex, ts * (base + vec2(3.5, -0.5)), 0);
vec4 g01 = textureGather(hiresTex, ts * (base + vec2(-0.5, 1.5)), 0);
vec4 g11 = textureGather(hiresTex, ts * (base + vec2(1.5, 1.5)), 0);
vec4 g21 = textureGather(hiresTex, ts * (base + vec2(3.5, 1.5)), 0);
vec4 g02 = textureGather(hiresTex, ts * (base + vec2(-0.5, 3.5)), 0);
vec4 g12 = textureGather(hiresTex, ts * (base + vec2(1.5, 3.5)), 0);
vec4 g22 = textureGather(hiresTex, ts * (base + vec2(3.5, 3.5)), 0);
// stored luma at 0.99 and above is an encoding of [0.99, 1): rare enough to branch on
const vec4 gm = max(max(max(g00, g10), max(g20, g01)), max(max(g11, g21), max(max(g02, g12), g22)));
if (!(max(max(gm.x, gm.y), max(gm.z, gm.w)) < 0.99))
{
g00 = softIn(g00); g10 = softIn(g10); g20 = softIn(g20); g01 = softIn(g01); g11 = softIn(g11);
g21 = softIn(g21); g02 = softIn(g02); g12 = softIn(g12); g22 = softIn(g22);
}
float T[5][5]; // [row][col]
T[0][0] = g00.w; T[0][1] = g00.z; T[1][0] = g00.x; T[1][1] = g00.y;
T[0][2] = g10.w; T[0][3] = g10.z; T[1][2] = g10.x; T[1][3] = g10.y;
T[0][4] = g20.w; T[1][4] = g20.x;
T[2][0] = g01.w; T[2][1] = g01.z; T[3][0] = g01.x; T[3][1] = g01.y;
T[2][2] = g11.w; T[2][3] = g11.z; T[3][2] = g11.x; T[3][3] = g11.y;
T[2][4] = g21.w; T[3][4] = g21.x;
T[4][0] = g02.w; T[4][1] = g02.z; T[4][2] = g12.w; T[4][3] = g12.z; T[4][4] = g22.w;
// Each row filtered for the two sub-pixel columns. Stored luma lies in [0, 1), so the clamps of the
// sharpened taps do nothing here; the column pass below keeps them (rows can overshoot).
const CrAxis cx = crAxis(x0, x1, x2, x3, mx), cy = crAxis(y0, y1, y2, y3, my);
float R[5][2];
for (int r = 0; r < 5; ++r)
for (int c = 0; c < 2; ++c)
R[r][c] = crPass(T[r][c], T[r][c + 1], T[r][c + 2], T[r][c + 3], cx);
for (int sy = 0; sy < 2; ++sy)
for (int sx = 0; sx < 2; ++sx)
{
const float v = crPass(R[sy][sx], R[sy + 1][sx], R[sy + 2][sx], R[sy + 3][sx], cy);
const float lo = min(min(T[sy + 1][sx + 1], T[sy + 1][sx + 2]), min(T[sy + 2][sx + 1], T[sy + 2][sx + 2]));
const float hi = max(max(T[sy + 1][sx + 1], T[sy + 1][sx + 2]), max(T[sy + 2][sx + 1], T[sy + 2][sx + 2]));
Hq[sy * 2 + sx] = max(min(v, hi), lo);
}
// the history was compressed with last frame's exposure. The reference always re-exposes
// (v' = L * k(L) with L = ratio * (v * g(v))); skipping it when the ratio is exactly 1 leaves the
// pre-rounding value, which is not what k(g(v)) costs on a real frame.
for (int s = 0; s < 4; ++s)
{
const float L = p.expoRatio * (Hq[s] * expandK(Hq[s]));
Hq[s] = L * compressK(L);
}
}
{
precise const float chk = fma(sig, hist.b, fma(hist.r, hist.g, fma(Hq.x, Hq.y, Hq.z * Hq.w)));
noHist = noHist || !(abs(chk) < uintBitsToFloat(0x7F800000u));
}
const float s1 = noHist ? 1.0 : sig, ha = noHist ? 1.0 : hist.a;
precise const float f343 = fma(s1, 1.0 - ha, ha);
precise const float f344 = fma(ha, 1.0 - s1 * s1, s1 * s1);
if (noHist) Hq = vec4(0.0);
const vec3 hc = noHist ? vec3(0.0) : hist.rgb;
// ---------------------------------------------------------------- current colour, bilinear from the tile
// Positions, weights and sums of this part are half floats, as in the reference: the filter below is sensitive
// to where the samples sit, and half precision moves them by up to 1/64 texel.
const f16vec2 u = f16vec2(pc * p.out2rnd + p.rndOffs - vec2(t0));
const ivec2 ui = ivec2(floor(vec2(u)));
const f16vec2 uf = clamp(u - f16vec2(ui), float16_t(0.0), float16_t(1.0)), ug = float16_t(1.0) - uf;
const int ci = ui.y * 24 + ui.x;
f16vec4 curh = tileC[ci] * (ug.x * ug.y);
curh = fma(tileC[ci + 1], f16vec4(uf.x * ug.y), curh);
curh = fma(tileC[ci + 24], f16vec4(ug.x * uf.y), curh);
curh = fma(tileC[ci + 25], f16vec4(uf.x * uf.y), curh);
vec4 cur = vec4(curh);
// "bright" over the 2x2 pixel quad: the lanes of a subgroup are the pixels of two rows, x + 16 * (y & 1)
uint bright = cur.x > 0.8 ? 1u : 0u;
bright |= subgroupShuffleXor(bright, 16u);
bright |= subgroupShuffleXor(bright, 1u);
// ---------------------------------------------------------------- 4. kernel-predicted luma filter
const f16vec2 hm = f16vec2(head.h[hbase], head.h[hbase + 1u]);
const f16vec2 m = f16vec2(vec2(hm) / vec2(float16_t(1.0) + abs(hm)));
const ivec2 mi = clamp(ivec2(roundEven(vec2(m * float16_t(p.lutScale)))) + 8, 0, 15);
const uint fl = (bright << 2) | ((sig < 0.25 && f343 < 0.25) ? 2u : 0u) | (p.alphaThr > f343 ? 1u : 0u);
const uvec4 le = lut.e[(((p.flags >> 8) & 15u) << 11) | (fl << 8) | uint(mi.x << 4) | uint(mi.y)];
const float16_t e0 = uint16BitsToFloat16(uint16_t(le.x & 0xffffu)), e1 = uint16BitsToFloat16(uint16_t(le.x >> 16));
const float16_t e2 = uint16BitsToFloat16(uint16_t(le.y & 0xffffu)), slope = uint16BitsToFloat16(uint16_t(le.y >> 16));
const bool alongX = (le.z & 0xffffu) == 0u; // rows of taps step in y, taps within a row step in x
const float16_t e5 = uint16BitsToFloat16(uint16_t(le.z >> 16)), e6 = uint16BitsToFloat16(uint16_t(le.w & 0xffffu));
// sub-pixel positions in tile coordinates: s = sy * 2 + sx
// The reference fuses the scale and the offset and subtracts the tile origin afterwards. The order matters
// where a position falls on a half-float rounding tie (a whole output column at some size ratios): any other
// order moves those samples by one half-float step, which the recurrent history then spreads over the frame.
precise const vec2 sa0 = fma(vec2(pix * 2) + 0.5, p.sub2rnd, p.rndOffs), sa1 = fma(vec2(pix * 2 + 1) + 0.5, p.sub2rnd, p.rndOffs);
precise const vec2 sq0 = sa0 - vec2(t0), sq1 = sa1 - vec2(t0);
const f16vec2 sp0 = f16vec2(sq0), sp1 = f16vec2(sq1);
const f16vec4 subX = f16vec4(sp0.x, sp1.x, sp0.x, sp1.x), subY = f16vec4(sp0.y, sp0.y, sp1.y, sp1.y);
const f16vec4 subA = alongX ? subY : subX, subB = alongX ? subX : subY;
const float16_t cA = alongX ? u.y : u.x, cB = alongX ? u.x : u.y;
const int stepB = alongX ? 1 : 24, stepA = alongX ? 24 : 1;
const int a0 = int(floor(float(cA))) - 1;
// Every product and sum of the filter is rounded to half precision, as in the reference: with single precision
// the result differs visibly more (49 dB against 116 dB on the luma history of the test frames).
f16vec4 sumY = f16vec4(0.0), sumW = f16vec4(0.0);
const float16_t line0 = cB + (float16_t(a0) - cA) * slope;
float16_t line = line0;
for (int r = 0; r < 4; ++r)
{
const int bs = int(roundEven(float(line))) - 1;
const f16vec4 dA = float16_t(a0 + r) - subA;
f16vec4 db = float16_t(bs) - subB;
const f16vec4 cross = dA * e2;
f16vec4 lin = fma(db, cross, (dA * dA) * e0);
const int ti = (a0 + r) * stepA + bs * stepB;
for (int j = 0; j < 3; ++j)
{
// Keep the rounded exponent in f16 through exp2; avoids f16/f32 conversions.
const f16vec4 w = exp2(fma(db * db, f16vec4(e1), lin));
sumY = fma(w, f16vec4(tileC[ti + j * stepB].x), sumY); sumW += w;
db += float16_t(1.0); lin += cross;
}
line += slope;
}
vec4 F = vec4(sumY) / vec4(sumW);
// Where every exponent is below -12 the result is the tap with the largest exponent instead. That needs all
// twelve weights below 2^-12, so the sum of weights rules it out for almost every pixel.
if (any(lessThan(sumW, f16vec4(12.5 / 4096.0))))
{
f16vec4 eMax = f16vec4(-60000.0), yMax = f16vec4(0.0);
line = line0;
for (int r = 0; r < 4; ++r)
{
const int bs = int(roundEven(float(line))) - 1;
const f16vec4 dA = float16_t(a0 + r) - subA;
f16vec4 db = float16_t(bs) - subB;
const f16vec4 cross = dA * e2;
f16vec4 lin = fma(db, cross, (dA * dA) * e0);
const int ti = (a0 + r) * stepA + bs * stepB;
for (int j = 0; j < 3; ++j)
{
const f16vec4 E = fma(db * db, f16vec4(e1), lin);
const bvec4 gt = not(greaterThan(eMax, E)); // a later tap wins ties
eMax = mix(eMax, E, gt); yMax = mix(yMax, f16vec4(tileC[ti + j * stepB].x), gt);
db += float16_t(1.0); lin += cross;
}
line += slope;
}
F = mix(F, vec4(yMax), lessThan(eMax, f16vec4(-12.0)));
}
// The reference holds the filtered current luma in half precision: the blend at the end of the stage reads
// it as an f16 value widened to f32 (dlss-0046 converts every lane with cvt.rn.f16.f32 before the
// `fma bl, F - Hq, Hq` that forms N), and the normalisation and the low-exponent fallback also produce
// halves. Keeping it f32 pulls N a fraction of a half code towards the current luma wherever the history
// and the current frame differ, which the recurrent history then carries into the next frames.
F = roundHalfValue(F);
// ---------------------------------------------------------------- 5. blend
f16vec4 conf;
{
const f16vec4 dA = roundEven(subA) - subA;
const f16vec4 ln = fma(dA, f16vec4(slope), subB);
const f16vec4 dB = abs(roundEven(ln) - ln);
const float16_t kA = float16_t(alongX ? p.kB : p.kA), kB = float16_t(alongX ? p.kA : p.kB);
const f16vec4 sA = clamp(fma(abs(dA), f16vec4(kA), f16vec4(4.5)), float16_t(0.0), float16_t(1.0));
const f16vec4 sB = clamp(fma(dB, f16vec4(kB), f16vec4(4.5)), float16_t(0.0), float16_t(1.0));
conf = fma(sA, f16vec4(e5), f16vec4(e6)) * sB;
}
const float16_t hv = float16_t(f343), hv2 = float16_t(0.1) + float16_t(0.9) * hv;
const float16_t lo = (hv - float16_t(p.c184)) * float16_t(p.c186), hi = (hv2 - float16_t(p.c184)) * float16_t(p.c186);
const float16_t avg = float16_t(0.25) * ((conf.x + conf.y) + (conf.z + conf.w));
const float alphaC = float(clamp(fma(avg, hv2 - hi, hi), float16_t(0.0), float16_t(1.0)));
const vec4 bl = vec4(clamp(fma(conf, f16vec4(hv - lo), f16vec4(lo)), float16_t(0.0), float16_t(1.0)));
if (!(abs(cur.x * F.x * F.y * F.z * F.w) < uintBitsToFloat(0x7F800000u))) { cur.xyz = vec3(0.0); F = vec4(0.0); }
// a single-rounded fma, as the reference: fma.rn.ftz.f32 N, bl, F - Hq, Hq
const vec4 N = fma(bl, F - Hq, Hq);
// Native averages the four f32 blend registers, before the encoded half-surface stores.
{
const vec4 enc = mix(fma(N - 0.99, vec4(1000.0), vec4(0.99)), N, lessThan(N, vec4(0.99)));
const u16vec4 b = float16BitsToUint16(f16vec4(enc));
imageStore(hiresOut, pix * 2, uvec4(b.x)); imageStore(hiresOut, pix * 2 + ivec2(1, 0), uvec4(b.y));
imageStore(hiresOut, pix * 2 + ivec2(0, 1), uvec4(b.z)); imageStore(hiresOut, pix * 2 + ivec2(1, 1), uvec4(b.w));
}
// the reference fuses the history product: fma(hc, 1 - alphaC, alphaC * cur)
precise const vec3 C = fma(hc, vec3(1.0 - alphaC), alphaC * cur.xyz);
precise const float Nm = 0.25 * (((N.x + N.y) + N.z) + N.w);
const vec2 chroma = C.yz / max(C.x, 1e-5) * Nm;
{
// history colour: chroma limited so that the RGB it stands for stays non-negative
const float Y = max(Nm, 0.0);
float lim = Y;
if (chroma.y - 1e-5 < -Y) lim = 1e-5 - chroma.y;
if (chroma.x - chroma.y - 1e-5 < -lim) lim = chroma.y - chroma.x + 1e-5;
if (-chroma.x - chroma.y - 1e-5 < -lim) lim = chroma.x + chroma.y + 1e-5;
const float sc = lim <= Y ? 1.0 : Y / lim;
imageStore(histOut, pix, vec4(Y, chroma * sc, clamp(max(f344 / (f344 + 1.0), p.minAlpha), 0.0, 1.0)));
}
if (pix.x < p.outSize.x && pix.y < p.outSize.y)
{
// the reference multiplies by ginv(Nm) first and divides by the exposure afterwards; the other
// association (ginv / exposure first) rounds differently before the RGB reconstruction
const vec3 ycc = vec3(Nm, chroma) * expandK(Nm) / p.expo;
imageStore(finalOut, p.outOrigin + pix, finalOutputValue(vec4(ycc.x + ycc.y - ycc.z, ycc.x + ycc.z, ycc.x - ycc.y - ycc.z, cur.a)));
}
// ---------------------------------------------------------------- per-token feature for the next frame's input stage
if (pix.x < p.tokW && pix.y < p.tokH)
{
const uint tb = uint(pix.y * p.tokW + pix.x) * 40u;
precise vec4 ms;
for (uint s = 0u; s < 4u; ++s)
ms[s] = 0.2 * (float(head.h[tb + s * 10u + 3u]) + float(head.h[tb + s * 10u + 4u]) + float(head.h[tb + s * 10u + 5u])
+ float(head.h[tb + s * 10u + 6u]) + float(head.h[tb + s * 10u + 7u]));
// Match the native scalar FMA chain; dot() permits a different association at half-store boundaries.
precise float norm = fma(ms.x, ms.x, ms.y * ms.y);
norm = fma(ms.z, ms.z, norm);
norm = fma(ms.w, ms.w, norm);
norm += 1.0;
precise const vec4 feature = ms * inversesqrt(norm);
imageStore(tokOut, pix, feature);
}
}