rr: add FSR Ray Regeneration, exact RDNA3 inference and motion experiments
Add an experimental FSR Ray Regeneration backend for D3D12 RR requests: - Convert depth, motion, normals and albedos into AMD denoiser inputs. - Split and recompose diffuse/specular signals, then run Super Sampling. - Add stability/detail controls, demodulation guards and halo limiting. - Handle AMD provider discovery under NVIDIA GPU spoofing. - Add runtime probes, synthetic reference scenes and per-dispatch profiling. Add exact gfx11 DLSS-RR inference using f16 cooperative matrices: - Reproduce FP8 conversions, accumulation and camera reprojection arithmetic. - Widen model weights and avoid requiring VK_EXT_shader_float8. - Extend model building, replay tools and descriptor metadata. Expand motion preprocessing and experimental AW2 native-motion support: - Add tiled LDS dilation, padded allocation handling and thin-only sky fill. - Order dilation with input waits and NGX on the selected stream. - Add jitter-aligned native composition, hybrid routing and wire history repair. - Add prepared masks, sparse repair and independent motion/repair controls. Add optional fused R11G11B10 input/output conversion, sampled stage timings and profile summarization. Extend configuration, packaging, documentation, FidelityFX headers and CPU/GPU regression coverage.
This commit is contained in:
ayo committed
1 parent
17c1828f97
commit
34e6b0fe71
83 files changed
+11300
-195
No files matched your search
@@ -0,0 +1,55 @@
|
||||
#version 450
|
||||
// FSR Ray Regeneration backend, stage 3: the denoised diffuse and specular signals remodulated by
|
||||
// their albedos, plus the residual the denoiser never saw (particles, what demodulation could not
|
||||
// carry, sky). The result is render-resolution colour for the Super Sampling stage.
|
||||
layout(local_size_x = 8, local_size_y = 8) in;
|
||||
|
||||
layout(binding = 0, rgba16f) uniform readonly image2D denoisedDiffuse;
|
||||
layout(binding = 1, rgba16f) uniform readonly image2D denoisedSpecular;
|
||||
layout(binding = 2, rgba16f) uniform readonly image2D diffuseAlbedo;
|
||||
layout(binding = 3, rgba16f) uniform readonly image2D specularAlbedo;
|
||||
layout(binding = 4, rgba16f) uniform readonly image2D residual;
|
||||
layout(binding = 5, rgba16f) uniform writeonly image2D composed;
|
||||
layout(binding = 6, rgba16f) uniform readonly image2D noisyDiffuse;
|
||||
layout(binding = 7, rgba16f) uniform readonly image2D noisySpecular;
|
||||
|
||||
layout(push_constant) uniform Constants
|
||||
{
|
||||
uvec2 renderSize;
|
||||
float haloLimit; // > 0: each denoised lobe at most this multiple of the brightest noisy input of that lobe around it
|
||||
uint unused;
|
||||
} pc;
|
||||
|
||||
void main()
|
||||
{
|
||||
const ivec2 px = ivec2(gl_GlobalInvocationID.xy);
|
||||
if (px.x >= int(pc.renderSize.x) || px.y >= int(pc.renderSize.y))
|
||||
return;
|
||||
const vec3 da = imageLoad(diffuseAlbedo, px).rgb, sa = imageLoad(specularAlbedo, px).rgb;
|
||||
vec3 diffuse = imageLoad(denoisedDiffuse, px).rgb * da;
|
||||
vec3 specular = imageLoad(denoisedSpecular, px).rgb * sa;
|
||||
if (pc.haloLimit > 0.0)
|
||||
{
|
||||
// The denoiser can carry a thin object's large demodulated signal (dark albedo) across its edge, where this
|
||||
// texel's albedo turns it into a bright fringe. A denoised value above everything around it in the noisy
|
||||
// input is such a leak: capped, by luminance (the hue is kept).
|
||||
const vec3 w = vec3(0.2126, 0.7152, 0.0722);
|
||||
float diffuseMax = 0.0, specularMax = 0.0;
|
||||
for (int dy = -1; dy <= 1; ++dy)
|
||||
for (int dx = -1; dx <= 1; ++dx)
|
||||
{
|
||||
const ivec2 q = clamp(px + ivec2(dx, dy), ivec2(0), ivec2(pc.renderSize) - 1);
|
||||
// what the lobe contributes there (the neighbour's own albedo): a leaked demodulated signal
|
||||
// remodulated by this texel's albedo exceeds all of these
|
||||
diffuseMax = max(diffuseMax, dot(imageLoad(noisyDiffuse, q).rgb * imageLoad(diffuseAlbedo, q).rgb, w));
|
||||
specularMax = max(specularMax, dot(imageLoad(noisySpecular, q).rgb * imageLoad(specularAlbedo, q).rgb, w));
|
||||
}
|
||||
const float dl = dot(diffuse, w), sl = dot(specular, w);
|
||||
if (dl > pc.haloLimit * diffuseMax)
|
||||
diffuse *= pc.haloLimit * diffuseMax / dl;
|
||||
if (sl > pc.haloLimit * specularMax)
|
||||
specular *= pc.haloLimit * specularMax / sl;
|
||||
}
|
||||
const vec3 color = diffuse + specular + imageLoad(residual, px).rgb;
|
||||
imageStore(composed, px, vec4(clamp(color, 0.0, 65000.0), 1.0));
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
#version 450
|
||||
// FSR Ray Regeneration backend, stage 1: a game's DLSS Ray Reconstruction inputs -> the inputs of
|
||||
// AMD's FidelityFX denoiser 1.2 (ffx_denoiser.h), at render resolution.
|
||||
//
|
||||
// The split of the game's single noisy colour into a diffuse and a specular signal follows the
|
||||
// DLSS-RR -> FSR-RR adapter of Zach Hembree's OptiScaler branch ffx-denoise-experimental (GPL-3.0,
|
||||
// 18ac686, FSRDInputConv.hlsl, "Mode 2") and burak113's continuation (5a6adb1): each lobe gets the
|
||||
// colour in proportion to its albedo and is demodulated by it. That is an approximation - one
|
||||
// colour cannot be split exactly - and what does not survive demodulation goes to the residual,
|
||||
// which bypasses the denoiser. Unlike that reference this writes FSR-RR 1.2's contract: signed
|
||||
// linear depth reconstructed from hardware or linear depth, motion as previous-minus-current UV,
|
||||
// linear (non-gamma) albedo and the colour before particles as the denoised base.
|
||||
layout(local_size_x = 8, local_size_y = 8) in;
|
||||
|
||||
layout(binding = 0) uniform sampler2D inColor;
|
||||
layout(binding = 1) uniform sampler2D inDepth;
|
||||
layout(binding = 2) uniform sampler2D inMotion;
|
||||
layout(binding = 3) uniform sampler2D inNormals;
|
||||
layout(binding = 4) uniform sampler2D inRoughness; // the normals again when roughness is packed
|
||||
layout(binding = 5) uniform sampler2D inDiffuseAlbedo;
|
||||
layout(binding = 6) uniform sampler2D inSpecularAlbedo;
|
||||
layout(binding = 7) uniform sampler2D inSpecularHit; // bound to the normals when absent
|
||||
layout(binding = 8) uniform sampler2D inDiffuseHit; // bound to the normals when absent
|
||||
layout(binding = 9) uniform sampler2D inBeforeParticles; // bound to the colour when absent
|
||||
|
||||
layout(binding = 10, r32f) uniform writeonly image2D outLinearDepth;
|
||||
layout(binding = 11, rgba16f) uniform writeonly image2D outMotion;
|
||||
layout(binding = 12, rgba16f) uniform writeonly image2D outNormals;
|
||||
layout(binding = 13, rgba16f) uniform writeonly image2D outDiffuseAlbedo;
|
||||
layout(binding = 14, rgba16f) uniform writeonly image2D outSpecularAlbedo;
|
||||
layout(binding = 15, rgba16f) uniform writeonly image2D outDiffuse;
|
||||
layout(binding = 16, rgba16f) uniform writeonly image2D outSpecular;
|
||||
layout(binding = 17, rgba16f) uniform writeonly image2D outResidual;
|
||||
|
||||
// Matrices are uploaded as the row-major, row-vector (v' = vM) float[16] NGX and FFX use. Read as a
|
||||
// GLSL (column-major) mat4 that is the transpose, so M * v here is the row-vector product v * M.
|
||||
layout(push_constant) uniform Constants
|
||||
{
|
||||
mat4 clipToView; // inverse(ViewToClipMatrix)
|
||||
mat4 viewToPreviousView; // inverse(WorldToView) * previous WorldToView
|
||||
vec4 renderSize; // width, height, 1 / width, 1 / height
|
||||
vec4 motion; // xy: MV texel -> UV delta (MV.Scale / MV resolution); zw: MV texel per render pixel
|
||||
uvec4 colorDepthBase; // colour subrect origin xy, depth subrect origin xy
|
||||
uvec4 motionBaseFlags; // motion subrect origin xy, flags, unused
|
||||
vec4 depthParams; // x: sign of view z in front of the camera, y: sky depth (hardware), z: far marker
|
||||
vec4 detail; // x: share of the colour that bypasses the denoiser (0: none); y: smallest albedo a
|
||||
// signal is divided by
|
||||
} pc;
|
||||
|
||||
const uint kHardwareDepth = 1u;
|
||||
const uint kRoughnessPacked = 2u;
|
||||
const uint kSpecularHit = 4u;
|
||||
const uint kSpecularHitInW = 8u; // DLSSD.SpecularRayDirectionHitDistance: the distance is .w
|
||||
const uint kDiffuseHit = 16u;
|
||||
const uint kDiffuseHitInW = 32u;
|
||||
const uint kNormalsUnorm = 64u; // [0,1]-encoded normals (UNORM formats)
|
||||
const uint kBeforeParticles = 128u;
|
||||
const uint kInvalidNormalPassthrough = 256u;
|
||||
|
||||
const float kHalfMax = 65000.0;
|
||||
|
||||
vec3 safe(vec3 v)
|
||||
{
|
||||
// NaN fails both comparisons and becomes 0
|
||||
return vec3(v.x >= 0.0 ? min(v.x, kHalfMax) : 0.0, v.y >= 0.0 ? min(v.y, kHalfMax) : 0.0,
|
||||
v.z >= 0.0 ? min(v.z, kHalfMax) : 0.0);
|
||||
}
|
||||
|
||||
// AMD's NormalToOctahedronUv (FidelityFX denoising.md 5.3.1)
|
||||
vec2 octahedron(vec3 n)
|
||||
{
|
||||
n /= abs(n.x) + abs(n.y) + abs(n.z);
|
||||
vec2 k = vec2(n.x >= 0.0 ? 1.0 : -1.0, n.y >= 0.0 ? 1.0 : -1.0);
|
||||
vec2 xy = n.z < 0.0 ? (1.0 - abs(n.yx)) * k : n.xy;
|
||||
return xy * 0.5 + 0.5;
|
||||
}
|
||||
|
||||
void main()
|
||||
{
|
||||
const ivec2 px = ivec2(gl_GlobalInvocationID.xy);
|
||||
if (px.x >= int(pc.renderSize.x) || px.y >= int(pc.renderSize.y))
|
||||
return;
|
||||
const uint flags = pc.motionBaseFlags.z;
|
||||
const vec2 uv = (vec2(px) + 0.5) * pc.renderSize.zw;
|
||||
const vec2 ndc = vec2(uv.x * 2.0 - 1.0, 1.0 - uv.y * 2.0);
|
||||
|
||||
// Signed view-space position. Hardware depth goes through the inverse projection, which carries
|
||||
// the game's handedness and (reversed) depth range; linear depth is the distance along the view
|
||||
// axis, given its sign by the projection.
|
||||
const float depth = texelFetch(inDepth, ivec2(pc.colorDepthBase.zw) + px, 0).r;
|
||||
vec3 viewPos;
|
||||
bool sky;
|
||||
if ((flags & kHardwareDepth) != 0u)
|
||||
{
|
||||
const vec4 p = pc.clipToView * vec4(ndc, depth, 1.0);
|
||||
viewPos = p.xyz / p.w;
|
||||
sky = depth == pc.depthParams.y || !(abs(p.w) > 0.0);
|
||||
}
|
||||
else
|
||||
{
|
||||
const vec4 p = pc.clipToView * vec4(ndc, 0.5, 1.0);
|
||||
const vec3 ray = p.xyz / p.w;
|
||||
const float z = depth * pc.depthParams.x;
|
||||
viewPos = ray * (z / ray.z);
|
||||
sky = !(depth > 0.0) || depth >= pc.depthParams.z;
|
||||
}
|
||||
|
||||
const vec3 color = safe(texelFetch(inColor, ivec2(pc.colorDepthBase.xy) + px, 0).rgb);
|
||||
const vec3 base = (flags & kBeforeParticles) != 0u ? safe(texelFetch(inBeforeParticles, px, 0).rgb) : color;
|
||||
// Particles (colour after minus before) stay out of the denoiser; signed, so smoke can darken.
|
||||
const vec3 particles = color - base;
|
||||
|
||||
vec3 diffuseAlbedo = clamp(texelFetch(inDiffuseAlbedo, px, 0).rgb, 0.0, 1.0);
|
||||
vec3 specularAlbedo = clamp(texelFetch(inSpecularAlbedo, px, 0).rgb, 0.0, 1.0);
|
||||
// Energy above 1 is taken from the diffuse lobe first (reference: albedo overshoot clamp).
|
||||
diffuseAlbedo = max(diffuseAlbedo - max(diffuseAlbedo + specularAlbedo - 1.0, 0.0), 0.0);
|
||||
const float albedoSum = dot(diffuseAlbedo + specularAlbedo, vec3(1.0));
|
||||
sky = sky || !(albedoSum > 3e-3) || any(isnan(viewPos)) || any(isinf(viewPos));
|
||||
const vec4 normalTexel = texelFetch(inNormals, px, 0);
|
||||
vec3 normal = (flags & kNormalsUnorm) != 0u ? normalTexel.xyz * 2.0 - 1.0 : normalTexel.xyz;
|
||||
const float normalLength = dot(normal, normal);
|
||||
// A zero/invalid normal is no surface, not an invented world-Z-facing mirror. Alan Wake 2
|
||||
// supplies zero normals for its sky with finite FP16-max depth, so the far bound alone misses it.
|
||||
if ((flags & kInvalidNormalPassthrough) != 0u)
|
||||
sky = sky || !(normalLength > 1e-8) || isnan(normalLength) || isinf(normalLength);
|
||||
|
||||
if (sky)
|
||||
{
|
||||
// Outside linearDepthBounds: the denoiser passes these texels through, and the residual carries
|
||||
// the whole colour.
|
||||
imageStore(outLinearDepth, px, vec4(pc.depthParams.z * 10.0 * pc.depthParams.x));
|
||||
imageStore(outMotion, px, vec4(0.0));
|
||||
imageStore(outNormals, px, vec4(0.5, 0.5, 1.0, 0.0));
|
||||
imageStore(outDiffuseAlbedo, px, vec4(0.0));
|
||||
imageStore(outSpecularAlbedo, px, vec4(0.0));
|
||||
imageStore(outDiffuse, px, vec4(0.0));
|
||||
imageStore(outSpecular, px, vec4(0.0));
|
||||
imageStore(outResidual, px, vec4(color, 0.0));
|
||||
return;
|
||||
}
|
||||
imageStore(outLinearDepth, px, vec4(viewPos.z));
|
||||
|
||||
// 2.5D motion: DLSS motion vectors point from this pixel to where it was (previous - current), in
|
||||
// the MV texture's pixels. The depth delta is the camera's: the previous view's z of this point
|
||||
// (moving geometry is not reprojected; there is no previous depth for it).
|
||||
const ivec2 mvTexel = ivec2(pc.motionBaseFlags.xy) + ivec2(vec2(px) * pc.motion.zw);
|
||||
const vec2 mv = texelFetch(inMotion, mvTexel, 0).xy * pc.motion.xy;
|
||||
const vec4 previous = pc.viewToPreviousView * vec4(viewPos, 1.0);
|
||||
imageStore(outMotion, px, vec4(mv, previous.z / previous.w - viewPos.z, 0.0));
|
||||
|
||||
normal = dot(normal, normal) > 1e-8 ? normalize(normal) : vec3(0.0, 0.0, 1.0);
|
||||
const float roughness = clamp((flags & kRoughnessPacked) != 0u ? normalTexel.w : texelFetch(inRoughness, px, 0).r,
|
||||
0.0, 1.0);
|
||||
// material type 0 (the default material) everywhere
|
||||
imageStore(outNormals, px, vec4(octahedron(normal), roughness, 0.0));
|
||||
imageStore(outDiffuseAlbedo, px, vec4(diffuseAlbedo, 0.0));
|
||||
imageStore(outSpecularAlbedo, px, vec4(specularAlbedo, 0.0));
|
||||
|
||||
// Split by albedo and demodulate (reference "Mode 2"). Per channel, so a tinted metal keeps its tint.
|
||||
const vec3 total = diffuseAlbedo + specularAlbedo;
|
||||
const vec3 specularShare = specularAlbedo / max(total, vec3(1e-4));
|
||||
const vec3 specularColor = base * specularShare;
|
||||
const vec3 diffuseColor = base - specularColor;
|
||||
// A dark albedo would turn light it did not shape (fog, sky light, the background at a thin object's edge) into
|
||||
// a huge signal that the denoiser clips (fog lost) or spreads (bright fringes): no albedo below the floor
|
||||
// divides a signal, and what the division cannot carry stays in the residual.
|
||||
vec3 diffuse = safe(diffuseColor / max(diffuseAlbedo, vec3(pc.detail.y)));
|
||||
vec3 specular = safe(specularColor / max(specularAlbedo, vec3(pc.detail.y)));
|
||||
if (pc.detail.x > 0.0)
|
||||
{
|
||||
// Fine detail the denoiser treats as noise (hair, water ripples): part of the colour goes around it,
|
||||
// through the residual.
|
||||
const float denoised = 1.0 - clamp(pc.detail.x, 0.0, 1.0);
|
||||
diffuse *= denoised;
|
||||
specular *= denoised;
|
||||
}
|
||||
const vec3 remodulated = diffuse * diffuseAlbedo + specular * specularAlbedo;
|
||||
|
||||
// A: the ray hit distance for an indirect signal (ignored, but non-negative, for a direct one)
|
||||
float specularHit = 0.0, diffuseHit = 0.0;
|
||||
if ((flags & kSpecularHit) != 0u)
|
||||
{
|
||||
const vec4 hit = texelFetch(inSpecularHit, px, 0);
|
||||
specularHit = clamp((flags & kSpecularHitInW) != 0u ? hit.w : hit.x, 0.0, kHalfMax);
|
||||
}
|
||||
if ((flags & kDiffuseHit) != 0u)
|
||||
{
|
||||
const vec4 hit = texelFetch(inDiffuseHit, px, 0);
|
||||
diffuseHit = clamp((flags & kDiffuseHitInW) != 0u ? hit.w : hit.x, 0.0, kHalfMax);
|
||||
}
|
||||
imageStore(outDiffuse, px, vec4(diffuse, diffuseHit));
|
||||
imageStore(outSpecular, px, vec4(specular, specularHit));
|
||||
imageStore(outResidual, px, vec4(base - remodulated + particles, 0.0));
|
||||
}
|
||||
@@ -62,3 +62,11 @@ void d4r_motion_dilate(D4rMotionDilationParams p)
|
||||
}
|
||||
*(uint32_t*)(output + (uint64_t)y * p.motion_pitch + 4 * x) = value;
|
||||
}
|
||||
|
||||
#include "motion_dilate_lds.hip"
|
||||
|
||||
#include "native_motion_compose.hip"
|
||||
|
||||
#include "wire_history.hip"
|
||||
|
||||
#include "native_motion_prepared.hip"
|
||||
@@ -0,0 +1,164 @@
|
||||
// Exact low-res, unjittered specialization. Compile as wave32 only.
|
||||
#include "../../tools/d4r_motion_dilation.h"
|
||||
#pragma clang fp contract(off)
|
||||
#define D4R_LDS_SYNC() do{__builtin_amdgcn_fence(__ATOMIC_RELEASE,"workgroup");__builtin_amdgcn_s_barrier();__builtin_amdgcn_fence(__ATOMIC_ACQUIRE,"workgroup");}while(0)
|
||||
#define LDS __attribute__((shared))
|
||||
__attribute__((device)) static inline int clampi(int x, int n) { return x < 0 ? 0 : x >= n ? n - 1 : x; }
|
||||
template<bool ThinSky>
|
||||
__attribute__((device, always_inline)) static inline
|
||||
void motion_dilate_lds(D4rMotionDilationParams p, uint32_t allocation_height)
|
||||
{
|
||||
LDS float tile[32][33];
|
||||
LDS unsigned rowmask[32];
|
||||
LDS unsigned anyfg;
|
||||
LDS uint64_t extended[ThinSky ? 38 : 1];
|
||||
const int t = __builtin_amdgcn_workitem_id_x();
|
||||
const int tx = t & 15, ty = t >> 4;
|
||||
const int ox = __builtin_amdgcn_workgroup_id_x() * 16 - 8, oy = __builtin_amdgcn_workgroup_id_y() * 16 - 8;
|
||||
const int W = p.render_width, H = p.render_height;
|
||||
const bool rev = p.reversed_depth;
|
||||
const uint8_t* depth = (const uint8_t*)p.depth;
|
||||
const uint8_t* motion = (const uint8_t*)p.motion;
|
||||
uint8_t* output = (uint8_t*)p.output;
|
||||
// Extra tiles and inactive lanes copy allocation padding, including pitch bytes.
|
||||
// The bridge launches over pitch/4 by allocation_height, not the active grid.
|
||||
const int x = ox + 8 + tx, y = oy + 8 + ty;
|
||||
const bool allocated = (uint64_t)x < p.motion_pitch / 4 && y < (int)allocation_height;
|
||||
if (allocated && (x >= W || y >= H))
|
||||
*(uint32_t*)(output + (uint64_t)y * p.motion_pitch + 4 * x) =
|
||||
*(const uint32_t*)(motion + (uint64_t)y * p.motion_pitch + 4 * x);
|
||||
// This exit is uniform across the workgroup; partial tiles reach both barriers.
|
||||
if (ox + 8 >= W || oy + 8 >= H) return;
|
||||
if (t == 0) anyfg = 0;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 2; ++j)
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 2; ++i)
|
||||
{
|
||||
const int ly = ty + 16 * j, lx = tx + 16 * i;
|
||||
tile[ly][lx] = *(const float*)(depth + (uint64_t)clampi(oy + ly, H) * p.depth_pitch + 4 * clampi(ox + lx, W));
|
||||
}
|
||||
D4R_LDS_SYNC();
|
||||
if constexpr (ThinSky)
|
||||
{
|
||||
// Source candidates reach the edge of the 32x32 tile. Three extra
|
||||
// foreground bits on each side prove whether a candidate belongs to a
|
||||
// run of four, without truncating runs at a workgroup boundary.
|
||||
const int lane = t & 31, wave = t >> 5;
|
||||
for (int r = wave; r < 38; r += 8)
|
||||
{
|
||||
const int row = r - 3, gy = oy + row;
|
||||
bool fg[2];
|
||||
#pragma unroll
|
||||
for (int part = 0; part < 2; ++part)
|
||||
{
|
||||
const int col = lane - 3 + part * 32, gx = ox + col;
|
||||
const bool valid = (part == 0 || lane < 6) && gx >= 0 && gx < W && gy >= 0 && gy < H;
|
||||
float d = 0;
|
||||
if (valid)
|
||||
d = row >= 0 && row < 32 && col >= 0 && col < 32 ? tile[row][col] :
|
||||
*(const float*)(depth + (uint64_t)gy * p.depth_pitch + 4 * gx);
|
||||
fg[part] = valid && (rev ? d > 1.e-7f : d < 1.0f - 1.e-7f);
|
||||
}
|
||||
const unsigned low = __builtin_amdgcn_ballot_w32(fg[0]);
|
||||
const unsigned high = __builtin_amdgcn_ballot_w32(fg[1]);
|
||||
if (lane == 0) extended[r] = low | ((uint64_t)high << 32);
|
||||
}
|
||||
D4R_LDS_SYNC();
|
||||
#pragma unroll
|
||||
for (int r = 0; r < 4; ++r)
|
||||
if (lane == 0)
|
||||
{
|
||||
const int row = wave * 4 + r + 3;
|
||||
const uint64_t m = extended[row];
|
||||
const uint64_t heads = m & (m >> 1) & (m >> 2) & (m >> 3);
|
||||
const uint64_t horizontal = heads | (heads << 1) | (heads << 2) | (heads << 3);
|
||||
uint64_t vertical = 0;
|
||||
#pragma unroll
|
||||
for (int k = -3; k <= 0; ++k)
|
||||
vertical |= extended[row+k] & extended[row+k+1] & extended[row+k+2] & extended[row+k+3];
|
||||
// Preserve long wires/poles: short in either cross-section.
|
||||
const unsigned thin = (unsigned)((m & ~(horizontal & vertical)) >> 3);
|
||||
rowmask[row-3] = thin;
|
||||
if (thin) __atomic_fetch_or(&anyfg, 1u, __ATOMIC_RELAXED);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Each wave32 handles four rows, one ballot per row. Out-of-image halo
|
||||
// values are clamped for the opaque scan but excluded from the sky masks.
|
||||
{
|
||||
const int lane = t & 31, wave = t >> 5;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < 4; ++r)
|
||||
{
|
||||
const int row = wave * 4 + r, gy = oy + row, gx = ox + lane;
|
||||
const float d = tile[row][lane];
|
||||
const bool fg = (rev ? d > 1.e-7f : d < 1.0f - 1.e-7f) && gy >= 0 && gy < H && gx >= 0 && gx < W;
|
||||
const unsigned m = __builtin_amdgcn_ballot_w32(fg);
|
||||
if (lane == 0) { rowmask[row] = m; if (m) __atomic_fetch_or(&anyfg, 1u, __ATOMIC_RELAXED); }
|
||||
}
|
||||
}
|
||||
}
|
||||
D4R_LDS_SYNC();
|
||||
if (x >= W || y >= H) return;
|
||||
const int lx = tx + 8, ly = ty + 8;
|
||||
const float current = tile[ly][lx];
|
||||
float best = current;
|
||||
int bx = x, by = y;
|
||||
const int R = (int)p.radius;
|
||||
#pragma unroll
|
||||
for (int dy = -2; dy <= 2; ++dy)
|
||||
#pragma unroll
|
||||
for (int dx = -2; dx <= 2; ++dx)
|
||||
{
|
||||
if (dy < -R || dy > R || dx < -R || dx > R) continue;
|
||||
const float c = tile[ly + dy][lx + dx];
|
||||
if (rev ? c > best : c < best)
|
||||
best = c, bx = x + dx, by = y + dy;
|
||||
}
|
||||
bx = clampi(bx, W); by = clampi(by, H);
|
||||
const bool sky = rev ? current <= 1.e-7f : current >= 1.0f - 1.e-7f;
|
||||
bool sky_filled = false;
|
||||
if (sky && anyfg)
|
||||
{
|
||||
const int wd[9] = {8, 7, 7, 7, 6, 6, 5, 3, 0};
|
||||
int bd = 65, fx = 0, fy = 0;
|
||||
#pragma unroll
|
||||
for (int dy = -8; dy <= 8; ++dy)
|
||||
{
|
||||
const int w = wd[dy < 0 ? -dy : dy];
|
||||
unsigned m = rowmask[ly + dy];
|
||||
const int lo = lx - w, hi = lx + w;
|
||||
m &= (hi >= 31 ? 0xffffffffu : ((1u << (hi + 1)) - 1u)) & (lo <= 0 ? 0xffffffffu : ~((1u << lo) - 1u));
|
||||
if (!m) continue;
|
||||
const unsigned left = m & ((2u << lx) - 1u), right = m & ~((1u << lx) - 1u);
|
||||
int cdx = 99;
|
||||
if (left) cdx = (31 - __builtin_clz(left)) - lx;
|
||||
if (right) { const int b = __builtin_ctz(right) - lx; if (cdx == 99 || b < -cdx) cdx = b; }
|
||||
const int d2 = cdx * cdx + dy * dy, nx = x + cdx;
|
||||
if (d2 < bd || (d2 == bd && nx < fx)) bd = d2, fx = nx, fy = y + dy;
|
||||
}
|
||||
if (bd <= 64) bx = fx, by = fy, sky_filled = true;
|
||||
}
|
||||
uint32_t value = *(const uint32_t*)(motion + (uint64_t)y * p.motion_pitch + 4 * x);
|
||||
const bool nearer = rev ? best > current * 1.02f + 1.e-7f : best < current * 0.98f - 1.e-7f;
|
||||
if (sky_filled || nearer)
|
||||
value = *(const uint32_t*)(motion + (uint64_t)by * p.motion_pitch + 4 * bx);
|
||||
*(uint32_t*)(output + (uint64_t)y * p.motion_pitch + 4 * x) = value;
|
||||
}
|
||||
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256, 256)))
|
||||
void d4r_motion_dilate_lds(D4rMotionDilationParams p, uint32_t allocation_height)
|
||||
{
|
||||
motion_dilate_lds<false>(p, allocation_height);
|
||||
}
|
||||
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256, 256)))
|
||||
void d4r_motion_dilate_lds_thin(D4rMotionDilationParams p, uint32_t allocation_height)
|
||||
{
|
||||
motion_dilate_lds<true>(p, allocation_height);
|
||||
}
|
||||
|
||||
#undef LDS
|
||||
#undef D4R_LDS_SYNC
|
||||
@@ -0,0 +1,232 @@
|
||||
// Complete AW2 native geometry + camera composition and output-grid dilation.
|
||||
// Private prototype; input buffers are immutable and no NGX routing is changed.
|
||||
#include "../../tools/d4r_native_motion.h"
|
||||
#pragma clang fp contract(off)
|
||||
__attribute__((device,always_inline)) static inline float d4r_native_project(const float* m,int c,float x,float y,float z)
|
||||
{
|
||||
float value=m[c]*x;
|
||||
value=__builtin_fmaf(m[4+c],y,value);
|
||||
value=__builtin_fmaf(m[8+c],z,value);
|
||||
return value+m[12+c];
|
||||
}
|
||||
__attribute__((device,always_inline)) static inline void d4r_native_motion_finish(const D4rNativeMotionParams& p,
|
||||
uint32_t x,uint32_t y,uint32_t bx,uint32_t by,float best)
|
||||
{
|
||||
const uint32_t geometry=*(const uint32_t*)((const uint8_t*)p.geometry+(uint64_t)by*p.geometry_pitch+4*bx);
|
||||
const uint16_t lo=geometry,hi=geometry>>16;
|
||||
const bool lo_nan=(lo&0x7c00)==0x7c00 && (lo&0x3ff);
|
||||
const bool hi_nan=(hi&0x7c00)==0x7c00 && (hi&0x3ff);
|
||||
uint32_t result=geometry;
|
||||
if(geometry==UINT32_C(0xfbff7bff) || (lo_nan && hi_nan))
|
||||
{
|
||||
const float* scene=(const float*)p.scene;
|
||||
const float* matrix=(const float*)p.matrix;
|
||||
const float px=(bx+.5f)*((float)p.render_width/p.width);
|
||||
const float py=(by+.5f)*((float)p.render_height/p.height);
|
||||
const float cx=d4r_native_project(matrix,0,px,py,best),cy=d4r_native_project(matrix,1,px,py,best);
|
||||
const float cw=d4r_native_project(matrix,3,px,py,best);
|
||||
const float reciprocal=cw!=0.f ? 1.f/cw : 0.f;
|
||||
const _Float16 mx=(_Float16)(cx*reciprocal-scene[6]*px);
|
||||
const _Float16 my=(_Float16)(cy*reciprocal-scene[7]*py);
|
||||
result=(uint32_t)__builtin_bit_cast(uint16_t,mx) | (uint32_t)__builtin_bit_cast(uint16_t,my)<<16;
|
||||
}
|
||||
*(uint32_t*)((uint8_t*)p.output+(uint64_t)y*p.output_pitch+4*x)=result;
|
||||
}
|
||||
|
||||
__attribute__((device,always_inline)) static inline void d4r_native_motion_compose_impl(D4rNativeMotionParams p,float jitter_x=0,float jitter_y=0)
|
||||
{
|
||||
const uint32_t t=__builtin_amdgcn_workitem_id_x();
|
||||
const uint32_t x=__builtin_amdgcn_workgroup_id_x()*16+(t&15);
|
||||
const uint32_t y=__builtin_amdgcn_workgroup_id_y()*16+(t>>4);
|
||||
if(x>=p.width || y>=p.height) return;
|
||||
const uint8_t* depth=(const uint8_t*)p.depth;
|
||||
const int cx=clamp_index((int)__builtin_floorf(x+.5f+jitter_x*p.width/p.render_width),p.width);
|
||||
const int cy=clamp_index((int)__builtin_floorf(y+.5f+jitter_y*p.height/p.render_height),p.height);
|
||||
uint32_t bx=cx,by=cy;
|
||||
float best=*(const float*)(depth+(uint64_t)cy*p.depth_pitch+4*cx);
|
||||
// Same center-preserving, row-major strict-nearer tie rule as the CPU reference.
|
||||
for(int dy=-1;dy<=1;dy++) for(int dx=-1;dx<=1;dx++)
|
||||
{
|
||||
const int nx=cx+dx,ny=cy+dy;
|
||||
if(nx<0 || ny<0 || nx>=(int)p.width || ny>=(int)p.height) continue;
|
||||
const float z=*(const float*)(depth+(uint64_t)ny*p.depth_pitch+4*(uint32_t)nx);
|
||||
if(z<best) { best=z;bx=nx;by=ny; }
|
||||
}
|
||||
d4r_native_motion_finish(p,x,y,bx,by,best);
|
||||
}
|
||||
|
||||
// Thin-silhouette halo: shared depth plus row/column bit masks replace repeated
|
||||
// foreground run scans. Unusual footprints retain the direct depth fallback.
|
||||
#define D4R_LDS_SYNC() do{__builtin_amdgcn_fence(__ATOMIC_RELEASE,"workgroup");__builtin_amdgcn_s_barrier();__builtin_amdgcn_fence(__ATOMIC_ACQUIRE,"workgroup");}while(0)
|
||||
#define D4R_LDS __attribute__((shared))
|
||||
template<bool Hybrid=false>
|
||||
__attribute__((device,always_inline)) static inline void d4r_native_motion_compose_thin_impl(D4rNativeMotionParams p,float jitter_x,float jitter_y,uint64_t game_motion=0,uint64_t game_pitch=0)
|
||||
{
|
||||
// Align the tile to the jittered grid; halo and run checks reach at most ten pixels.
|
||||
constexpr int margin=Hybrid ? 11 : 10,span=16+2*margin;
|
||||
D4R_LDS float tile[span*span];
|
||||
D4R_LDS uint64_t candidate_row[span]; // one bit per eligible thin foreground texel
|
||||
D4R_LDS uint64_t foreground_row[span],foreground_column[span];
|
||||
D4R_LDS uint64_t wave_candidates[Hybrid ? 8 : 1];
|
||||
const uint32_t t=__builtin_amdgcn_workitem_id_x();
|
||||
const int x0=(int)__builtin_amdgcn_workgroup_id_x()*16,y0=(int)__builtin_amdgcn_workgroup_id_y()*16;
|
||||
const int gx0=x0+(int)__builtin_floorf(.5f+jitter_x*p.width/p.render_width)-margin;
|
||||
const int gy0=y0+(int)__builtin_floorf(.5f+jitter_y*p.height/p.render_height)-margin;
|
||||
const uint8_t* depth=(const uint8_t*)p.depth;
|
||||
const float outside=__builtin_inff(); // behaves as a missing neighbour in every comparison below
|
||||
for(int i=t;i<span*span;i+=256)
|
||||
{
|
||||
const int gx=gx0+i%span,gy=gy0+i/span;
|
||||
const float z=gx>=0 && gy>=0 && gx<(int)p.width && gy<(int)p.height ?
|
||||
*(const float*)(depth+(uint64_t)gy*p.depth_pitch+4*(uint32_t)gx) : outside;
|
||||
tile[i]=z;
|
||||
}
|
||||
D4R_LDS_SYNC();
|
||||
const int lane=t&31,wave=t>>5;
|
||||
// Pack the binary foreground test once in both directions. A foreground
|
||||
// center is thick on an axis iff its nine-bit neighbourhood contains a
|
||||
// contiguous run of five (all five possible runs include the center).
|
||||
for(int row=wave;row<span;row+=8)
|
||||
{
|
||||
const unsigned r0=__builtin_amdgcn_ballot_w32(tile[row*span+lane]<.99999f);
|
||||
const unsigned c0=__builtin_amdgcn_ballot_w32(tile[lane*span+row]<.99999f);
|
||||
const unsigned r1=__builtin_amdgcn_ballot_w32(lane<span-32 && tile[row*span+32+lane]<.99999f);
|
||||
const unsigned c1=__builtin_amdgcn_ballot_w32(lane<span-32 && tile[(32+lane)*span+row]<.99999f);
|
||||
if(lane==0)
|
||||
{
|
||||
foreground_row[row]=(uint64_t)r0 | (uint64_t)r1<<32;
|
||||
foreground_column[row]=(uint64_t)c0 | (uint64_t)c1<<32;
|
||||
}
|
||||
}
|
||||
D4R_LDS_SYNC();
|
||||
uint64_t nearby_candidates=0;
|
||||
for(int row=wave;row<span;row+=8)
|
||||
{
|
||||
uint64_t bits=0;
|
||||
for(int part=0;part<2;part++)
|
||||
{
|
||||
const int col=lane+32*part;
|
||||
bool thin=false;
|
||||
if(col>=4 && col<span-4 && row>=4 && row<span-4 && (foreground_row[row]>>col&1))
|
||||
{
|
||||
const unsigned horizontal=(foreground_row[row]>>(col-4))&511;
|
||||
const unsigned vertical=(foreground_column[col]>>(row-4))&511;
|
||||
const bool thick_x=(horizontal & (horizontal>>1) & (horizontal>>2) & (horizontal>>3) & (horizontal>>4))!=0;
|
||||
const bool thick_y=(vertical & (vertical>>1) & (vertical>>2) & (vertical>>3) & (vertical>>4))!=0;
|
||||
thin=!thick_x || !thick_y;
|
||||
}
|
||||
const unsigned mask=__builtin_amdgcn_ballot_w32(thin);
|
||||
bits|=(uint64_t)mask<<(32*part);
|
||||
}
|
||||
if(lane==0) candidate_row[row]=bits;
|
||||
if constexpr(Hybrid) nearby_candidates|=bits;
|
||||
}
|
||||
if constexpr(Hybrid) if(lane==0) wave_candidates[wave]=nearby_candidates;
|
||||
D4R_LDS_SYNC();
|
||||
const uint32_t x=x0+(t&15),y=y0+(t>>4);
|
||||
if(x>=p.width || y>=p.height) return;
|
||||
auto depth_at=[&](int nx,int ny)->float
|
||||
{
|
||||
const int lx=nx-gx0,ly=ny-gy0;
|
||||
if((unsigned)lx<(unsigned)span && (unsigned)ly<(unsigned)span) return tile[ly*span+lx];
|
||||
if(nx<0 || ny<0 || nx>=(int)p.width || ny>=(int)p.height) return outside;
|
||||
return *(const float*)(depth+(uint64_t)ny*p.depth_pitch+4*(uint32_t)nx);
|
||||
};
|
||||
const int cx=clamp_index((int)__builtin_floorf(x+.5f+jitter_x*p.width/p.render_width),p.width);
|
||||
const int cy=clamp_index((int)__builtin_floorf(y+.5f+jitter_y*p.height/p.render_height),p.height);
|
||||
if constexpr(Hybrid)
|
||||
{
|
||||
uint64_t any=0;for(int i=0;i<8;i++) any|=wave_candidates[i];
|
||||
bool keep_native=false;
|
||||
const bool roi_fits=cx-gx0>=10 && cx-gx0<span-10 && cy-gy0>=10 && cy-gy0<span-10;
|
||||
if(!roi_fits) keep_native=true; // unusual clamped footprints keep the complete native path
|
||||
else if(any)
|
||||
{
|
||||
const uint64_t window=UINT64_C(8191)<<(cx-gx0-6);
|
||||
for(int dy=-6;dy<=6;dy++) keep_native|=(candidate_row[cy+dy-gy0]&window)!=0;
|
||||
}
|
||||
if(!keep_native)
|
||||
{
|
||||
const unsigned mx=(unsigned)((x+.5f)*p.render_width/p.width);
|
||||
const unsigned my=(unsigned)((y+.5f)*p.render_height/p.height);
|
||||
*(uint32_t*)((uint8_t*)p.output+(uint64_t)y*p.output_pitch+4*x)=
|
||||
*(const uint32_t*)(game_motion+(uint64_t)my*game_pitch+4*mx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
uint32_t bx=cx,by=cy;
|
||||
const float centre=depth_at(cx,cy);
|
||||
float best=centre;
|
||||
for(int dy=-1;dy<=1;dy++) for(int dx=-1;dx<=1;dx++)
|
||||
{
|
||||
const int nx=cx+dx,ny=cy+dy;
|
||||
if(nx<0 || ny<0 || nx>=(int)p.width || ny>=(int)p.height) continue;
|
||||
const float z=depth_at(nx,ny);
|
||||
if(z<best) { best=z;bx=nx;by=ny; }
|
||||
}
|
||||
// Cover the render filter footprint only around thin silhouettes against
|
||||
// sky. Long runs on both axes belong to thick surfaces and stay excluded.
|
||||
const bool tile_reaches=cx-gx0>=10 && cx-gx0<span-10 && cy-gy0>=10 && cy-gy0<span-10;
|
||||
if(centre>=.99999f)
|
||||
{
|
||||
const int sx=(p.width+p.render_width/2)/p.render_width;
|
||||
const int sy=(p.height+p.render_height/2)/p.render_height;
|
||||
const int radius=2*(sx>sy ? sx : sy)>6 ? 6 : 2*(sx>sy ? sx : sy);
|
||||
if(tile_reaches)
|
||||
{
|
||||
// Row-major, strictly nearer wins, exactly the scan below but visiting only candidate pixels.
|
||||
const int left=cx-radius-gx0;
|
||||
const uint64_t window=((uint64_t)1<<(2*radius+1))-1;
|
||||
for(int dy=-radius;dy<=radius;dy++)
|
||||
{
|
||||
const int row=cy+dy-gy0;
|
||||
uint64_t m=(candidate_row[row]>>left)&window;
|
||||
if(dy>=-1 && dy<=1) m&=~(((uint64_t)7)<<(radius-1));
|
||||
while(m)
|
||||
{
|
||||
const int bit=__builtin_ctzll(m);
|
||||
m&=m-1;
|
||||
const float z=tile[row*span+left+bit];
|
||||
if(z<best) { best=z;bx=cx-radius+bit;by=cy+dy; }
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
for(int dy=-radius;dy<=radius;dy++) for(int dx=-radius;dx<=radius;dx++)
|
||||
{
|
||||
if(dx>=-1 && dx<=1 && dy>=-1 && dy<=1) continue;
|
||||
const int nx=cx+dx,ny=cy+dy;
|
||||
if(nx<0 || ny<0 || nx>=(int)p.width || ny>=(int)p.height) continue;
|
||||
const float z=depth_at(nx,ny);
|
||||
if(!(z<best && z<.99999f)) continue;
|
||||
bool thin=false;
|
||||
for(int axis=0;axis<2 && !thin;axis++)
|
||||
{
|
||||
int run=1;
|
||||
for(int sign=-1;sign<=1;sign+=2)
|
||||
for(int step=1;step<5 && run<5;step++)
|
||||
{
|
||||
const int tx=nx+(axis==0 ? sign*step : 0),ty=ny+(axis==1 ? sign*step : 0);
|
||||
if(tx<0 || ty<0 || tx>=(int)p.width || ty>=(int)p.height ||
|
||||
!(depth_at(tx,ty)<.99999f)) break;
|
||||
run++;
|
||||
}
|
||||
thin=run<5;
|
||||
}
|
||||
if(thin) { best=z;bx=nx;by=ny; }
|
||||
}
|
||||
}
|
||||
d4r_native_motion_finish(p,x,y,bx,by,best);
|
||||
}
|
||||
|
||||
extern "C" __attribute__((global)) void d4r_native_motion_compose(D4rNativeMotionParams p)
|
||||
{ d4r_native_motion_compose_impl(p); }
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256, 256))) void d4r_native_motion_compose_thin(D4rNativeMotionThinParams p)
|
||||
{ d4r_native_motion_compose_thin_impl(p.base,p.jitter_x,p.jitter_y); }
|
||||
|
||||
// Alignment without the extra silhouette halo; wire-color repair is independent.
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256,256))) void d4r_native_motion_compose_aligned(D4rNativeMotionThinParams p)
|
||||
{ d4r_native_motion_compose_impl(p.base,p.jitter_x,p.jitter_y); }
|
||||
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256,256))) void d4r_native_motion_compose_hybrid(D4rNativeMotionHybridParams p)
|
||||
{ d4r_native_motion_compose_thin_impl<true>(p.native.base,p.native.jitter_x,p.native.jitter_y,p.game_motion,p.game_motion_pitch); }
|
||||
@@ -0,0 +1,133 @@
|
||||
// Exact native composition with shared wire geometry; wave64 compose, wave32 preparation.
|
||||
#pragma clang fp contract(off)
|
||||
#define NMP_SYNC() do { __builtin_amdgcn_fence(__ATOMIC_RELEASE,"workgroup"); __builtin_amdgcn_s_barrier(); __builtin_amdgcn_fence(__ATOMIC_ACQUIRE,"workgroup"); } while(0)
|
||||
__attribute__((device,always_inline,target("wavefrontsize64"))) static inline float d4r_nmp_project64(const float* m,int c,float x,float y,float z)
|
||||
{
|
||||
float value=m[c]*x;
|
||||
value=__builtin_fmaf(m[4+c],y,value);
|
||||
value=__builtin_fmaf(m[8+c],z,value);
|
||||
return value+m[12+c];
|
||||
}
|
||||
__attribute__((device,always_inline,target("wavefrontsize64"))) static inline void d4r_nmp_finish64(const D4rNativeMotionParams& p,
|
||||
uint32_t x,uint32_t y,uint32_t bx,uint32_t by,float best)
|
||||
{
|
||||
const uint32_t geometry=*(const uint32_t*)((const uint8_t*)p.geometry+(uint64_t)by*p.geometry_pitch+4*bx);
|
||||
const uint16_t lo=geometry,hi=geometry>>16;
|
||||
const bool lo_nan=(lo&0x7c00)==0x7c00 && (lo&0x3ff);
|
||||
const bool hi_nan=(hi&0x7c00)==0x7c00 && (hi&0x3ff);
|
||||
uint32_t result=geometry;
|
||||
if(geometry==UINT32_C(0xfbff7bff) || (lo_nan && hi_nan))
|
||||
{
|
||||
const float* scene=(const float*)p.scene;
|
||||
const float* matrix=(const float*)p.matrix;
|
||||
const float px=(bx+.5f)*((float)p.render_width/p.width);
|
||||
const float py=(by+.5f)*((float)p.render_height/p.height);
|
||||
const float cx=d4r_nmp_project64(matrix,0,px,py,best),cy=d4r_nmp_project64(matrix,1,px,py,best);
|
||||
const float cw=d4r_nmp_project64(matrix,3,px,py,best);
|
||||
const float reciprocal=cw!=0.f ? 1.f/cw : 0.f;
|
||||
const _Float16 mx=(_Float16)(cx*reciprocal-scene[6]*px);
|
||||
const _Float16 my=(_Float16)(cy*reciprocal-scene[7]*py);
|
||||
result=(uint32_t)__builtin_bit_cast(uint16_t,mx) | (uint32_t)__builtin_bit_cast(uint16_t,my)<<16;
|
||||
}
|
||||
*(uint32_t*)((uint8_t*)p.output+(uint64_t)y*p.output_pitch+4*x)=result;
|
||||
}
|
||||
|
||||
__attribute__((device,always_inline,target("wavefrontsize64"))) static inline unsigned d4r_nmp_bits64(const unsigned* data,int words,int left)
|
||||
{
|
||||
int word=left>=0 ? left/32 : -1,shift=left&31;
|
||||
unsigned lo=word>=0 && word<words ? data[word] : 0;
|
||||
unsigned hi=word+1>=0 && word+1<words ? data[word+1] : 0;
|
||||
return (unsigned)(((uint64_t)lo | (uint64_t)hi<<32)>>shift);
|
||||
}
|
||||
__attribute__((device,always_inline,target("wavefrontsize64"))) static inline int d4r_nmp_clamp64(int x,unsigned n) {return x<0 ? 0 : x>=(int)n ? (int)n-1 : x;}
|
||||
extern "C" __attribute__((global,amdgpu_flat_work_group_size(256,256),target("wavefrontsize64"))) void d4r_native_motion_prepared(D4rNativeMotionPreparedParams a)
|
||||
{
|
||||
const auto& p=a.native.base;
|
||||
unsigned t=__builtin_amdgcn_workitem_id_x();
|
||||
int x0=__builtin_amdgcn_workgroup_id_x()*16,y0=__builtin_amdgcn_workgroup_id_y()*16;
|
||||
int left=x0+(int)__builtin_floorf(.5f+a.native.jitter_x*p.width/p.render_width)-8;
|
||||
int top=y0+(int)__builtin_floorf(.5f+a.native.jitter_y*p.height/p.render_height)-8;
|
||||
const int rw=(p.width+31)/32;
|
||||
__attribute__((shared)) unsigned rows[32],any;
|
||||
if(t<32) {
|
||||
int y=top+t;unsigned bits=y>=0 && y<(int)p.height ? d4r_nmp_bits64((const unsigned*)a.thin_mask+(uint64_t)y*rw,rw,left) : 0;
|
||||
rows[t]=bits;unsigned nonempty=(unsigned)__builtin_amdgcn_ballot_w64(bits!=0);if(t==0) any=nonempty;
|
||||
}
|
||||
NMP_SYNC();
|
||||
unsigned x=x0+(t&15),y=y0+(t>>4);
|
||||
if(x>=p.width || y>=p.height) return;
|
||||
const int cx=d4r_nmp_clamp64((int)__builtin_floorf(x+.5f+a.native.jitter_x*p.width/p.render_width),p.width);
|
||||
const int cy=d4r_nmp_clamp64((int)__builtin_floorf(y+.5f+a.native.jitter_y*p.height/p.render_height),p.height);
|
||||
const uint8_t* depth=(const uint8_t*)p.depth;
|
||||
const float center=*(const float*)(depth+(uint64_t)cy*p.depth_pitch+4*cx);
|
||||
float best=center;unsigned bx=cx,by=cy;
|
||||
for(int dy=-1;dy<=1;dy++)for(int dx=-1;dx<=1;dx++) {
|
||||
int nx=cx+dx,ny=cy+dy;if(nx<0 || ny<0 || nx>=(int)p.width || ny>=(int)p.height)continue;
|
||||
float z=*(const float*)(depth+(uint64_t)ny*p.depth_pitch+4*nx);
|
||||
if(z<best){best=z;bx=nx;by=ny;}
|
||||
}
|
||||
if(center>=.99999f) {
|
||||
int sx=(p.width+p.render_width/2)/p.render_width,sy=(p.height+p.render_height/2)/p.render_height;
|
||||
int radius=2*(sx>sy ? sx : sy);if(radius>6)radius=6;
|
||||
bool cached=cx-radius>=left && cx+radius<left+32 && cy-radius>=top && cy+radius<top+32;
|
||||
if(any || !cached)for(int dy=-radius;dy<=radius;dy++) {
|
||||
int ny=cy+dy;if(ny<0 || ny>=(int)p.height)continue;
|
||||
unsigned m=(cached ? rows[ny-top]>>(cx-radius-left) : d4r_nmp_bits64((const unsigned*)a.thin_mask+(uint64_t)ny*rw,rw,cx-radius)) & ((1u<<(2*radius+1))-1);
|
||||
if(dy>=-1 && dy<=1)m&=~(7u<<(radius-1));
|
||||
while(m) {int bit=__builtin_ctz(m);m&=m-1;int nx=cx-radius+bit;
|
||||
float z=*(const float*)(depth+(uint64_t)ny*p.depth_pitch+4*nx);
|
||||
if(z<best){best=z;bx=nx;by=ny;}
|
||||
}
|
||||
}
|
||||
}
|
||||
d4r_nmp_finish64(p,x,y,bx,by,best);
|
||||
}
|
||||
|
||||
extern "C" __attribute__((global,amdgpu_flat_work_group_size(256,256))) void d4r_native_motion_prepare(D4rNativeMotionPreparedParams a)
|
||||
{
|
||||
const auto& p=a.native.base;
|
||||
__attribute__((shared)) float z[40][41];
|
||||
__attribute__((shared)) uint64_t row_bits[40],col_bits[40];
|
||||
unsigned t=__builtin_amdgcn_workitem_id_x(),lane=t&31,wave=t>>5;
|
||||
const int x0=(int)__builtin_amdgcn_workgroup_id_x()*32,y0=(int)__builtin_amdgcn_workgroup_id_y()*32;
|
||||
for(int i=t;i<1600;i+=256) {
|
||||
int ly=i/40,lx=i%40,gx=clamp_index(x0+lx-4,p.width),gy=clamp_index(y0+ly-4,p.height);
|
||||
z[ly][lx]=*(const float*)(p.depth+(uint64_t)gy*p.depth_pitch+4*gx);
|
||||
}
|
||||
NMP_SYNC();
|
||||
for(int r=wave;r<40;r+=8) {
|
||||
uint64_t rb=0,cb=0;
|
||||
for(int part=0;part<2;part++) {
|
||||
int c=lane+32*part;
|
||||
bool rf=c<40 && x0+c-4>=0 && x0+c-4<(int)p.width && y0+r-4>=0 && y0+r-4<(int)p.height && z[r][c]<.99999f;
|
||||
bool cf=c<40 && y0+c-4>=0 && y0+c-4<(int)p.height && x0+r-4>=0 && x0+r-4<(int)p.width && z[c][r]<.99999f;
|
||||
rb|=(uint64_t)__builtin_amdgcn_ballot_w32(rf)<<(part*32);cb|=(uint64_t)__builtin_amdgcn_ballot_w32(cf)<<(part*32);
|
||||
}
|
||||
if(lane==0){row_bits[r]=rb;col_bits[r]=cb;}
|
||||
}
|
||||
NMP_SYNC();
|
||||
D4rWireHistoryParams wp={};wp.width=p.width;wp.height=p.height;wp.native_depth=p.depth;wp.depth_pitch=p.depth_pitch;wp.reserved=a.wire_flags;
|
||||
auto depth=[&](int gx,int gy)->float {return z[gy-y0+4][gx-x0+4];};
|
||||
for(int r=wave;r<32;r+=8) {
|
||||
const int gx=x0+lane,gy=y0+r,lx=lane+4,ly=r+4;
|
||||
unsigned h=(row_bits[ly]>>lane)&511,v=(col_bits[lx]>>r)&511;
|
||||
bool hx=(h&(h>>1)&(h>>2)&(h>>3)&(h>>4))!=0,hy=(v&(v>>1)&(v>>2)&(v>>3)&(v>>4))!=0;
|
||||
bool thin=(h&16)!=0 && (!hx || !hy);
|
||||
unsigned bits=__builtin_amdgcn_ballot_w32(thin);
|
||||
if(lane==0 && gy<(int)p.height) ((unsigned*)a.thin_mask)[(uint64_t)gy*((p.width+31)/32)+x0/32]=bits;
|
||||
if(a.wire_axes && gx<(int)p.width && gy<(int)p.height) {
|
||||
// Existing foreground bits reject sky centers and non-sky endpoints
|
||||
// without repeating LDS depth reads. The reference still handles NaNs,
|
||||
// edge clamping and relative-depth separation for every surviving axis.
|
||||
unsigned axes=0;
|
||||
if(h&16) {
|
||||
const bool sky_only=(a.wire_flags & D4R_WIRE_HISTORY_SKY_ONLY)!=0;
|
||||
if(!sky_only || !(h&257)) axes|=wh_thin_interior(wp,gx,gy,0,depth)?1u:0u;
|
||||
if(!sky_only || !(v&257)) axes|=wh_thin_interior(wp,gx,gy,1,depth)?2u:0u;
|
||||
}
|
||||
((uint8_t*)a.wire_axes)[(uint64_t)gy*p.width+gx]=axes;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#undef NMP_SYNC
|
||||
@@ -0,0 +1,248 @@
|
||||
// Geometry-confirmed short-lived color history for locally separated thin silhouettes.
|
||||
#include "../../tools/d4r_wire_history.h"
|
||||
#pragma clang fp contract(off)
|
||||
#define WH_SYNC() do { __builtin_amdgcn_fence(__ATOMIC_RELEASE,"workgroup"); __builtin_amdgcn_s_barrier(); __builtin_amdgcn_fence(__ATOMIC_ACQUIRE,"workgroup"); } while(0)
|
||||
__attribute__((device,always_inline)) static inline int wh_clamp(int v,int lo,int hi)
|
||||
{ return v<lo ? lo : v>hi ? hi : v; }
|
||||
__attribute__((device,always_inline)) static inline float wh_depth(const D4rWireHistoryParams& p,int x,int y)
|
||||
{ x=wh_clamp(x,0,p.width-1);y=wh_clamp(y,0,p.height-1);return *(const float*)(p.native_depth+(uint64_t)y*p.depth_pitch+4*x); }
|
||||
template<class Depth>
|
||||
__attribute__((device,always_inline)) static inline bool wh_thin_axis(const D4rWireHistoryParams& p,int x,int y,int axis,Depth depth)
|
||||
{
|
||||
float center=depth(x,y);if(!(center>=0 && center<.99999f)) return false;
|
||||
if((p.reserved & D4R_WIRE_HISTORY_SKY_ONLY) &&
|
||||
(!(depth(x-4*(axis==0),y-4*(axis==1))>=.99999f) || !(depth(x+4*(axis==0),y+4*(axis==1))>=.99999f))) return false;
|
||||
float epsilon=.25f*(1-center);if(epsilon<2e-6f) epsilon=2e-6f;
|
||||
float limit=center+epsilon;
|
||||
int run=1;
|
||||
for(int sign=-1;sign<=1;sign+=2) for(int step=1;step<5 && run<5;step++)
|
||||
{
|
||||
int tx=x+(axis==0 ? sign*step : 0),ty=y+(axis==1 ? sign*step : 0);
|
||||
if(tx<0 || ty<0 || tx>=(int)p.width || ty>=(int)p.height || !(depth(tx,ty)<limit)) break;
|
||||
run++;
|
||||
}
|
||||
if(run>=5) return false;
|
||||
float gap=.5f*(1-center);if(gap<2e-6f) gap=2e-6f;
|
||||
bool sides[2]={false,false};
|
||||
for(int sign=-1;sign<=1;sign+=2) for(int step=1;step<=4;step++)
|
||||
sides[(sign+1)/2]|=depth(x+sign*step*(axis==0),y+sign*step*(axis==1))>center+gap;
|
||||
return sides[0] && sides[1];
|
||||
}
|
||||
// Interior specialization: fixed eight depth loads, bit scans for the first
|
||||
// non-near neighbour on each side, and the original strict separation test.
|
||||
// Edge samples keep the bounds-aware reference above.
|
||||
template<class Depth>
|
||||
__attribute__((device,always_inline)) static inline bool wh_thin_interior(const D4rWireHistoryParams& p,int x,int y,int axis,Depth depth)
|
||||
{
|
||||
if(x<4 || y<4 || x>=(int)p.width-4 || y>=(int)p.height-4)
|
||||
return wh_thin_axis(p,x,y,axis,depth);
|
||||
const float center=depth(x,y);
|
||||
if(!(center>=0 && center<.99999f)) return false;
|
||||
if((p.reserved & D4R_WIRE_HISTORY_SKY_ONLY) &&
|
||||
(!(depth(x-4*(axis==0),y-4*(axis==1))>=.99999f) || !(depth(x+4*(axis==0),y+4*(axis==1))>=.99999f))) return false;
|
||||
float epsilon=.25f*(1-center);if(epsilon<2e-6f) epsilon=2e-6f;
|
||||
float gap=.5f*(1-center);if(gap<2e-6f) gap=2e-6f;
|
||||
const float limit=center+epsilon,far_limit=center+gap;
|
||||
unsigned negative=0,positive=0;
|
||||
bool far_negative=false,far_positive=false;
|
||||
#pragma unroll
|
||||
for(int step=1;step<=4;step++)
|
||||
{
|
||||
const float n=depth(x-(axis==0 ? step : 0),y-(axis==1 ? step : 0));
|
||||
negative|=(unsigned)(!(n<limit))<<(step-1);
|
||||
far_negative|=n>far_limit;
|
||||
}
|
||||
// Four contiguous near neighbours already prove a run of at least five.
|
||||
// No positive-side depths or separation checks can make it thin.
|
||||
if(!negative) return false;
|
||||
#pragma unroll
|
||||
for(int step=1;step<=4;step++)
|
||||
{
|
||||
const float v=depth(x+(axis==0 ? step : 0),y+(axis==1 ? step : 0));
|
||||
positive|=(unsigned)(!(v<limit))<<(step-1);
|
||||
far_positive|=v>far_limit;
|
||||
}
|
||||
if(!positive) return false;
|
||||
return __builtin_ctz(negative|16)+__builtin_ctz(positive|16)<4 && far_negative && far_positive;
|
||||
}
|
||||
__attribute__((device,always_inline)) static inline float wh_luma(uint64_t rgb,uint64_t pitch,int x,int y)
|
||||
{
|
||||
const _Float16* c=(const _Float16*)(rgb+(uint64_t)y*pitch+8*x);
|
||||
float l=(float)c[0]*.2126f+(float)c[1]*.7152f+(float)c[2]*.0722f;
|
||||
return l>1e-5f ? l : 1e-5f;
|
||||
}
|
||||
// Median in log space is the geometric mean of the two central luminances.
|
||||
__attribute__((device,always_inline)) static inline float wh_background(uint64_t rgb,uint64_t pitch,int x,int y,int ax,int ay,bool input)
|
||||
{
|
||||
float v[6];int count=input ? 4 : 6;
|
||||
const int offsets[6]={-12,-10,-8,8,10,12};
|
||||
const int small[4]={-5,-4,4,5};
|
||||
for(int i=0;i<count;i++) { int d=input ? small[i] : offsets[i];v[i]=wh_luma(rgb,pitch,x+d*ax,y+d*ay); }
|
||||
for(int i=1;i<count;i++) for(int j=i;j>0 && v[j]<v[j-1];j--) {float a=v[j];v[j]=v[j-1];v[j-1]=a;}
|
||||
return __builtin_sqrtf(v[count/2-1]*v[count/2]);
|
||||
}
|
||||
template<bool Sparse,bool CachedAxes=false>
|
||||
__attribute__((device,always_inline)) static inline void wh_run(D4rWireHistoryParams p,uint64_t patch_mask,uint64_t prepared_axes=0)
|
||||
{
|
||||
uint32_t t=__builtin_amdgcn_workitem_id_x();
|
||||
int x=__builtin_amdgcn_workgroup_id_x()*16+(t&15),y=__builtin_amdgcn_workgroup_id_y()*16+(t>>4);
|
||||
// Classify each nearby depth texel once, shared by the four bilinear samples
|
||||
// of adjacent output pixels. All lanes reach both barriers, including partial tiles.
|
||||
constexpr int span=26,margin=5;
|
||||
__attribute__((shared)) float tile[span*span];
|
||||
__attribute__((shared)) uint8_t axes[span*span];
|
||||
const int ox=(int)__builtin_amdgcn_workgroup_id_x()*16+(int)__builtin_floorf(p.jitter_x*p.width/p.render_width)-margin;
|
||||
const int oy=(int)__builtin_amdgcn_workgroup_id_y()*16+(int)__builtin_floorf(p.jitter_y*p.height/p.render_height)-margin;
|
||||
if constexpr(!CachedAxes) {
|
||||
for(int i=t;i<span*span;i+=256) tile[i]=wh_depth(p,ox+i%span,oy+i/span);
|
||||
WH_SYNC();
|
||||
}
|
||||
auto depth=[&](int tx,int ty)->float {
|
||||
int lx=tx-ox,ly=ty-oy;
|
||||
return (unsigned)lx<span && (unsigned)ly<span ? tile[ly*span+lx] : wh_depth(p,tx,ty);
|
||||
};
|
||||
auto tile_depth=[&](int tx,int ty)->float { return tile[(ty-oy)*span+tx-ox]; };
|
||||
if constexpr(!CachedAxes) {
|
||||
for(int i=t;i<span*span;i+=256)
|
||||
{
|
||||
int lx=i%span,ly=i/span;
|
||||
uint8_t mask=0;
|
||||
if(lx>=4 && lx<span-4 && ly>=4 && ly<span-4)
|
||||
{
|
||||
if(wh_thin_interior(p,ox+lx,oy+ly,0,tile_depth)) mask|=1;
|
||||
if(wh_thin_interior(p,ox+lx,oy+ly,1,tile_depth)) mask|=2;
|
||||
}
|
||||
axes[i]=mask;
|
||||
}
|
||||
WH_SYNC();
|
||||
}
|
||||
auto thin_axes=[&](int tx,int ty)->uint8_t {
|
||||
if constexpr(CachedAxes) return *(const uint8_t*)(prepared_axes+(uint64_t)ty*p.width+tx);
|
||||
else {
|
||||
int lx=tx-ox,ly=ty-oy;
|
||||
if(lx>=4 && lx<span-4 && ly>=4 && ly<span-4) return axes[ly*span+lx];
|
||||
return (wh_thin_axis(p,tx,ty,0,depth) ? 1 : 0) | (wh_thin_axis(p,tx,ty,1,depth) ? 2 : 0);
|
||||
}
|
||||
};
|
||||
if(x>=(int)p.width || y>=(int)p.height) return;
|
||||
uint64_t offset=(uint64_t)y*p.output_pitch+8*x,index=(uint64_t)y*p.width+x;
|
||||
if constexpr(Sparse) *(uint8_t*)(patch_mask+index)=0;
|
||||
else *(uint64_t*)(p.output+offset)=*(const uint64_t*)(p.current+offset);
|
||||
// Age 255 makes this entry unreadable by the next frame. The prepared
|
||||
// path need not copy raw RGB or initialize depth for an unobserved entry.
|
||||
if constexpr(!CachedAxes) *(uint64_t*)(p.output_history+offset)=*(const uint64_t*)(p.current+offset);
|
||||
*(uint8_t*)(p.output_age+index)=255;
|
||||
if constexpr(!CachedAxes) *(float*)(p.output_depth+4*index)=1.f;
|
||||
if(x<16 || y<16 || x>=(int)p.width-16 || y>=(int)p.height-16) return;
|
||||
float nx=x+p.jitter_x*p.width/p.render_width,ny=y+p.jitter_y*p.height/p.render_height;
|
||||
int fx=(int)__builtin_floorf(nx),fy=(int)__builtin_floorf(ny),sx=0,sy=0;
|
||||
float coverage=0,best_weight=-1;
|
||||
for(int dy=0;dy<=1;dy++) for(int dx=0;dx<=1;dx++)
|
||||
{
|
||||
int tx=wh_clamp(fx+dx,0,p.width-1),ty=wh_clamp(fy+dy,0,p.height-1);
|
||||
if(!thin_axes(tx,ty)) continue;
|
||||
float weight=(1-__builtin_fabsf(nx-tx))*(1-__builtin_fabsf(ny-ty));coverage+=weight;
|
||||
if(weight>best_weight) { best_weight=weight;sx=tx;sy=ty; }
|
||||
}
|
||||
if(coverage<=.05f) return;
|
||||
int ay=(thin_axes(sx,sy)&2) ? 1 : 0,ax=1-ay;
|
||||
const bool conservative=(p.reserved & D4R_WIRE_HISTORY_CONSERVATIVE)!=0;
|
||||
const int radius=conservative ? 1 : 2,age_limit=conservative ? 4 : 16;
|
||||
float current_z=wh_depth(p,sx,sy);
|
||||
float tolerance=(conservative ? .01f : .02f)*(1-current_z);
|
||||
const float depth_floor=conservative ? 2e-7f : 2e-6f;
|
||||
if(tolerance<depth_floor) tolerance=depth_floor;
|
||||
if(p.reserved & D4R_WIRE_HISTORY_SKY_ONLY)
|
||||
{
|
||||
// Fractional wire coverage may overlap unrelated foreground geometry.
|
||||
// Preserve that surface rather than dragging wire color onto it. Sky
|
||||
// and depth-matched wire centers remain eligible; NaN depth is rejected.
|
||||
const float nearest_z=wh_depth(p,(int)__builtin_floorf(nx+.5f),(int)__builtin_floorf(ny+.5f));
|
||||
if(!(nearest_z>=.99999f) && !(__builtin_fabsf(nearest_z-current_z)<tolerance)) return;
|
||||
}
|
||||
*(float*)(p.output_depth+4*index)=current_z;
|
||||
int ix=wh_clamp((int)__builtin_floorf((x+.5f)*p.render_width/p.width+p.jitter_x),5,p.render_width-6);
|
||||
int iy=wh_clamp((int)__builtin_floorf((y+.5f)*p.render_height/p.height+p.jitter_y),5,p.render_height-6);
|
||||
float source_x=(x+.5f)*p.render_width/p.width+p.jitter_x;
|
||||
float source_y=(y+.5f)*p.render_height/p.height+p.jitter_y;
|
||||
|
||||
float best_distance=1e30f;int source_ix=ix,source_iy=iy;
|
||||
for(int dy=-radius;dy<=radius;dy++) for(int dx=-radius;dx<=radius;dx++)
|
||||
{
|
||||
int tx=ix+dx,ty=iy+dy;
|
||||
float value=*(const float*)(p.input_depth+(uint64_t)ty*p.input_depth_pitch+4*tx);
|
||||
if(!(__builtin_fabsf(value-current_z)<tolerance)) continue;
|
||||
float xx=tx+.5f-source_x,yy=ty+.5f-source_y,distance=xx*xx+yy*yy;
|
||||
if(distance<best_distance) {best_distance=distance;source_ix=tx;source_iy=ty;}
|
||||
}
|
||||
uint64_t target=0;
|
||||
if(best_distance!=1e30f)
|
||||
{
|
||||
target=p.color+(uint64_t)source_iy*p.color_pitch+8*source_ix;
|
||||
*(uint64_t*)(p.output_history+offset)=*(const uint64_t*)target;
|
||||
*(uint8_t*)(p.output_age+index)=0;
|
||||
}
|
||||
else if(p.history_valid)
|
||||
{
|
||||
const _Float16* motion=(const _Float16*)(p.motion+(uint64_t)y*p.motion_pitch+4*x);
|
||||
float px=x+(float)motion[0]*p.width,py=y+(float)motion[1]*p.height;
|
||||
if(!(px>=16 && py>=16 && px<p.width-16 && py<p.height-16)) return;
|
||||
float best=1e30f;int hx=0,hy=0;
|
||||
for(int dy=-radius;dy<=radius;dy++) for(int dx=-radius;dx<=radius;dx++)
|
||||
{
|
||||
int tx=wh_clamp((int)__builtin_rintf(px+dx),16,p.width-17),ty=wh_clamp((int)__builtin_rintf(py+dy),16,p.height-17);
|
||||
uint64_t previous_index=(uint64_t)ty*p.width+tx;
|
||||
if(*(const uint8_t*)(p.previous_age+previous_index)>=age_limit) continue;
|
||||
float previous_z=*(const float*)(p.previous_depth+4*previous_index),limit=(conservative ? .01f : .05f)*(1-current_z);if(limit<depth_floor) limit=depth_floor;
|
||||
if(!(__builtin_fabsf(previous_z-current_z)<limit)) continue;
|
||||
// Brightness is not correspondence: selecting the darkest neighbour can
|
||||
// drag a previous wire sample sideways along a camera pan.
|
||||
const float ddx=tx-px,ddy=ty-py;
|
||||
float value=conservative ? ddx*ddx+ddy*ddy : wh_luma(p.previous,p.output_pitch,tx,ty);
|
||||
if(value<best) {best=value;hx=tx;hy=ty;}
|
||||
}
|
||||
if(best!=1e30f)
|
||||
{
|
||||
uint64_t hi=(uint64_t)hy*p.width+hx;
|
||||
target=p.previous+(uint64_t)hy*p.output_pitch+8*hx;
|
||||
*(uint64_t*)(p.output_history+offset)=*(const uint64_t*)target;
|
||||
*(uint8_t*)(p.output_age+index)=*(const uint8_t*)(p.previous_age+hi)+1;
|
||||
}
|
||||
}
|
||||
if(!target) return;
|
||||
float background=wh_background(p.current,p.output_pitch,x,y,ax,ay,false);
|
||||
float current=wh_luma(p.current,p.output_pitch,x,y);
|
||||
float value=wh_luma(target,8,0,0),ratio=value/background;
|
||||
float threshold=background*__builtin_sqrtf(ratio*__builtin_sqrtf(ratio));
|
||||
if(!((ratio<.8408964153f && current>threshold) || (ratio>1.189207115f && current<threshold))) return;
|
||||
if constexpr(Sparse)
|
||||
{
|
||||
*(uint64_t*)(p.output+offset)=*(const uint64_t*)(p.current+offset);
|
||||
*(uint8_t*)(patch_mask+index)=1;
|
||||
}
|
||||
if(coverage>1) coverage=1;
|
||||
const _Float16* observed=(const _Float16*)target;
|
||||
const _Float16* now=(_Float16*)(p.current+offset);_Float16* out=(_Float16*)(p.output+offset);
|
||||
for(int c=0;c<3;c++)
|
||||
{
|
||||
float result=(float)now[c]*(1-coverage)+(float)observed[c]*coverage;
|
||||
if(result<0) result=0;if(result>65504) result=65504;out[c]=(_Float16)result;
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256,256))) void d4r_wire_history(D4rWireHistoryParams p)
|
||||
{ wh_run<false>(p,0); }
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256,256))) void d4r_wire_history_sparse(D4rWireHistorySparseParams p)
|
||||
{ wh_run<true>(p.base,p.patch_mask); }
|
||||
extern "C" __attribute__((global, amdgpu_flat_work_group_size(256,256))) void d4r_wire_history_apply(D4rWireHistorySparseParams p)
|
||||
{
|
||||
const unsigned t=__builtin_amdgcn_workitem_id_x();
|
||||
const unsigned x=__builtin_amdgcn_workgroup_id_x()*16+(t&15),y=__builtin_amdgcn_workgroup_id_y()*16+(t>>4);
|
||||
if(x>=p.base.width || y>=p.base.height || !*(const uint8_t*)(p.patch_mask+(uint64_t)y*p.base.width+x)) return;
|
||||
const uint64_t offset=(uint64_t)y*p.base.output_pitch+8*x;
|
||||
*(uint64_t*)(p.base.current+offset)=*(const uint64_t*)(p.base.output+offset);
|
||||
}
|
||||
#undef WH_SYNC
|
||||
|
||||
extern "C" __attribute__((global,amdgpu_flat_work_group_size(256,256))) void d4r_wire_history_preclassified(D4rWireHistoryPreparedParams p)
|
||||
{ wh_run<true,true>(p.sparse.base,p.sparse.patch_mask,p.axes); }
|
||||
Reference in new issue
Block a user