engine: per-layer GPU timestamps for the M / L network (D4R_ENGINE_M_LAYER_STAMPS)

Timing only: with D4R_ENGINE_M_LAYER_STAMPS=1 the runtime writes a timestamp
after each of the ten network layers (queries first + 7 on) and d4r-m prints
the per-layer medians. RX 9070 XT, 1280x720 -> 1440p: enc1 210, enc2 115,
tube 6 x 50, dec2 114, dec1 209 us.

Signed-off-by: itchyfeetleech <[email protected]>

engine/replay_m.cpp

@@ -107,7 +107,7 @@ int main(int argc, char** argv) try {
     CK(vkCreateCommandPool(c.device, &pi, nullptr, &c.pool));
     VkCommandBufferAllocateInfo cai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; cai.commandPool = c.pool; cai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; cai.commandBufferCount = 1;
     VkCommandBuffer cb; CK(vkAllocateCommandBuffers(c.device, &cai, &cb));
-    VkQueryPoolCreateInfo qpi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qpi.queryType = VK_QUERY_TYPE_TIMESTAMP; qpi.queryCount = 7;
+    VkQueryPoolCreateInfo qpi{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; qpi.queryType = VK_QUERY_TYPE_TIMESTAMP; qpi.queryCount = 17;
     VkQueryPool qp; CK(vkCreateQueryPool(c.device, &qpi, nullptr, &qp));
     VkPhysicalDeviceProperties props; vkGetPhysicalDeviceProperties(c.physical, &props);
     auto submit = [&] {
@@ -151,7 +151,7 @@ int main(int argc, char** argv) try {
                 if (data.size() != sizes[k]) throw std::runtime_error("wrong input size");
                 frameInputs.push_back(std::move(data));
             }
-        std::vector<double> times, walls; std::vector<std::vector<double>> stage(6);
+        std::vector<double> times, walls; std::vector<std::vector<double>> stage(6), layer(10);
         const auto now = [] { return std::chrono::duration<double, std::micro>(std::chrono::steady_clock::now().time_since_epoch()).count(); };
         // D4R_M_RESET_ALL=1: every frame is a reset frame, as the D3D12 harness passes them without its temporal option
         const bool resetAll = std::getenv("D4R_M_RESET_ALL") != nullptr;
@@ -169,7 +169,7 @@ int main(int argc, char** argv) try {
                 s.renderWidth = rw; s.renderHeight = rh;
                 const double t0 = now();
                 engine.setFrame({color.view, motion.view, depth.view}, s);
-                vkCmdResetQueryPool(cb, qp, 0, 7);
+                vkCmdResetQueryPool(cb, qp, 0, 17);
                 double networkUs = 0;
                 if (hip) {
 #ifdef D4R_ENGINE_HIP
@@ -186,12 +186,16 @@ int main(int argc, char** argv) try {
                 if (pass == 0) vkCmdCopyImageToBuffer(cb, engine.outputImage(), VK_IMAGE_LAYOUT_GENERAL, out.buffer, 1, &region);
                 submit();
                 if (pass) walls.push_back(now() - t0);
-                uint64_t t[7]; CK(vkGetQueryPoolResults(c.device, qp, 0, 7, sizeof(t), t, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT));
+                uint64_t t[17] = {};
+                // D4R_ENGINE_M_LAYER_STAMPS=1: the ten network layers' stamps as well (queries 7 - 16)
+                static const bool layerStamps = std::getenv("D4R_ENGINE_M_LAYER_STAMPS") != nullptr && !hip;
+                CK(vkGetQueryPoolResults(c.device, qp, 0, layerStamps ? 17 : 7, sizeof(t), t, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT));
                 const double tick = props.limits.timestampPeriod / 1000.0;
                 double d[6]; for (int k = 0; k < 6; ++k) d[k] = double(t[k + 1] - t[k]) * tick;
                 if (hip) d[2] = networkUs; // measured HIP network time; the complete span includes host handoffs
                 const double us = double(t[6] - t[0]) * tick;
                 if (pass) for (int k = 0; k < 6; ++k) stage[k].push_back(d[k]);
+                if (pass && layerStamps) for (int l = 0; l < 10; ++l) layer[l].push_back(double(t[7 + l] - (l ? t[6 + l] : t[2])) * tick);
                 if (pass == 0 && std::getenv("D4R_M_DUMP")) {
                     // development: every intermediate of this frame, to PREFIX-NAME-FRAME.bin
                     for (const auto& d : engine.debugResources()) {
@@ -216,6 +220,11 @@ int main(int argc, char** argv) try {
             std::sort(walls.begin(), walls.end()); std::printf("  wall clock, record to completion: median %.1f us, min %.1f us\n", walls[walls.size() / 2], walls[0]);
             const char* names[6] = {"exposure", "input", "network", "expansion", "reconstruction", "downsample"};
             for (int k = 0; k < 6; ++k) { std::sort(stage[k].begin(), stage[k].end()); std::printf("  %-14s median %.1f us\n", names[k], stage[k][stage[k].size() / 2]); }
+            if (!layer[0].empty()) {
+                std::printf("  layers (us):");
+                for (auto& v : layer) { std::sort(v.begin(), v.end()); std::printf(" %.1f", v[v.size() / 2]); }
+                std::printf("\n");
+            }
         }
         CK(vkDeviceWaitIdle(c.device));
     }

engine/runtime_m.inc

@@ -512,9 +512,12 @@ void MEngine::recordFrame(VkCommandBuffer cb, VkQueryPool stamps, uint32_t first
     auto& e = *impl; auto& m = e.model;
     if (e.external) throw std::runtime_error("this engine's network is external: record the front and the back");
     recordFront(cb, stamps, first);
+    // timing only: D4R_ENGINE_M_LAYER_STAMPS=1 also stamps after every network layer, from query first + 7 on
+    static const bool layerStamps = std::getenv("D4R_ENGINE_M_LAYER_STAMPS") != nullptr;
     for (unsigned l = 0; l < MLayers; ++l) {
         const uint32_t w = e.gw / 2 / MScale[l], h = e.gh / 2 / MScale[l];
         dispatch(cb, m.network[l], e.layerSets[l], (w + (MShift[l] ^ ((e.frame & 1u) * 2u)) + 7) / 8, (h + MShift[l] + 7) / 8); emptyStage(cb);
+        if (stamps && layerStamps) vkCmdWriteTimestamp(cb, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, stamps, first + 7 + l);
     }
     recordBack(cb, stamps, stamps ? first + 3 : 0);
 }