graphics: exclude overlay from screenshots (#865)

## Link to GitHub Issue or related Pull Request, if one exists
#0

## Description of change

Adds the `-screenshotoverlay` option.

By default, file screenshots exclude the Spice overlay. Enabling this
option captures screenshots after the overlay is rendered.

Also updates DX11 backend to allow screenshots even when overlay is
disabled.

## Testing

src/spice2x/hooks/graphics/backends/d3d11/d3d11_swapchain.cpp

@@ -105,8 +105,62 @@ Present1_t Present1_orig = nullptr;
 bool g_swapchain_hooked = false;
 bool g_swapchain1_hooked = false;
 
+// sub-screens / IME helpers are usually child or zero-sized windows.
+// visibility isn't checked - the game may present before showing the window.
+bool looks_like_game_window(HWND hwnd) {
+    RECT client {};
+    return GetAncestor(hwnd, GA_ROOT) == hwnd
+        && GetClientRect(hwnd, &client)
+        && client.right > client.left
+        && client.bottom > client.top;
+}
+
+// only the main game window; ignore sub-screens / IME helpers.
+bool is_main_game_swapchain(IDXGISwapChain *swapchain) {
+    DXGI_SWAP_CHAIN_DESC desc {};
+    if (!swapchain || FAILED(swapchain->GetDesc(&desc)) || !desc.OutputWindow) {
+        return false;
+    }
+
+    HWND main = d3d11_hooks::main_hwnd();
+    if (!main) {
+        // no creation hook recorded a window, so fall back to the presenting one;
+        // the choice is permanent, so require a plausible game window
+        if (!looks_like_game_window(desc.OutputWindow)) {
+            return false;
+        }
+
+        log_misc(
+            "graphics::d3d11",
+            "try to notemain hwnd from swapchain present: 0x{:x}",
+            (uintptr_t)desc.OutputWindow);
+
+        d3d11_hooks::note_main_hwnd(desc.OutputWindow);
+
+        // it may have been ignored, or another thread may have won the slot
+        main = d3d11_hooks::main_hwnd();
+    }
+    return desc.OutputWindow == main;
+}
+
+// checks are ordered cheapest first, since this runs on every present
 void try_create_overlay(IDXGISwapChain *swapchain) {
-    if (!swapchain || overlay::OVERLAY) {
+    if (!swapchain) {
+        return;
+    }
+
+    // overlay is disabled by user
+    if (!overlay::ENABLED) {
+        return;
+    }
+
+    // overlay is already enabled and attached
+    if (overlay::OVERLAY) {
+        return;
+    }
+
+    // ignore sub windows
+    if (!is_main_game_swapchain(swapchain)) {
         return;
     }
 
@@ -115,12 +169,6 @@ void try_create_overlay(IDXGISwapChain *swapchain) {
         return;
     }
 
-    // only attach to the main game window; ignore sub-screens / IME helpers.
-    HWND main = d3d11_hooks::main_hwnd();
-    if (main && desc.OutputWindow != main) {
-        return;
-    }
-
     // theme the native title bar; first present is the only reliable point for
     // windows whose swapchain bypasses our factory hooks (e.g. UnityPlayer.dll)
     set_window_dark_titlebar(desc.OutputWindow);
@@ -146,13 +194,23 @@ void try_create_overlay(IDXGISwapChain *swapchain) {
     device->Release();
 }
 
-void pump_overlay(IDXGISwapChain *swapchain) {
+// screenshots have to keep working with the overlay disabled, so they are not gated on it
-    if (!overlay::OVERLAY || !overlay::OVERLAY->uses_swapchain(swapchain)) {
+void pump_frame(IDXGISwapChain *swapchain) {
+    const bool has_overlay =
+        overlay::OVERLAY && overlay::OVERLAY->uses_swapchain(swapchain);
+    if (!has_overlay && !is_main_game_swapchain(swapchain)) {
         return;
     }
 
     graphics_poll_screenshot_hotkey();
 
+    // before the overlay render so the screenshot excludes it
+    if (!GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
+        d3d11_hooks::try_screenshot(swapchain);
+    }
+
+    if (has_overlay) {
+
         // size imgui to the backbuffer (not window client). dxgi may upscale
         // a small backbuffer into a larger client rect; without this override
         // imgui would draw past the RTV and the mouse mapping would be off.
@@ -166,10 +224,13 @@ void pump_overlay(IDXGISwapChain *swapchain) {
         overlay::OVERLAY->update();
         overlay::OVERLAY->new_frame();
         overlay::OVERLAY->render();
+    }
 
-    // after overlay render so toasts/menus end up in the saved image.
+    // after the overlay render so the screenshot includes toasts / menus
+    if (GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
         d3d11_hooks::try_screenshot(swapchain);
     }
+}
 
 // ----------------------------------------------------------------------
 // swapchain method hooks
@@ -177,8 +238,11 @@ void pump_overlay(IDXGISwapChain *swapchain) {
 HRESULT STDMETHODCALLTYPE Present_hook(
         IDXGISwapChain *swapchain, UINT SyncInterval, UINT Flags)
 {
+    // a test present doesn't display anything; don't pick a window or take a screenshot off it
+    if (!(Flags & DXGI_PRESENT_TEST)) {
         try_create_overlay(swapchain);
-    pump_overlay(swapchain);
+        pump_frame(swapchain);
+    }
     return Present_orig(swapchain, SyncInterval, Flags);
 }
 
@@ -186,8 +250,10 @@ HRESULT STDMETHODCALLTYPE Present1_hook(
         IDXGISwapChain1 *swapchain, UINT SyncInterval, UINT Flags,
         const DXGI_PRESENT_PARAMETERS *pParams)
 {
+    if (!(Flags & DXGI_PRESENT_TEST)) {
         try_create_overlay(swapchain);
-    pump_overlay(swapchain);
+        pump_frame(swapchain);
+    }
     return Present1_orig(swapchain, SyncInterval, Flags, pParams);
 }
 

src/spice2x/hooks/graphics/backends/d3d9/d3d9_backend.cpp

@@ -1489,6 +1489,13 @@ void graphics_d3d9_on_present(
         SurfaceHook(device);
     }
 
+    graphics_poll_screenshot_hotkey();
+
+    // before the overlay render so the screenshot excludes it
+    if (!GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
+        graphics_d3d9_process_screenshot(device, SUB_SWAP_CHAIN);
+    }
+
     // Do overlay init as many d3d9 hooks create a dummy instance to get vtable offsets and never
     // call `Present`. This avoids race conditions on `IDirect3D9::CreateDevice` like with
     // `dx9osd.dll` for pfreepanic.
@@ -1508,6 +1515,11 @@ void graphics_d3d9_on_present(
         device->EndScene();
     }
 
+    // after the overlay render so the screenshot includes toasts / menus
+    if (GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
+        graphics_d3d9_process_screenshot(device, SUB_SWAP_CHAIN);
+    }
+
     // for IIDX TDJ / SDVX UFC, handle subscreen
     const bool is_vm = games::sdvx::is_valkyrie_model();
     const bool is_tdj = avs::game::is_model("LDJ") && games::iidx::TDJ_MODE;
@@ -1522,8 +1534,8 @@ void graphics_d3d9_on_present(
         wintouchemu::update();
     }
 
-    graphics_poll_screenshot_hotkey();
+    // API capture always includes the overlay
-    graphics_d3d9_process_screenshot_and_capture(device, SUB_SWAP_CHAIN);
+    graphics_d3d9_process_capture(device, SUB_SWAP_CHAIN);
 }
 
 void update_backbuffer_dimensions(D3DPRESENT_PARAMETERS *params) {

src/spice2x/hooks/graphics/backends/d3d9/d3d9_screenshot.cpp

@@ -348,43 +348,43 @@ static void dispatch_surface_save(
     }
 }
 
-static std::optional<ImageRequest> consume_image_request() {
+static void process_image_request(
-    if (graphics_screenshot_consume()) {
-        return ImageRequest {
-            .kind = ImageRequestKind::Screenshot,
-            .screen = 0,
-        };
-    }
-
-    int capture_screen = 0;
-    if (graphics_capture_consume(&capture_screen)) {
-        return ImageRequest {
-            .kind = ImageRequestKind::Capture,
-            .screen = capture_screen,
-        };
-    }
-
-    return std::nullopt;
-}
-
-void graphics_d3d9_process_screenshot_and_capture(
         IDirect3DDevice9 *device,
-        IDirect3DSwapChain9 *sub_swap_chain) {
+        IDirect3DSwapChain9 *sub_swap_chain,
-    const auto request = consume_image_request();
+        const ImageRequest &request) {
-    if (!request.has_value()) {
-        return;
-    }
-
     auto copy = acquire_backbuffer_copy(
             device,
             sub_swap_chain,
-            request->screen);
+            request.screen);
     if (!copy.has_value()) {
-        if (request->kind == ImageRequestKind::Capture) {
+        if (request.kind == ImageRequestKind::Capture) {
-            graphics_capture_skip(request->screen);
+            graphics_capture_skip(request.screen);
         }
         return;
     }
 
-    dispatch_surface_save(*request, std::move(*copy));
+    dispatch_surface_save(request, std::move(*copy));
+}
+
+void graphics_d3d9_process_screenshot(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain) {
+    if (graphics_screenshot_consume()) {
+        process_image_request(device, sub_swap_chain, ImageRequest {
+            .kind = ImageRequestKind::Screenshot,
+            .screen = 0,
+        });
+    }
+}
+
+void graphics_d3d9_process_capture(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain) {
+    int screen = 0;
+    if (graphics_capture_consume(&screen)) {
+        process_image_request(device, sub_swap_chain, ImageRequest {
+            .kind = ImageRequestKind::Capture,
+            .screen = screen,
+        });
+    }
 }

src/spice2x/hooks/graphics/backends/d3d9/d3d9_screenshot.h

@@ -2,6 +2,10 @@
 
 #include <d3d9.h>
 
-void graphics_d3d9_process_screenshot_and_capture(
+void graphics_d3d9_process_screenshot(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain);
+
+void graphics_d3d9_process_capture(
         IDirect3DDevice9 *device,
         IDirect3DSwapChain9 *sub_swap_chain);

src/spice2x/hooks/graphics/graphics.cpp

@@ -118,6 +118,7 @@ uint32_t GRAPHICS_FS_ORIGINAL_HEIGHT = 0;
 // settings
 std::string GRAPHICS_DEVICEID = "PCI\\VEN_1002&DEV_7146";
 std::string GRAPHICS_SCREENSHOT_DIR = ".\\screenshots";
+bool GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY = false;
 
 static decltype(ChangeDisplaySettingsA) *ChangeDisplaySettingsA_orig = nullptr;
 static decltype(ChangeDisplaySettingsExA) *ChangeDisplaySettingsExA_orig = nullptr;

src/spice2x/hooks/graphics/graphics.h

@@ -111,6 +111,7 @@ extern bool FAKE_SUBSCREEN_ADAPTER;
 // settings
 extern std::string GRAPHICS_DEVICEID;
 extern std::string GRAPHICS_SCREENSHOT_DIR;
+extern bool GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY;
 
 // Direct3D 9 settings
 extern std::optional<UINT> D3D9_ADAPTER;

src/spice2x/launcher/launcher.cpp

@@ -1103,6 +1103,9 @@ int main_implementation(int argc, char *argv[]) {
     if (options[launcher::Options::ScreenshotFolder].is_active()) {
         GRAPHICS_SCREENSHOT_DIR = options[launcher::Options::ScreenshotFolder].value_text();
     }
+    if (options[launcher::Options::ScreenshotIncludeOverlay].value_bool()) {
+        GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY = true;
+    }
     if (options[launcher::Options::DisableColoredOutput].value_bool()) {
         logger::COLOR = false;
     }

src/spice2x/launcher/options.cpp

@@ -3397,6 +3397,14 @@ static const std::vector<OptionDefinition> OPTION_DEFINITIONS = {
         .type = OptionType::Bool,
         .category = "OBS Control",
     },
+    {
+        // ScreenshotIncludeOverlay
+        .title = "Include Overlay in Screenshots",
+        .name = "screenshotoverlay",
+        .desc = "Includes Spice overlay in screenshots.",
+        .type = OptionType::Bool,
+        .category = "General Overlay",
+    },
 };
 
 const std::vector<std::string> &launcher::get_categories(Options::OptionsCategory category) {

src/spice2x/launcher/options.h

@@ -320,7 +320,8 @@ namespace launcher {
             OBSWebSocketHost,
             OBSWebSocketPort,
             OBSWebSocketPassword,
-            OBSWebSocketDebug
+            OBSWebSocketDebug,
+            ScreenshotIncludeOverlay
         };
 
         enum class OptionsCategory {