graphics: exclude overlay from screenshots (#865)

## Link to GitHub Issue or related Pull Request, if one exists
#0

## Description of change

Adds the `-screenshotoverlay` option.

By default, file screenshots exclude the Spice overlay. Enabling this
option captures screenshots after the overlay is rendered.

Also updates DX11 backend to allow screenshots even when overlay is
disabled.

## Testing

src/spice2x/hooks/graphics/backends/d3d9/d3d9_backend.cpp

@@ -1489,6 +1489,13 @@ void graphics_d3d9_on_present(
         SurfaceHook(device);
     }
 
+    graphics_poll_screenshot_hotkey();
+
+    // before the overlay render so the screenshot excludes it
+    if (!GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
+        graphics_d3d9_process_screenshot(device, SUB_SWAP_CHAIN);
+    }
+
     // Do overlay init as many d3d9 hooks create a dummy instance to get vtable offsets and never
     // call `Present`. This avoids race conditions on `IDirect3D9::CreateDevice` like with
     // `dx9osd.dll` for pfreepanic.
@@ -1508,6 +1515,11 @@ void graphics_d3d9_on_present(
         device->EndScene();
     }
 
+    // after the overlay render so the screenshot includes toasts / menus
+    if (GRAPHICS_SCREENSHOT_INCLUDE_OVERLAY) {
+        graphics_d3d9_process_screenshot(device, SUB_SWAP_CHAIN);
+    }
+
     // for IIDX TDJ / SDVX UFC, handle subscreen
     const bool is_vm = games::sdvx::is_valkyrie_model();
     const bool is_tdj = avs::game::is_model("LDJ") && games::iidx::TDJ_MODE;
@@ -1522,8 +1534,8 @@ void graphics_d3d9_on_present(
         wintouchemu::update();
     }
 
-    graphics_poll_screenshot_hotkey();
-    graphics_d3d9_process_screenshot_and_capture(device, SUB_SWAP_CHAIN);
+    // API capture always includes the overlay
+    graphics_d3d9_process_capture(device, SUB_SWAP_CHAIN);
 }
 
 void update_backbuffer_dimensions(D3DPRESENT_PARAMETERS *params) {

src/spice2x/hooks/graphics/backends/d3d9/d3d9_screenshot.cpp

@@ -348,43 +348,43 @@ static void dispatch_surface_save(
     }
 }
 
-static std::optional<ImageRequest> consume_image_request() {
-    if (graphics_screenshot_consume()) {
-        return ImageRequest {
-            .kind = ImageRequestKind::Screenshot,
-            .screen = 0,
-        };
-    }
-
-    int capture_screen = 0;
-    if (graphics_capture_consume(&capture_screen)) {
-        return ImageRequest {
-            .kind = ImageRequestKind::Capture,
-            .screen = capture_screen,
-        };
-    }
-
-    return std::nullopt;
-}
-
-void graphics_d3d9_process_screenshot_and_capture(
+static void process_image_request(
         IDirect3DDevice9 *device,
-        IDirect3DSwapChain9 *sub_swap_chain) {
-    const auto request = consume_image_request();
-    if (!request.has_value()) {
-        return;
-    }
-
+        IDirect3DSwapChain9 *sub_swap_chain,
+        const ImageRequest &request) {
     auto copy = acquire_backbuffer_copy(
             device,
             sub_swap_chain,
-            request->screen);
+            request.screen);
     if (!copy.has_value()) {
-        if (request->kind == ImageRequestKind::Capture) {
-            graphics_capture_skip(request->screen);
+        if (request.kind == ImageRequestKind::Capture) {
+            graphics_capture_skip(request.screen);
         }
         return;
     }
 
-    dispatch_surface_save(*request, std::move(*copy));
+    dispatch_surface_save(request, std::move(*copy));
+}
+
+void graphics_d3d9_process_screenshot(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain) {
+    if (graphics_screenshot_consume()) {
+        process_image_request(device, sub_swap_chain, ImageRequest {
+            .kind = ImageRequestKind::Screenshot,
+            .screen = 0,
+        });
+    }
+}
+
+void graphics_d3d9_process_capture(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain) {
+    int screen = 0;
+    if (graphics_capture_consume(&screen)) {
+        process_image_request(device, sub_swap_chain, ImageRequest {
+            .kind = ImageRequestKind::Capture,
+            .screen = screen,
+        });
+    }
 }

src/spice2x/hooks/graphics/backends/d3d9/d3d9_screenshot.h

@@ -2,6 +2,10 @@
 
 #include <d3d9.h>
 
-void graphics_d3d9_process_screenshot_and_capture(
+void graphics_d3d9_process_screenshot(
+        IDirect3DDevice9 *device,
+        IDirect3DSwapChain9 *sub_swap_chain);
+
+void graphics_d3d9_process_capture(
         IDirect3DDevice9 *device,
         IDirect3DSwapChain9 *sub_swap_chain);