diff --git a/src/spice2x/README.md b/src/spice2x/README.md index 11f5bfd..4425487 100644 --- a/src/spice2x/README.md +++ b/src/spice2x/README.md @@ -127,6 +127,22 @@ doesn't matter since the TCP protocol doesn't allow for out of order data, however this may change when/if support for UDP is being introduced. The only restriction is that the ID has to be a valid 64-bit unsigned integer. +#### Capture +- get_screens() + - returns the screen numbers the game has registered for capture +- get_jpg(screen: uint, quality: uint, divide: uint) + - returns the timestamp, width, height and base64 encoded JPEG of one screen + - all parameters are optional and default to screen 0, quality 70, divide 1 + - divide shrinks the image by that factor before encoding +- get_streams() + - returns a dict describing the HTTP video stream, or no data at all when + `-apistream` is not enabled and there is nothing to describe + - `port` is the stream server port + - `formats` lists the wire formats this build serves, each with a `name` + (`h264` or `mjpeg`) and the `path` to request them on + - `screens` lists every capturable screen with its `width`, `height`, and + `busy` + #### Card - insert(index: uint, card_id: hex) - inserts a card which gets read by the emulated card readers for the game @@ -298,6 +314,10 @@ stream over plain HTTP. Enable it with `-apistream`. It listens on the API port plus two, in the same way the WebSocket server uses the API port plus one, so `-api 1337` puts the stream on 1339. This means `-api` has to be enabled too. +Rather than working the port out, clients should ask the JSON API for it with +`capture.get_streams()`, which also reports which of the formats below this +build serves, the size of each screen and whether one is already taken. + Two formats are served: http://host:1339/stream.mjpg JPEG frames, multipart/x-mixed-replace @@ -323,9 +343,10 @@ The stream is view only. Touch and other input still go through the JSON API, so a companion app needs both. There is no authentication on the stream port - anyone who can reach it can watch the screen. -WinXP builds have no video stream. Neither encoder is compiled in, so every -endpoint returns 404, and the JSON API's JPEG screen capture is unavailable for -the same reason. +WinXP builds have no video stream. Neither encoder is compiled in, so nothing +listens on the stream port even with `-apistream`, `capture.get_streams()` +returns no data, and the JSON API's JPEG screen capture is unavailable for the +same reason. ## Native wrapper libraries Spicetools provides wrapper libraries in: Arduino, C++, Dart, and Python. diff --git a/src/spice2x/api/modules/capture.cpp b/src/spice2x/api/modules/capture.cpp index b8ab2b1..644d944 100644 --- a/src/spice2x/api/modules/capture.cpp +++ b/src/spice2x/api/modules/capture.cpp @@ -1,8 +1,11 @@ #include "capture.h" +#include #include #include #include #include "api/capture_pump.h" +#include "api/stream_format.h" +#include "api/stream_server.h" #include "external/rapidjson/document.h" #include "hooks/graphics/graphics.h" #include "hooks/graphics/jpeg_encoder.h" @@ -71,6 +74,7 @@ namespace api::modules { Capture::Capture() : Module("capture") { functions["get_screens"] = std::bind(&Capture::get_screens, this, _1, _2); functions["get_jpg"] = std::bind(&Capture::get_jpg, this, _1, _2); + functions["get_streams"] = std::bind(&Capture::get_streams, this, _1, _2); } /** @@ -141,4 +145,98 @@ namespace api::modules { CAPTURE_BUFFER.clear(); try_cached_response(screen, res); } + + /** + * get_streams() + */ + void Capture::get_streams(Request &req, Response &res) { + + auto &alloc = res.doc()->GetAllocator(); + + // nothing is listening without -apistream, so there is no stream to describe + const unsigned short port = stream_server_port(); + if (port == 0) { + return; + } + + Value formats(kArrayType); + for (const auto &[name, path] : stream_formats()) { + Value entry(kObjectType); + entry.AddMember("name", Value(name.c_str(), alloc), alloc); + entry.AddMember("path", Value(path.c_str(), alloc), alloc); + formats.PushBack(entry, alloc); + } + + std::vector screen_numbers; + graphics_screens_get(screen_numbers); + + // measuring a screen nobody has captured yet waits for the game to present, which can + // take as long as the whole request is allowed, so only one screen is measured per + // call and the rest are reported null until a later one settles them. which screen + // gets the attempt rotates, otherwise one that never presents would take every + // request and the screens behind it would stay unmeasured forever + int probe_screen = -1; + { + std::vector unmeasured; + for (const auto screen : screen_numbers) { + if (screen < static_cast(GRAPHICS_CAPTURE_SCREEN_NO) + && !graphics_capture_last_size(screen, nullptr, nullptr) + && !capture_pump::screen_claimed(screen)) { + unmeasured.push_back(screen); + } + } + + if (!unmeasured.empty()) { + static std::atomic probe_cursor { 0 }; + probe_screen = unmeasured[probe_cursor.fetch_add(1) % unmeasured.size()]; + } + } + + Value screens(kArrayType); + for (const auto screen : screen_numbers) { + if (screen >= static_cast(GRAPHICS_CAPTURE_SCREEN_NO)) { + continue; + } + + int width = 0; + int height = 0; + bool known = graphics_capture_last_size(screen, &width, &height); + + // a probe holds the screen for as long as it waits, so a second caller arriving + // during one would queue behind it and then take a wait of its own; let it report + // the screen as unmeasured instead and pick the size up once the first is done + static std::atomic probe_running { false }; + if (!known && screen == probe_screen && !probe_running.exchange(true)) { + std::shared_ptr pixels; + known = capture_pump::capture_direct( + screen, pixels, 1, nullptr, &width, &height); + probe_running = false; + } + + // a screen of unknown size cannot be described, and a client told about it could + // not size its decoder anyway; leaving it out until it has been measured beats + // handing over an entry that has to be treated as absent + if (!known) { + continue; + } + + Value entry(kObjectType); + entry.AddMember("screen", screen, alloc); + entry.AddMember("width", width, alloc); + entry.AddMember("height", height, alloc); + + // a screen carries one viewer at a time, so this is what decides whether a client + // can connect at all; still racy by the time it does, only more honest than not + entry.AddMember("busy", capture_pump::screen_claimed(screen), alloc); + + screens.PushBack(entry, alloc); + } + + Value info(kObjectType); + info.AddMember("port", port, alloc); + info.AddMember("formats", formats, alloc); + info.AddMember("screens", screens, alloc); + + res.add_data(info); + } } diff --git a/src/spice2x/api/modules/capture.h b/src/spice2x/api/modules/capture.h index fb8c55d..b994f3a 100644 --- a/src/spice2x/api/modules/capture.h +++ b/src/spice2x/api/modules/capture.h @@ -19,5 +19,6 @@ namespace api::modules { // function definitions void get_screens(Request &req, Response &res); void get_jpg(Request &req, Response &res); + void get_streams(Request &req, Response &res); }; } diff --git a/src/spice2x/api/resources/dart/spiceapi-websocket/spiceapi.dart b/src/spice2x/api/resources/dart/spiceapi-websocket/spiceapi.dart index 413db56..1491e74 100644 --- a/src/spice2x/api/resources/dart/spiceapi-websocket/spiceapi.dart +++ b/src/spice2x/api/resources/dart/spiceapi-websocket/spiceapi.dart @@ -11,6 +11,7 @@ part "src/exceptions.dart"; part "src/rc4.dart"; part "src/wrappers/analogs.dart"; part "src/wrappers/buttons.dart"; +part "src/wrappers/capture.dart"; part "src/wrappers/card.dart"; part "src/wrappers/coin.dart"; part "src/wrappers/control.dart"; diff --git a/src/spice2x/api/resources/dart/spiceapi-websocket/src/connection.dart b/src/spice2x/api/resources/dart/spiceapi-websocket/src/connection.dart index a27ec3f..089bbff 100644 --- a/src/spice2x/api/resources/dart/spiceapi-websocket/src/connection.dart +++ b/src/spice2x/api/resources/dart/spiceapi-websocket/src/connection.dart @@ -4,8 +4,8 @@ part of spiceapi; class Connection { // settings - static const _TIMEOUT = Duration(seconds: 2); - static const _BUFFER_SIZE = 1024 * 8; + static const _TIMEOUT = Duration(seconds: 3); + static const _BUFFER_SIZE = 1024 * 1024 * 8; // state final String host, pass; diff --git a/src/spice2x/api/resources/dart/spiceapi-websocket/src/wrappers/capture.dart b/src/spice2x/api/resources/dart/spiceapi-websocket/src/wrappers/capture.dart new file mode 100644 index 0000000..a6c3b88 --- /dev/null +++ b/src/spice2x/api/resources/dart/spiceapi-websocket/src/wrappers/capture.dart @@ -0,0 +1,47 @@ +part of spiceapi; + +class CaptureData { + int timestamp; + int width, height; + Uint8List data; +} + +var _base64DecoderInstance = Base64Decoder(); + +Future captureGetScreens(Connection con) { + var req = Request("capture", "get_screens"); + return con.request(req).then((res) { + return res.getData(); + }); +} + +Future captureGetJPG(Connection con, { + int screen = 0, + int quality = 70, + int divide = 1, +}) { + var req = Request("capture", "get_jpg"); + req.addParam(screen); + req.addParam(quality); + req.addParam(divide); + return con.request(req).then((res) { + var captureData = CaptureData(); + var data = res.getData(); + if (data.length > 0) captureData.timestamp = data[0]; + if (data.length > 1) captureData.width = data[1]; + if (data.length > 2) captureData.height = data[2]; + if (data.length > 3) { + captureData.data = _base64DecoderInstance.convert(data[3]); + } + return captureData; + }); +} + +/// Describes the HTTP video stream, or null when this spice2x serves none. +Future captureGetStreams(Connection con) { + var req = Request("capture", "get_streams"); + return con.request(req).then((res) { + var data = res.getData(); + return data.length > 0 ? data[0] : null; + }); +} diff --git a/src/spice2x/api/resources/dart/spiceapi/src/wrappers/capture.dart b/src/spice2x/api/resources/dart/spiceapi/src/wrappers/capture.dart index be15f14..a6c3b88 100644 --- a/src/spice2x/api/resources/dart/spiceapi/src/wrappers/capture.dart +++ b/src/spice2x/api/resources/dart/spiceapi/src/wrappers/capture.dart @@ -17,7 +17,7 @@ Future captureGetScreens(Connection con) { Future captureGetJPG(Connection con, { int screen = 0, - int quality = 60, + int quality = 70, int divide = 1, }) { var req = Request("capture", "get_jpg"); @@ -36,3 +36,12 @@ Future captureGetJPG(Connection con, { return captureData; }); } + +/// Describes the HTTP video stream, or null when this spice2x serves none. +Future captureGetStreams(Connection con) { + var req = Request("capture", "get_streams"); + return con.request(req).then((res) { + var data = res.getData(); + return data.length > 0 ? data[0] : null; + }); +} diff --git a/src/spice2x/api/resources/python/spiceapi/__init__.py b/src/spice2x/api/resources/python/spiceapi/__init__.py index f0da213..161201a 100644 --- a/src/spice2x/api/resources/python/spiceapi/__init__.py +++ b/src/spice2x/api/resources/python/spiceapi/__init__.py @@ -2,6 +2,7 @@ from .connection import Connection from .request import Request from .analogs import * from .buttons import * +from .capture import * from .card import * from .coin import * from .control import * diff --git a/src/spice2x/api/resources/python/spiceapi/capture.py b/src/spice2x/api/resources/python/spiceapi/capture.py new file mode 100644 index 0000000..f35b094 --- /dev/null +++ b/src/spice2x/api/resources/python/spiceapi/capture.py @@ -0,0 +1,33 @@ +import base64 + +from .connection import Connection +from .request import Request + + +def capture_get_screens(con: Connection): + res = con.request(Request("capture", "get_screens")) + return res.get_data() + + +def capture_get_jpg(con: Connection, screen: int = 0, quality: int = 70, divide: int = 1): + req = Request("capture", "get_jpg") + req.add_param(screen) + req.add_param(quality) + req.add_param(divide) + data = con.request(req).get_data() + + if len(data) < 4: + return None + + return { + "timestamp": data[0], + "width": data[1], + "height": data[2], + "data": base64.b64decode(data[3]), + } + + +def capture_get_streams(con: Connection): + """Describes the HTTP video stream, or None when this spice2x serves none.""" + data = con.request(Request("capture", "get_streams")).get_data() + return data[0] if data else None diff --git a/src/spice2x/api/stream_format.cpp b/src/spice2x/api/stream_format.cpp index 0fce3ce..4a06971 100644 --- a/src/spice2x/api/stream_format.cpp +++ b/src/spice2x/api/stream_format.cpp @@ -67,4 +67,18 @@ namespace api { return nullptr; } + + std::vector> stream_formats() { + std::vector> formats; + +#ifdef SPICE_JPEG + formats.emplace_back("mjpeg", "/stream.mjpg"); +#endif + +#ifdef SPICE_H264 + formats.emplace_back("h264", "/stream.h264"); +#endif + + return formats; + } } diff --git a/src/spice2x/api/stream_format.h b/src/spice2x/api/stream_format.h index ed04fb9..a0268de 100644 --- a/src/spice2x/api/stream_format.h +++ b/src/spice2x/api/stream_format.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "capture_pump.h" @@ -35,4 +37,7 @@ namespace api { // null when the path does not name a format this build supports std::unique_ptr make_stream_writer( const std::string &path, int quality, int fps); + + // name and path of every format compiled into this build, for clients to pick from + std::vector> stream_formats(); } diff --git a/src/spice2x/api/stream_server.cpp b/src/spice2x/api/stream_server.cpp index bd3216c..6c9ee2f 100644 --- a/src/spice2x/api/stream_server.cpp +++ b/src/spice2x/api/stream_server.cpp @@ -195,11 +195,25 @@ namespace api { "\r\n"; send_all(socket, response); } + + std::atomic LISTENING_PORT { 0 }; + } + + unsigned short stream_server_port() { + return LISTENING_PORT.load(); } StreamServer::StreamServer(unsigned short port) : port(port) { + // WinXP builds compile in neither encoder, so there would be nothing to serve and + // every request would 404; taking the port instead only invites confused clients + if (stream_formats().empty()) { + log_warning("api::stream", + "this build has no video encoders, the video stream is unavailable"); + return; + } + if (!this->open_listener()) { // the stream was asked for explicitly, so say plainly that it is not there log_warning("api::stream", @@ -212,6 +226,8 @@ namespace api { this->accept_worker(); }); + LISTENING_PORT = this->port; + // deliberately not logging a full URL; local IPs would leak into shared logs log_info("api::stream", "video stream is listening on port: {}", this->port); log_warning("api::stream", @@ -269,6 +285,7 @@ namespace api { StreamServer::~StreamServer() { this->running = false; + LISTENING_PORT = 0; if (this->listener != INVALID_SOCKET) { closesocket(this->listener); diff --git a/src/spice2x/api/stream_server.h b/src/spice2x/api/stream_server.h index 0e2646e..d8747d0 100644 --- a/src/spice2x/api/stream_server.h +++ b/src/spice2x/api/stream_server.h @@ -11,6 +11,9 @@ namespace api { + // 0 while no stream server is listening, so the API can tell clients not to look for one + unsigned short stream_server_port(); + class StreamServer { public: diff --git a/src/spice2x/hooks/graphics/graphics.cpp b/src/spice2x/hooks/graphics/graphics.cpp index 6aa5866..61c5701 100644 --- a/src/spice2x/hooks/graphics/graphics.cpp +++ b/src/spice2x/hooks/graphics/graphics.cpp @@ -1481,6 +1481,27 @@ void graphics_capture_skip(int screen) { GRAPHICS_CAPTURE_CV[screen].notify_one(); } +bool graphics_capture_last_size(int screen, int *width, int *height) { + if (screen < 0 || screen >= static_cast(GRAPHICS_CAPTURE_SCREEN_NO)) { + return false; + } + + // consuming a frame clears the pixels but leaves the size, so this survives the read + std::lock_guard lock(GRAPHICS_CAPTURE_BUFFER_M[screen]); + const auto &capture = GRAPHICS_CAPTURE_BUFFER[screen]; + if (!capture.width || !capture.height) { + return false; + } + + if (width != nullptr) { + *width = capture.width; + } + if (height != nullptr) { + *height = capture.height; + } + return true; +} + bool graphics_capture_receive_raw(int screen, std::shared_ptr &out, int divide, uint64_t *timestamp, int *width, int *height) { diff --git a/src/spice2x/hooks/graphics/graphics.h b/src/spice2x/hooks/graphics/graphics.h index 4d85ed8..72e7d13 100644 --- a/src/spice2x/hooks/graphics/graphics.h +++ b/src/spice2x/hooks/graphics/graphics.h @@ -146,6 +146,9 @@ void graphics_capture_trigger(int screen); bool graphics_capture_consume(int *screen); void graphics_capture_enqueue(int screen, uint8_t *data, size_t width, size_t height); void graphics_capture_skip(int screen); +// size of the last frame captured off this screen, before any caller side downscale; false +// until one has been captured, so it cannot report a size for a screen the game never drew +bool graphics_capture_last_size(int screen, int *width, int *height); // on success `out` owns packed 24bpp RGB pixels, width * height * 3 bytes bool graphics_capture_receive_raw(int screen, std::shared_ptr &out, int divide = 0,