diff --git a/src/spice2x/CMakeLists.txt b/src/spice2x/CMakeLists.txt index aaad089..973b867 100644 --- a/src/spice2x/CMakeLists.txt +++ b/src/spice2x/CMakeLists.txt @@ -276,6 +276,28 @@ if(NOT SPICE_XP) endif() endif() +# x264 for the API H.264 video stream, installed into the mingw sysroots by the +# deps image. The WinXP toolchains deliberately go without it and serve MJPEG only. +add_library(spice_x264 INTERFACE) +if(NOT SPICE_XP) + # search static archives only: the mingw package also ships an import library, + # and linking that one would pull in a libx264 DLL at runtime + set(SPICE_X264_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX}) + find_library(X264_LIBRARY NAMES x264 libx264) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${SPICE_X264_SUFFIXES}) + + find_path(X264_INCLUDE_DIR NAMES x264.h) + if(X264_LIBRARY AND X264_INCLUDE_DIR) + target_include_directories(spice_x264 INTERFACE "${X264_INCLUDE_DIR}") + target_link_libraries(spice_x264 INTERFACE "${X264_LIBRARY}") + target_compile_definitions(spice_x264 INTERFACE SPICE_H264=1) + else() + message(WARNING + "x264 not found: the api video stream will only offer MJPEG") + endif() +endif() + # fpng's SIMD needs the whole unit built for SSE4.1, which its runtime CPU check # cannot undo, so keep it scalar rather than raising the CPU baseline set_source_files_properties(external/fpng/fpng.cpp PROPERTIES @@ -346,6 +368,10 @@ set(SOURCE_FILES ${SOURCE_FILES} # api api/controller.cpp api/websocket.cpp + api/capture_pump.cpp + api/h264_stream.cpp + api/stream_format.cpp + api/stream_server.cpp api/request.cpp api/response.cpp api/module.cpp @@ -771,7 +797,7 @@ endfunction() add_library(spicetools_spice_objs OBJECT ${SOURCE_FILES}) target_link_libraries(spicetools_spice_objs PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg spice_x264) target_link_libraries(spicetools_spice_objs PUBLIC winscard) if(NOT MSVC) @@ -811,7 +837,7 @@ set(RESOURCE_FILES build/manifest.manifest build/manifest.rc build/icon.rc cfg/W add_executable(spicetools_spice_linux ${SOURCE_FILES} ${RESOURCE_FILES}) target_link_libraries(spicetools_spice_linux PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg spice_x264) set_target_properties(spicetools_spice_linux PROPERTIES PREFIX "") set_target_properties(spicetools_spice_linux PROPERTIES OUTPUT_NAME "spice_linux") target_compile_definitions(spicetools_spice_linux PRIVATE NO_SCARD=1 PRIVATE SPICE_LINUX=1) @@ -829,7 +855,7 @@ add_executable(spicetools_spice64 ${SOURCE_FILES} ${RESOURCE_FILES}) # do NOT link against: mf, mfplat, mfreadwrite; otherwise unity games will break target_link_libraries(spicetools_spice64 PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp mfuuid strmiids dxva2 - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg spice_x264) target_link_libraries(spicetools_spice64 PUBLIC winscard) set_target_properties(spicetools_spice64 PROPERTIES PREFIX "") set_target_properties(spicetools_spice64 PROPERTIES OUTPUT_NAME "spice64") @@ -852,7 +878,7 @@ add_executable(spicetools_spice64_linux ${SOURCE_FILES} ${RESOURCE_FILES}) # do NOT link against: mf, mfplat, mfreadwrite; otherwise unity games will break target_link_libraries(spicetools_spice64_linux PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp mfuuid strmiids dxva2 - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg spice_x264) set_target_properties(spicetools_spice64_linux PROPERTIES PREFIX "") set_target_properties(spicetools_spice64_linux PROPERTIES OUTPUT_NAME "spice64_linux") target_compile_definitions(spicetools_spice64_linux PRIVATE SPICE64=1) @@ -871,9 +897,10 @@ endif() set(SOURCE_FILES ${SOURCE_FILES} launcher/options.h launcher/options.cpp) set(RESOURCE_FILES cfg/manifest.manifest cfg/manifest.rc cfg/icon.rc cfg/Win32D.rc) add_executable(spicetools_cfg WIN32 ${SOURCE_FILES} ${RESOURCE_FILES}) +# the configurator serves neither the API nor the video stream, so it needs no codecs target_link_libraries(spicetools_cfg PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp strmiids - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features) target_link_libraries(spicetools_cfg PUBLIC winscard) set_target_properties(spicetools_cfg PROPERTIES PREFIX "") set_target_properties(spicetools_cfg PROPERTIES OUTPUT_NAME "spicecfg") @@ -891,7 +918,7 @@ set(RESOURCE_FILES cfg/manifest.manifest cfg/manifest.rc cfg/icon.rc cfg/Win32D. add_executable(spicetools_cfg_linux WIN32 ${SOURCE_FILES} ${RESOURCE_FILES}) target_link_libraries(spicetools_cfg_linux PUBLIC d3d9 ws2_32 version comctl32 shlwapi iphlpapi hid secur32 setupapi psapi winmm winhttp strmiids - PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features spice_jpeg) + PRIVATE fmt::fmt-header-only discord-rpc imgui hash-library minhook imm32 dwmapi CpuFeatures::cpu_features) set_target_properties(spicetools_cfg_linux PROPERTIES PREFIX "") set_target_properties(spicetools_cfg_linux PROPERTIES OUTPUT_NAME "spicecfg_linux") target_compile_definitions(spicetools_cfg_linux PRIVATE SPICETOOLS_SPICECFG_STANDALONE=1) diff --git a/src/spice2x/README.md b/src/spice2x/README.md index e6eae30..11f5bfd 100644 --- a/src/spice2x/README.md +++ b/src/spice2x/README.md @@ -291,6 +291,42 @@ which also means that your hex edits are applicable directly. - image_resize_set_scene(scene: int) - sets the active scene for image resize state; set to 0 to disable resize +## Video Stream + +Separate from the JSON API, spice can serve the mirrored screen as a video +stream over plain HTTP. Enable it with `-apistream`. It listens on the API port +plus two, in the same way the WebSocket server uses the API port plus one, so +`-api 1337` puts the stream on 1339. This means `-api` has to be enabled too. + +Two formats are served: + + http://host:1339/stream.mjpg JPEG frames, multipart/x-mixed-replace + http://host:1339/stream.h264 H.264 annex-b, no container + +All accept the same optional query parameters: + +- `screen` - which screen to mirror, 0-3. Defaults to the subscreen when the + game has one, otherwise the main screen. +- `fps` - frames per second, 1-60. Default 30. +- `q` - quality, 1-100. Default 70. This is the JPEG quality for `stream.mjpg` + and is mapped onto the H.264 rate factor for `stream.h264`, so the same number + does not mean the same thing for both. + +For example: + + http://host:1339/stream.h264?screen=1&fps=30&q=70 + +See the wiki for format tradeoffs, latency tuning, testing commands and client +notes. + +The stream is view only. Touch and other input still go through the JSON API, +so a companion app needs both. There is no authentication on the stream port - +anyone who can reach it can watch the screen. + +WinXP builds have no video stream. Neither encoder is compiled in, so every +endpoint returns 404, and the JSON API's JPEG screen capture is unavailable for +the same reason. + ## Native wrapper libraries Spicetools provides wrapper libraries in: Arduino, C++, Dart, and Python. Python is the only one that is fully spec compliant. diff --git a/src/spice2x/api/capture_pump.cpp b/src/spice2x/api/capture_pump.cpp new file mode 100644 index 0000000..fe75e56 --- /dev/null +++ b/src/spice2x/api/capture_pump.cpp @@ -0,0 +1,58 @@ +#include "capture_pump.h" + +#include +#include + +#include "hooks/graphics/graphics.h" + +namespace api::capture_pump { + + namespace { + + std::array CONSUMER_M; + + std::mutex CLAIMED_M; + std::array CLAIMED {}; + + bool valid_screen(int screen) { + return 0 <= screen && screen < static_cast(GRAPHICS_CAPTURE_SCREEN_NO); + } + } + + bool capture_direct(int screen, std::shared_ptr &out, int divide, + uint64_t *timestamp, int *width, int *height) { + + if (!valid_screen(screen)) { + return false; + } + + std::lock_guard lock(CONSUMER_M[screen]); + graphics_capture_trigger(screen); + return graphics_capture_receive_raw( + screen, out, divide, timestamp, width, height); + } + + bool claim_screen(int screen) { + if (!valid_screen(screen)) { + return false; + } + + std::lock_guard lock(CLAIMED_M); + + if (CLAIMED[screen]) { + return false; + } + + CLAIMED[screen] = true; + return true; + } + + void release_screen(int screen) { + if (!valid_screen(screen)) { + return; + } + + std::lock_guard lock(CLAIMED_M); + CLAIMED[screen] = false; + } +} diff --git a/src/spice2x/api/capture_pump.h b/src/spice2x/api/capture_pump.h new file mode 100644 index 0000000..b8ae9d0 --- /dev/null +++ b/src/spice2x/api/capture_pump.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +namespace api::capture_pump { + + struct Frame { + // packed 24bpp RGB, width * height * 3 bytes + std::shared_ptr pixels; + uint64_t timestamp = 0; + int width = 0; + int height = 0; + }; + + // the graphics layer has one capture slot per screen, so concurrent waiters would steal + // each other's frames; everything that captures goes through here to keep it serialized + bool capture_direct(int screen, std::shared_ptr &out, int divide, + uint64_t *timestamp = nullptr, int *width = nullptr, int *height = nullptr); + + // a screen carries one stream at a time; false when another connection already holds it + bool claim_screen(int screen); + void release_screen(int screen); +} diff --git a/src/spice2x/api/h264_stream.cpp b/src/spice2x/api/h264_stream.cpp new file mode 100644 index 0000000..7774220 --- /dev/null +++ b/src/spice2x/api/h264_stream.cpp @@ -0,0 +1,224 @@ +#include "h264_stream.h" + +#ifdef SPICE_H264 + +#include + +#include + +#include "util/logging.h" + +namespace api { + + namespace { + + // BT.601 limited range, the range every decoder assumes for H.264 without + // explicit colour metadata + inline uint8_t rgb_to_y(int r, int g, int b) { + return static_cast(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16); + } + + inline uint8_t rgb_to_u(int r, int g, int b) { + return static_cast(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128); + } + + inline uint8_t rgb_to_v(int r, int g, int b) { + return static_cast(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128); + } + + // a bare annex-b elementary stream, one encoder per connection so every client + // starts on its own keyframe. no container, so nothing here keeps a media clock + class H264Writer : public StreamWriter { + public: + + H264Writer(int quality, int fps) : quality(quality), fps(fps) {} + + ~H264Writer() override { + this->close(); + } + + std::string content_type() const override { + return "video/h264"; + } + + bool write(const StreamSend &send, const capture_pump::Frame &frame) override { + + // I420 needs even dimensions + const int width = frame.width & ~1; + const int height = frame.height & ~1; + if (width <= 0 || height <= 0) { + return true; + } + + if (this->encoder == nullptr) { + if (!this->open(width, height)) { + return false; + } + } else if (width != this->width || height != this->height) { + // the encoder is fixed at the size it opened with; let the client reconnect + log_info("api::stream", "capture size changed, ending H.264 client"); + return false; + } + + this->convert(frame.pixels.get(), frame.width); + + this->picture.i_pts = this->frame_index; + + x264_nal_t *nals = nullptr; + int nal_count = 0; + x264_picture_t picture_out; + const int size = x264_encoder_encode( + this->encoder, &nals, &nal_count, &this->picture, &picture_out); + + if (size < 0) { + log_warning("api::stream", "H.264 encode failed"); + return false; + } + + this->frame_index++; + + if (size == 0) { + return true; + } + + // x264 lays every NAL of the frame out back to back. an SEI or delimiter + // carries no picture, so only the parameter sets and the slice go through + this->annexb.clear(); + for (int i = 0; i < nal_count; i++) { + switch (nals[i].i_type) { + case NAL_SEI: + case NAL_AUD: + case NAL_FILLER: + continue; + default: + break; + } + this->annexb.insert(this->annexb.end(), + nals[i].p_payload, nals[i].p_payload + nals[i].i_payload); + } + + if (this->annexb.empty()) { + return true; + } + + return send(this->annexb.data(), this->annexb.size()); + } + + private: + + bool open(int width, int height) { + + x264_param_t param; + if (x264_param_default_preset(¶m, "ultrafast", "zerolatency") < 0) { + return false; + } + + param.i_csp = X264_CSP_I420; + param.i_width = width; + param.i_height = height; + param.i_fps_num = this->fps; + param.i_fps_den = 1; + param.i_threads = 1; + param.b_annexb = 1; + // SPS/PPS ahead of every IDR, so a client can start decoding cold + param.b_repeat_headers = 1; + // a keyframe every two seconds bounds how long a new client waits + param.i_keyint_max = this->fps * 2; + param.i_log_level = X264_LOG_NONE; + param.rc.i_rc_method = X264_RC_CRF; + param.rc.f_rf_constant = 40.0f - (this->quality * 0.25f); + + // baseline keeps hardware decode available on the widest range of phones + if (x264_param_apply_profile(¶m, "baseline") < 0) { + return false; + } + + this->encoder = x264_encoder_open(¶m); + if (this->encoder == nullptr) { + log_warning("api::stream", "could not open the H.264 encoder"); + return false; + } + + if (x264_picture_alloc(&this->picture, X264_CSP_I420, width, height) < 0) { + this->close(); + return false; + } + this->picture_ready = true; + + this->width = width; + this->height = height; + return true; + } + + void close() { + if (this->picture_ready) { + x264_picture_clean(&this->picture); + this->picture_ready = false; + } + if (this->encoder != nullptr) { + x264_encoder_close(this->encoder); + this->encoder = nullptr; + } + } + + // packed 24bpp RGB to I420, averaging each 2x2 block for the chroma planes + void convert(const uint8_t *rgb, int source_width) { + + uint8_t *plane_y = this->picture.img.plane[0]; + uint8_t *plane_u = this->picture.img.plane[1]; + uint8_t *plane_v = this->picture.img.plane[2]; + const int stride_y = this->picture.img.i_stride[0]; + const int stride_u = this->picture.img.i_stride[1]; + const int stride_v = this->picture.img.i_stride[2]; + + for (int y = 0; y < this->height; y++) { + const uint8_t *row = rgb + static_cast(y) * source_width * 3; + uint8_t *out_y = plane_y + static_cast(y) * stride_y; + + for (int x = 0; x < this->width; x++) { + const uint8_t *pixel = row + x * 3; + out_y[x] = rgb_to_y(pixel[0], pixel[1], pixel[2]); + } + } + + for (int y = 0; y < this->height / 2; y++) { + const uint8_t *row0 = rgb + static_cast(y * 2) * source_width * 3; + const uint8_t *row1 = row0 + static_cast(source_width) * 3; + uint8_t *out_u = plane_u + static_cast(y) * stride_u; + uint8_t *out_v = plane_v + static_cast(y) * stride_v; + + for (int x = 0; x < this->width / 2; x++) { + const uint8_t *p00 = row0 + (x * 2) * 3; + const uint8_t *p01 = p00 + 3; + const uint8_t *p10 = row1 + (x * 2) * 3; + const uint8_t *p11 = p10 + 3; + + const int r = (p00[0] + p01[0] + p10[0] + p11[0] + 2) / 4; + const int g = (p00[1] + p01[1] + p10[1] + p11[1] + 2) / 4; + const int b = (p00[2] + p01[2] + p10[2] + p11[2] + 2) / 4; + + out_u[x] = rgb_to_u(r, g, b); + out_v[x] = rgb_to_v(r, g, b); + } + } + } + + int quality; + int fps; + int width = 0; + int height = 0; + int64_t frame_index = 0; + std::vector annexb; + + x264_t *encoder = nullptr; + x264_picture_t picture {}; + bool picture_ready = false; + }; + } + + std::unique_ptr make_h264_writer(int quality, int fps) { + return std::make_unique(quality, fps); + } +} + +#endif // SPICE_H264 diff --git a/src/spice2x/api/h264_stream.h b/src/spice2x/api/h264_stream.h new file mode 100644 index 0000000..494b0a5 --- /dev/null +++ b/src/spice2x/api/h264_stream.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#include "stream_format.h" + +namespace api { + + // bare annex-b H.264; null when the build has no encoder + std::unique_ptr make_h264_writer(int quality, int fps); +} diff --git a/src/spice2x/api/modules/capture.cpp b/src/spice2x/api/modules/capture.cpp index ed96937..b8ab2b1 100644 --- a/src/spice2x/api/modules/capture.cpp +++ b/src/spice2x/api/modules/capture.cpp @@ -2,8 +2,10 @@ #include #include #include +#include "api/capture_pump.h" #include "external/rapidjson/document.h" #include "hooks/graphics/graphics.h" +#include "hooks/graphics/jpeg_encoder.h" #include "util/crypt.h" using namespace std::placeholders; @@ -118,9 +120,16 @@ namespace api::modules { uint64_t timestamp = 0; int width = 0; int height = 0; - graphics_capture_trigger(screen); - bool success = graphics_capture_receive_jpeg( - screen, CAPTURE_BUFFER, quality, divide, ×tamp, &width, &height); + + std::shared_ptr pixels; + bool success = capture_pump::capture_direct( + screen, pixels, divide, ×tamp, &width, &height); + + if (success) { + CAPTURE_BUFFER.clear(); + success = jpeg_encoder::encode( + CAPTURE_BUFFER, pixels.get(), width, height, quality); + } if (success) { add_jpeg_response(screen, timestamp, width, height, CAPTURE_BUFFER, res); diff --git a/src/spice2x/api/stream_format.cpp b/src/spice2x/api/stream_format.cpp new file mode 100644 index 0000000..0fce3ce --- /dev/null +++ b/src/spice2x/api/stream_format.cpp @@ -0,0 +1,70 @@ +#include "stream_format.h" + +#include + +#include "h264_stream.h" +#include "hooks/graphics/jpeg_encoder.h" + +namespace api { + + namespace { + +#ifdef SPICE_JPEG + constexpr const char *MJPEG_BOUNDARY = "spice2xframe"; + + // multipart/x-mixed-replace: every frame is a standalone JPEG, no inter-frame state + class MjpegWriter : public StreamWriter { + public: + + explicit MjpegWriter(int quality) : quality(quality) {} + + std::string content_type() const override { + return std::string("multipart/x-mixed-replace; boundary=") + MJPEG_BOUNDARY; + } + + bool write(const StreamSend &send, const capture_pump::Frame &frame) override { + this->jpeg.clear(); + if (!jpeg_encoder::encode( + this->jpeg, frame.pixels.get(), + frame.width, frame.height, this->quality)) { + // a frame the encoder rejects is not worth dropping the client over + return true; + } + + const std::string part = + "--" + std::string(MJPEG_BOUNDARY) + "\r\n" + "Content-Type: image/jpeg\r\n" + "Content-Length: " + std::to_string(this->jpeg.size()) + "\r\n" + "\r\n"; + + return send(part.data(), part.size()) + && send(this->jpeg.data(), this->jpeg.size()) + && send("\r\n", 2); + } + + private: + int quality; + std::vector jpeg; + }; +#endif + } + + // both parameters go unused on toolchains that compile in neither format + std::unique_ptr make_stream_writer( + const std::string &path, [[maybe_unused]] int quality, [[maybe_unused]] int fps) { + +#ifdef SPICE_JPEG + if (path == "/stream.mjpg") { + return std::make_unique(quality); + } +#endif + +#ifdef SPICE_H264 + if (path == "/stream.h264") { + return make_h264_writer(quality, fps); + } +#endif + + return nullptr; + } +} diff --git a/src/spice2x/api/stream_format.h b/src/spice2x/api/stream_format.h new file mode 100644 index 0000000..ed04fb9 --- /dev/null +++ b/src/spice2x/api/stream_format.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +#include "capture_pump.h" + +namespace api { + + // writes bytes to the client; false once the connection is gone + using StreamSend = std::function; + + // one wire format, instantiated per connection so it can keep encoder state across frames + class StreamWriter { + public: + virtual ~StreamWriter() = default; + + StreamWriter(const StreamWriter &) = delete; + StreamWriter &operator=(const StreamWriter &) = delete; + + // value for the HTTP Content-Type response header + virtual std::string content_type() const = 0; + + // for formats that open with an init segment; runs once before any frame + virtual bool begin(const StreamSend &send) { return true; } + + virtual bool write(const StreamSend &send, const capture_pump::Frame &frame) = 0; + + protected: + StreamWriter() = default; + }; + + // null when the path does not name a format this build supports + std::unique_ptr make_stream_writer( + const std::string &path, int quality, int fps); +} diff --git a/src/spice2x/api/stream_server.cpp b/src/spice2x/api/stream_server.cpp new file mode 100644 index 0000000..a0769b2 --- /dev/null +++ b/src/spice2x/api/stream_server.cpp @@ -0,0 +1,427 @@ +#include +#include + +#include "stream_server.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "capture_pump.h" +#include "hooks/graphics/graphics.h" +#include "stream_format.h" +#include "util/logging.h" +#include "util/utils.h" + +namespace api { + + namespace { + + struct HttpRequest { + std::string method; + std::string path; + std::map query; + }; + + bool send_all(SOCKET socket, const void *data, size_t size) { + auto cursor = reinterpret_cast(data); + size_t remaining = size; + + while (remaining > 0) { + const int sent = send(socket, cursor, static_cast(remaining), 0); + if (sent <= 0) { + return false; + } + cursor += sent; + remaining -= static_cast(sent); + } + + return true; + } + + bool send_all(SOCKET socket, const std::string &text) { + return send_all(socket, text.data(), text.size()); + } + + std::string url_decode(const std::string &input) { + std::string out; + out.reserve(input.size()); + + for (size_t i = 0; i < input.size(); i++) { + if (input[i] == '+') { + out.push_back(' '); + } else if (input[i] == '%' && i + 2 < input.size() + && isxdigit(static_cast(input[i + 1])) + && isxdigit(static_cast(input[i + 2]))) { + out.push_back(static_cast( + std::stoi(input.substr(i + 1, 2), nullptr, 16))); + i += 2; + } else { + out.push_back(input[i]); + } + } + + return out; + } + + void parse_query(const std::string &query, HttpRequest &request) { + size_t pos = 0; + + while (pos < query.size()) { + auto end = query.find('&', pos); + if (end == std::string::npos) { + end = query.size(); + } + + const auto pair = query.substr(pos, end - pos); + const auto split = pair.find('='); + if (split != std::string::npos && split > 0) { + request.query[url_decode(pair.substr(0, split))] = + url_decode(pair.substr(split + 1)); + } + + pos = end + 1; + } + } + + // reads the request head only; anything oversized or malformed is refused + bool read_request(SOCKET socket, size_t size_limit, HttpRequest &request) { + std::string head; + char buffer[1024]; + + while (head.find("\r\n\r\n") == std::string::npos) { + if (head.size() >= size_limit) { + return false; + } + + // read no further than the limit, so the head cannot overshoot it + const size_t budget = std::min(sizeof(buffer), size_limit - head.size()); + const int received = recv(socket, buffer, static_cast(budget), 0); + if (received <= 0) { + return false; + } + + head.append(buffer, static_cast(received)); + } + + const auto line_end = head.find("\r\n"); + const auto line = head.substr(0, line_end); + + const auto method_end = line.find(' '); + if (method_end == std::string::npos) { + return false; + } + + const auto target_end = line.find(' ', method_end + 1); + if (target_end == std::string::npos) { + return false; + } + + request.method = line.substr(0, method_end); + auto target = line.substr(method_end + 1, target_end - method_end - 1); + + const auto query_start = target.find('?'); + if (query_start != std::string::npos) { + parse_query(target.substr(query_start + 1), request); + target = target.substr(0, query_start); + } + + request.path = url_decode(target); + return true; + } + + int query_int(const HttpRequest &request, const std::string &name, int fallback, + int min, int max) { + + const auto pos = request.query.find(name); + if (pos == request.query.end()) { + return fallback; + } + + try { + return std::clamp(std::stoi(pos->second), min, max); + } catch (const std::exception &) { + return fallback; + } + } + + void send_error(SOCKET socket, const char *status) { + const std::string response = + std::string("HTTP/1.0 ") + status + "\r\n" + "Content-Length: 0\r\n" + "Connection: close\r\n" + "\r\n"; + send_all(socket, response); + } + } + + StreamServer::StreamServer(unsigned short port) + : port(port) + { + if (!this->open_listener()) { + // the stream was asked for explicitly, so say plainly that it is not there + log_warning("api::stream", + "the video stream is not available on port {}", this->port); + return; + } + + this->running = true; + this->acceptor = std::thread([this] { + this->accept_worker(); + }); + + // deliberately not logging a full URL; local IPs would leak into shared logs + log_info("api::stream", "video stream is listening on port: {}", this->port); + log_warning("api::stream", + "the video stream is unauthenticated - anyone who can reach port {} can watch " + "the game screen", this->port); + } + + bool StreamServer::open_listener() { + WSADATA wsa_data; + const int error = WSAStartup(MAKEWORD(2, 2), &wsa_data); + if (error != 0) { + log_warning("api::stream", "WSAStartup() returned {}", error); + return false; + } + this->wsa_started = true; + + this->listener = socket(AF_INET, SOCK_STREAM, 0); + if (this->listener == INVALID_SOCKET) { + log_warning("api::stream", "could not create listener socket: {}", + get_last_error_string()); + return false; + } + + int opt_enable = 1; + if (setsockopt(this->listener, SOL_SOCKET, SO_REUSEADDR, + reinterpret_cast(&opt_enable), sizeof(int)) == -1) { + log_warning("api::stream", "could not set socket option SO_REUSEADDR: {}", + get_last_error_string()); + } + + sockaddr_in server_address {}; + server_address.sin_family = AF_INET; + server_address.sin_port = htons(this->port); + server_address.sin_addr.s_addr = INADDR_ANY; + + if (bind(this->listener, (sockaddr *) &server_address, sizeof(sockaddr)) == -1) { + log_warning("api::stream", "could not bind socket on port {}: {}", + this->port, get_last_error_string()); + closesocket(this->listener); + this->listener = INVALID_SOCKET; + return false; + } + + if (listen(this->listener, server_backlog) == -1) { + log_warning("api::stream", "could not listen on port {}: {}", + this->port, get_last_error_string()); + closesocket(this->listener); + this->listener = INVALID_SOCKET; + return false; + } + + return true; + } + + StreamServer::~StreamServer() { + + this->running = false; + + if (this->listener != INVALID_SOCKET) { + closesocket(this->listener); + this->listener = INVALID_SOCKET; + } + + // drops the client threads out of their blocking send/recv + { + std::lock_guard lock(this->clients_m); + for (auto &client : this->clients) { + if (client.socket != INVALID_SOCKET) { + ::shutdown(client.socket, SD_BOTH); + } + } + } + + if (this->acceptor.joinable()) { + this->acceptor.join(); + } + + // joining is what guarantees no client thread outlives this object + for (auto &client : this->clients) { + if (client.thread.joinable()) { + client.thread.join(); + } + } + + if (this->wsa_started) { + WSACleanup(); + } + } + + void StreamServer::accept_worker() { + + while (this->running) { + sockaddr_in client_address {}; + int client_address_size = sizeof(sockaddr_in); + + const SOCKET client = accept( + this->listener, (sockaddr *) &client_address, &client_address_size); + if (client == INVALID_SOCKET) { + // on shutdown the listener is closed under us; otherwise do not spin + if (this->running) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + continue; + } + + if (!this->running) { + closesocket(client); + break; + } + + char address_data[INET_ADDRSTRLEN] {}; + inet_ntop(AF_INET, &client_address.sin_addr, address_data, INET_ADDRSTRLEN); + std::string address(address_data); + + // every client costs an encode and real bandwidth, so the cap protects the game + int slot = -1; + { + std::lock_guard lock(this->clients_m); + for (size_t i = 0; i < this->clients.size(); i++) { + if (!this->clients[i].active) { + this->clients[i].active = true; + this->clients[i].socket = client; + slot = static_cast(i); + break; + } + } + } + + if (slot < 0) { + log_warning("api::stream", "client limit of {} hit", client_limit); + send_error(client, "503 Service Unavailable"); + closesocket(client); + continue; + } + + // this thread is the only one that touches the thread objects, so the slot's + // previous occupant gets reaped here rather than being detached + if (this->clients[slot].thread.joinable()) { + this->clients[slot].thread.join(); + } + + this->clients[slot].thread = std::thread([this, slot, client, address] { + this->client_worker(slot, client, address); + }); + } + } + + void StreamServer::client_worker(int slot, SOCKET socket, std::string address) { + + DWORD timeout = request_timeout_ms; + setsockopt(socket, SOL_SOCKET, SO_RCVTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); + + timeout = send_timeout_ms; + setsockopt(socket, SOL_SOCKET, SO_SNDTIMEO, + reinterpret_cast(&timeout), sizeof(timeout)); + + int opt_enable = 1; + setsockopt(socket, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast(&opt_enable), sizeof(int)); + + // whatever sits in the send buffer is already stale, and the default holds about a + // third of a second of H.264 because the bitrate is so low. keeping it small makes a + // slow reader block the sender, which then skips to the newest frame instead of + // handing over a backlog + int send_buffer = send_buffer_bytes; + setsockopt(socket, SOL_SOCKET, SO_SNDBUF, + reinterpret_cast(&send_buffer), sizeof(send_buffer)); + + HttpRequest request; + if (read_request(socket, request_size_limit, request)) { + if (request.method != "GET") { + send_error(socket, "405 Method Not Allowed"); + } else { + const int fps = query_int(request, "fps", 30, 1, fps_limit); + const int quality = query_int(request, "q", 70, 1, 100); + + auto writer = make_stream_writer(request.path, quality, fps); + if (!writer) { + send_error(socket, "404 Not Found"); + } else { + // screen 1 is the subscreen in every game that has one; single-screen games + // only ever register screen 0, so resolve the default against what exists + int screen = query_int(request, "screen", -1, 0, + static_cast(GRAPHICS_CAPTURE_SCREEN_NO) - 1); + if (screen < 0) { + std::vector screens; + graphics_screens_get(screens); + screen = std::find(screens.begin(), screens.end(), 1) != screens.end() + ? 1 : 0; + } + + if (!capture_pump::claim_screen(screen)) { + log_warning("api::stream", + "screen {} is already being streamed, refusing {}", + screen, address); + send_error(socket, "503 Service Unavailable"); + } else { + log_info("api::stream", + "client connected: {} ({}, screen={}, fps={}, quality={})", + address, request.path, screen, fps, quality); + + const std::string header = + "HTTP/1.0 200 OK\r\n" + "Connection: close\r\n" + "Cache-Control: no-store, no-cache, must-revalidate\r\n" + "Pragma: no-cache\r\n" + "Content-Type: " + writer->content_type() + "\r\n" + "\r\n"; + + const StreamSend stream_send = [socket](const void *data, size_t size) { + return send_all(socket, data, size); + }; + + if (send_all(socket, header) && writer->begin(stream_send)) { + const auto interval = std::chrono::microseconds(1000000 / fps); + + while (this->running) { + const auto started = std::chrono::steady_clock::now(); + + capture_pump::Frame frame; + const bool ok = capture_pump::capture_direct( + screen, frame.pixels, 1, + &frame.timestamp, &frame.width, &frame.height); + + if (ok && frame.pixels + && !writer->write(stream_send, frame)) { + break; + } + + // a failed capture still paces, or a stalled game spins this + std::this_thread::sleep_until(started + interval); + } + } + + capture_pump::release_screen(screen); + log_info("api::stream", "client disconnected: {}", address); + } + } + } + } + + { + std::lock_guard lock(this->clients_m); + this->clients[slot].socket = INVALID_SOCKET; + this->clients[slot].active = false; + } + + closesocket(socket); + } +} diff --git a/src/spice2x/api/stream_server.h b/src/spice2x/api/stream_server.h new file mode 100644 index 0000000..0e2646e --- /dev/null +++ b/src/spice2x/api/stream_server.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace api { + + class StreamServer { + public: + + explicit StreamServer(unsigned short port); + ~StreamServer(); + + StreamServer(const StreamServer &) = delete; + StreamServer &operator=(const StreamServer &) = delete; + + private: + + // configuration + static constexpr int server_backlog = 4; + static constexpr int client_limit = 4; + static constexpr int request_size_limit = 8 * 1024; + static constexpr int request_timeout_ms = 5000; + static constexpr int send_timeout_ms = 5000; + // small enough that a low bitrate stream cannot hide a backlog of stale frames in it + static constexpr int send_buffer_bytes = 16 * 1024; + static constexpr int fps_limit = 60; + + struct Client { + std::thread thread; + SOCKET socket = INVALID_SOCKET; + bool active = false; + }; + + void accept_worker(); + bool open_listener(); + void client_worker(int slot, SOCKET socket, std::string address); + + unsigned short port; + SOCKET listener = INVALID_SOCKET; + bool wsa_started = false; + std::atomic_bool running { false }; + std::thread acceptor; + std::mutex clients_m; + // socket and active are guarded by clients_m; only the acceptor touches thread + std::array clients; + }; +} diff --git a/src/spice2x/external/docker/Dockerfile b/src/spice2x/external/docker/Dockerfile index 2ed550e..5032225 100644 --- a/src/spice2x/external/docker/Dockerfile +++ b/src/spice2x/external/docker/Dockerfile @@ -27,4 +27,8 @@ RUN curl -fsSL "https://github.com/mon/windows-dll-compat-checker/releases/downl # libjpeg-turbo for JPEG encoding. Only the mingw-w64 toolchains get it; # the WinXP targets build without JPEG support entirely. -RUN su user -c "yay --noconfirm -S mingw-w64-libjpeg-turbo" \ No newline at end of file +RUN su user -c "yay --noconfirm -S mingw-w64-libjpeg-turbo" + +# x264 for the API H.264 video stream. Only the mingw-w64 toolchains need it; +# the WinXP targets build without the stream encoder. +RUN su user -c "yay --noconfirm -S mingw-w64-x264" diff --git a/src/spice2x/hooks/graphics/graphics.cpp b/src/spice2x/hooks/graphics/graphics.cpp index 04214e6..6aa5866 100644 --- a/src/spice2x/hooks/graphics/graphics.cpp +++ b/src/spice2x/hooks/graphics/graphics.cpp @@ -1481,11 +1481,11 @@ void graphics_capture_skip(int screen) { GRAPHICS_CAPTURE_CV[screen].notify_one(); } -bool graphics_capture_receive_jpeg(int screen, std::vector &out, - int quality, int divide, uint64_t *timestamp, +bool graphics_capture_receive_raw(int screen, std::shared_ptr &out, + int divide, uint64_t *timestamp, int *width, int *height) { - out.clear(); + out = nullptr; if (screen < 0 || screen >= static_cast(GRAPHICS_CAPTURE_SCREEN_NO)) { return false; @@ -1558,9 +1558,39 @@ bool graphics_capture_receive_jpeg(int screen, std::vector &out, capture_height = height_new; } + out = std::move(capture_data); + + // status + if (timestamp) { + *timestamp = capture_timestamp; + } + if (width) { + *width = capture_width; + } + if (height) { + *height = capture_height; + } + + return true; +} + +bool graphics_capture_receive_jpeg(int screen, std::vector &out, + int quality, int divide, uint64_t *timestamp, + int *width, int *height) { + + out.clear(); + + std::shared_ptr pixels; + int capture_width = 0; + int capture_height = 0; + if (!graphics_capture_receive_raw( + screen, pixels, divide, timestamp, &capture_width, &capture_height)) { + return false; + } + // compress const bool success = jpeg_encoder::encode( - out, capture_data.get(), + out, pixels.get(), capture_width, capture_height, quality); if (!success) { @@ -1568,9 +1598,6 @@ bool graphics_capture_receive_jpeg(int screen, std::vector &out, } // status - if (timestamp) { - *timestamp = capture_timestamp; - } if (width) { *width = capture_width; } diff --git a/src/spice2x/hooks/graphics/graphics.h b/src/spice2x/hooks/graphics/graphics.h index f88bb79..4d85ed8 100644 --- a/src/spice2x/hooks/graphics/graphics.h +++ b/src/spice2x/hooks/graphics/graphics.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -145,6 +146,11 @@ void graphics_capture_trigger(int screen); bool graphics_capture_consume(int *screen); void graphics_capture_enqueue(int screen, uint8_t *data, size_t width, size_t height); void graphics_capture_skip(int screen); +// on success `out` owns packed 24bpp RGB pixels, width * height * 3 bytes +bool graphics_capture_receive_raw(int screen, std::shared_ptr &out, + int divide = 0, + uint64_t *timestamp = nullptr, + int *width = nullptr, int *height = nullptr); // on success `out` holds the encoded JPEG; its storage is reused across calls bool graphics_capture_receive_jpeg(int screen, std::vector &out, int quality = 80, int divide = 0, diff --git a/src/spice2x/launcher/launcher.cpp b/src/spice2x/launcher/launcher.cpp index e47809e..b7f5b58 100644 --- a/src/spice2x/launcher/launcher.cpp +++ b/src/spice2x/launcher/launcher.cpp @@ -14,6 +14,7 @@ #include "acio/icca/icca.h" #include "acio/mdxf/mdxf.h" #include "api/controller.h" +#include "api/stream_server.h" #include "avs/automap.h" #include "avs/core.h" #include "avs/ea3.h" @@ -150,6 +151,7 @@ std::string CARD_OVERRIDES[2]; // sub-systems std::unique_ptr API_CONTROLLER; +std::unique_ptr API_STREAM_SERVER; std::unique_ptr RI_MGR; // trigger NVIDIA Optimus & AMD Enduro High Performance Graphics @@ -198,6 +200,7 @@ int main_implementation(int argc, char *argv[]) { bool api_pretty = false; bool api_debug = false; unsigned short api_port = 1337; + bool api_stream_enable = false; std::string api_pass = ""; std::vector api_serial_port; std::vector api_serial_baud; @@ -1032,6 +1035,9 @@ int main_implementation(int argc, char *argv[]) { if (options[launcher::Options::APIScreenMirrorDivide].is_active()) { api::modules::CAPTURE_DIVIDE = options[launcher::Options::APIScreenMirrorDivide].value_uint32(); } + if (options[launcher::Options::APIStreamEnable].value_bool() && !cfg::CONFIGURATOR_STANDALONE) { + api_stream_enable = true; + } if (options[launcher::Options::DisableDebugHooks].value_bool()) { debughook::DEBUGHOOK_LOGGING = false; @@ -2731,6 +2737,20 @@ int main_implementation(int argc, char *argv[]) { for (size_t i = 0; i < std::min(api_serial_port.size(), api_serial_baud.size()); i++) { API_CONTROLLER->listen_serial(api_serial_port[i], api_serial_baud[i]); } + // the websocket already sits on the API port plus one, so the stream takes plus two + if (api_stream_enable) { + if (!api_enable) { + log_fatal("launcher", "video stream requires API port to be set (-api)"); + } else if (api_port + 2 > 65535) { + log_fatal( + "launcher", + "ignoring the video stream, API port {} leaves no room for port plus two", + api_port); + } else { + API_STREAM_SERVER = std::make_unique( + static_cast(api_port + 2)); + } + } // pin macro if (!cfg::CONFIGURATOR_STANDALONE && PIN_MACRO_ENABLED) { @@ -2819,6 +2839,7 @@ int main_implementation(int argc, char *argv[]) { } // free api controller + API_STREAM_SERVER.reset(); API_CONTROLLER.reset(); eamuse_pin_macro_stop_thread(); diff --git a/src/spice2x/launcher/options.cpp b/src/spice2x/launcher/options.cpp index be55c36..de7beab 100644 --- a/src/spice2x/launcher/options.cpp +++ b/src/spice2x/launcher/options.cpp @@ -1743,6 +1743,21 @@ static const std::vector OPTION_DEFINITIONS = { .setting_name = "1", .category = "Companion & API", }, + { + // APIStreamEnable + .title = "API Video Stream Server Enable", + .name = "apistream", + .desc = "Serves the mirrored screen as a video stream, on the API port plus two; " + "alternative to API screen capture. Requires -api.\n\n" + "http://host:apiport+2/stream.mjpg - MJPEG\n\n" + "http://host:apiport+2/stream.h264 - H.264\n\n" + "Parameters: screen (0-3), fps (1-60, default 30), q (1-100, default 70).\n\n" + "Example with -api 1337: http://host:1339/stream.h264?fps=30&q=70\n\n" + "VIEW ONLY - touch input still requires -api. " + "No password protection or encryption of any kind; video sent in the clear!", + .type = OptionType::Bool, + .category = "Companion & API", + }, { .title = "Enable All IO Modules", .name = "io", diff --git a/src/spice2x/launcher/options.h b/src/spice2x/launcher/options.h index c14a061..24ff56a 100644 --- a/src/spice2x/launcher/options.h +++ b/src/spice2x/launcher/options.h @@ -170,6 +170,7 @@ namespace launcher { APIDebugMode, APIScreenMirrorQuality, APIScreenMirrorDivide, + APIStreamEnable, EnableAllIOModules, EnableACIOModule, EnableICCAModule, diff --git a/src/spice2x/licenses.txt b/src/spice2x/licenses.txt index ea32d87..e552c1f 100644 --- a/src/spice2x/licenses.txt +++ b/src/spice2x/licenses.txt @@ -1406,3 +1406,23 @@ Contributions ------------------------------------------- cardio - Felix - MIT License scard - nolm - MIT License + +x264 (GPL-2.0-or-later) +------------------------------------------- +https://www.videolan.org/developers/x264.html +Statically linked for the API video stream H.264 encoder. + +Copyright (C) 2003-2024 x264 project + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110-1301, USA. \ No newline at end of file