From 5eb44307eec5f06076f1d65f6263f68698688b60 Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Tue, 10 Nov 2026 08:21:55 +0100 Subject: [PATCH] Feature: Capture the virtual desktop for multi-monitor remote control The MCP Observe/Act loop requires screenshots and mouse injection to share one coordinate space. Previously the client sent only the primary monitor while injection used virtual-desktop coordinates, so multi-monitor hosts were rejected outright. - Client: COMMAND_SCREEN_SPY param buffer[11] (wire index 12) now forces whole-virtual-desktop GDI capture; preview JPEGs also cover the virtual desktop including negative origins. - Server: advertise CLIENT_CAP_VIRTUAL_DESKTOP, gate multi-monitor remote control on that capability, and pass the flag when starting the session. Linux stays exempt (X root capture/injection are already consistent). --- client/ScreenCapture.h | 11 +++++-- client/ScreenManager.cpp | 15 ++++++++-- client/ScreenPreview.cpp | 22 +++++++------- client/ScreenPreview.h | 6 ++-- client/ScreenSpy.cpp | 8 +++-- client/ScreenSpy.h | 3 +- common/commands.h | 3 ++ server/2015Remote/McpServer.cpp | 51 ++++++++++++++++++-------------- server/2015Remote/WebService.cpp | 7 ++++- server/2015Remote/WebService.h | 6 ++-- server/2015Remote/context.h | 9 ++++++ 11 files changed, 93 insertions(+), 48 deletions(-) diff --git a/client/ScreenCapture.h b/client/ScreenCapture.h index 6c06fda..5eee6d2 100644 --- a/client/ScreenCapture.h +++ b/client/ScreenCapture.h @@ -176,6 +176,7 @@ public: static int& MonitorNext() { static int v = 0; return v; } // 下一次"切换显示器"要去的屏 static void CommitMonitorIndex(int idx) { MonitorCur() = idx; MonitorNext() = idx + 1; } int m_nMonitorIndex = 0; // 本次构造选中的屏;后端初始化成功后由它提交 + bool m_bForceVirtualDesktop = false; // 外部强制整虚拟桌面(远程控制),不参与屏轮转 // 滚动检测相关 CScrollDetector* m_pScrollDetector; // 滚动检测器 @@ -221,8 +222,10 @@ protected: int m_nVScreenHeight = GetSystemMetrics(SM_CYVIRTUALSCREEN); public: + // forceVirtual:强制抓取整个虚拟桌面(屏索引 0),不参与「切换显示器」轮转。 + // 远程控制会话需要截图与注入共用同一坐标系,多显示器下必须整桌面抓屏。 ScreenCapture(int n = 32, BYTE algo = ALGORITHM_DIFF, BOOL all = FALSE, int level = LEVEL_H264_SOFT, - RECT rc = {0}, BOOL switchScreen = TRUE) : + RECT rc = {0}, BOOL switchScreen = TRUE, BOOL forceVirtual = FALSE) : m_ThreadPool(nullptr), m_FirstBuffer(nullptr), m_RectBuffer(nullptr), m_BitmapInfor_Full(nullptr), m_bAlgorithm(algo), m_SendQuality(100), m_ulFullWidth(0), m_ulFullHeight(0), m_bZoomed(false), m_wZoom(1), m_hZoom(1), @@ -232,6 +235,7 @@ public: m_bLastFrameWasScroll(false), m_nScrollDetectInterval(1), m_EncodeLevel(level) { SetAlgorithm(algo); + m_bForceVirtualDesktop = (forceVirtual != FALSE); m_ROI = rc; m_BitmapInfor_Send = nullptr; m_BmpZoomBuffer = nullptr; @@ -247,7 +251,10 @@ public: m_nScreenCount = monitors.size(); m_bEnableMultiScreen = all; if (all && !monitors.empty()) { - int idx = (switchScreen ? MonitorNext() : MonitorCur()) % (int)(monitors.size()+1); + // forceVirtual 时恒取 0(虚拟桌面),也不推进 MonitorNext:这个选择由 + // COMMAND_SCREEN_SPY 每次会话显式给出,不该影响人类的屏轮转顺序。 + int idx = forceVirtual ? 0 + : (switchScreen ? MonitorNext() : MonitorCur()) % (int)(monitors.size()+1); m_nMonitorIndex = idx; if (idx == 0) { m_iScreenX = GetSystemMetrics(SM_XVIRTUALSCREEN); diff --git a/client/ScreenManager.cpp b/client/ScreenManager.cpp index d5e5ae5..79f2be3 100644 --- a/client/ScreenManager.cpp +++ b/client/ScreenManager.cpp @@ -508,6 +508,7 @@ void CScreenManager::InitScreenSpy() BYTE algo = ALGORITHM_DIFF; BYTE* user = (BYTE*)m_ptrUser; BOOL all = FALSE; + BOOL forceVirtual = FALSE; RECT rect = m_ROI; // 「切换显示器」是一次性请求:取用即清除。之后的自适应重启(质量档、分辨率、 // ROI、换窗口)一律 switchScreen=false,沿用当前屏。不清除的话,工作线程里 @@ -521,6 +522,9 @@ void CScreenManager::InitScreenSpy() DXGI = param->buffer[0]; algo = param->length > 1 ? param->buffer[1] : algo; all = param->length > 2 ? param->buffer[2] : all; + // buffer[11](wire 索引 12,HWND 占 buffer[3..10]): + // 强制抓取整个虚拟桌面(远程控制会话用,多显示器下截图/注入同坐标系) + forceVirtual = param->length > 11 ? (param->buffer[11] != 0) : FALSE; // buffer[3..10]: HWND(uint64_t),启动时直接指定窗口捕获; // 值为 (uint64_t)-1 时进入动态前景模式,每帧自动跟踪当前前景窗口 if (param->length >= 3 + (int)sizeof(uint64_t) && !m_hTargetWnd && !m_bDynamicForeground) { @@ -539,6 +543,13 @@ void CScreenManager::InitScreenSpy() } else { DXGI = (int)user; } + // 强制整虚拟桌面(远程控制):DXGI 桌面复制按输出抓取,拿不到跨屏整桌面, + // 只能走 GDI;all=true 让基类进入虚拟桌面几何分支(原点可能为负)。 + if (forceVirtual) { + all = TRUE; + DXGI = USING_GDI; + Mprintf("CScreenManager: force virtual desktop capture (GDI)\n"); + } // 如果已设置质量等级,使用对应的算法(优先于启动参数) int level = m_ScreenSettings.QualityLevel; if (level >= 0 && level < QUALITY_COUNT) { @@ -615,12 +626,12 @@ void CScreenManager::InitScreenSpy() } else { SAFE_DELETE(s); m_isGDI = TRUE; - m_ScreenSpyObject = new CScreenSpy(32, algo, FALSE, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel, rect, switchScreen, m_hTargetWnd, m_bDynamicForeground); + m_ScreenSpyObject = new CScreenSpy(32, algo, FALSE, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel, rect, switchScreen, m_hTargetWnd, m_bDynamicForeground, forceVirtual); Mprintf("CScreenManager: DXGI SPY init failed!!! Using GDI instead.\n"); } } else { m_isGDI = TRUE; - m_ScreenSpyObject = new CScreenSpy(32, algo, DXGI == USING_VIRTUAL, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel, rect, switchScreen, m_hTargetWnd, m_bDynamicForeground); + m_ScreenSpyObject = new CScreenSpy(32, algo, DXGI == USING_VIRTUAL, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel, rect, switchScreen, m_hTargetWnd, m_bDynamicForeground, forceVirtual); } // 用已保存的质量配置初始化码率,避免 CMD_QUALITY_LEVEL 到达时 0→3000 触发不必要重启 if (m_ScreenSpyObject && level >= 0 && level < QUALITY_COUNT) { diff --git a/client/ScreenPreview.cpp b/client/ScreenPreview.cpp index 52e0194..9de8bdf 100644 --- a/client/ScreenPreview.cpp +++ b/client/ScreenPreview.cpp @@ -46,19 +46,19 @@ int GetJpegEncoderClsid(CLSID& clsid) return -1; } -// 抓主屏到 24bpp Bitmap,目标尺寸已等比换算。 +// 抓整个虚拟桌面到 24bpp Bitmap,目标尺寸已等比换算。 +// 虚拟桌面原点不一定是 (0,0):主屏左侧/上方存在显示器时 SM_X/YVIRTUALSCREEN 为负, +// 必须作为 StretchBlt 的源点,否则抓到的只是主屏那一块(与注入坐标系不一致)。 // 返回新分配的 Bitmap*,失败返回 nullptr。调用者负责 delete。 -Bitmap* GrabPrimaryScaled(int targetW, int targetH) +Bitmap* GrabDesktopScaled(int targetW, int targetH) { HDC hScreen = GetDC(NULL); if (!hScreen) return nullptr; - int srcX = GetSystemMetrics(SM_XVIRTUALSCREEN); // 主屏左上 — 仅取主屏时用 0,0 + int srcX = GetSystemMetrics(SM_XVIRTUALSCREEN); int srcY = GetSystemMetrics(SM_YVIRTUALSCREEN); - (void)srcX; (void)srcY; - - int srcW = GetSystemMetrics(SM_CXSCREEN); - int srcH = GetSystemMetrics(SM_CYSCREEN); + int srcW = GetSystemMetrics(SM_CXVIRTUALSCREEN); + int srcH = GetSystemMetrics(SM_CYVIRTUALSCREEN); if (srcW <= 0 || srcH <= 0) { ReleaseDC(NULL, hScreen); return nullptr; @@ -79,7 +79,7 @@ Bitmap* GrabPrimaryScaled(int targetW, int targetH) SetStretchBltMode(hMem, HALFTONE); SetBrushOrgEx(hMem, 0, 0, NULL); BOOL bb = StretchBlt(hMem, 0, 0, targetW, targetH, - hScreen, 0, 0, srcW, srcH, SRCCOPY | CAPTUREBLT); + hScreen, srcX, srcY, srcW, srcH, SRCCOPY | CAPTUREBLT); SelectObject(hMem, oldBmp); @@ -117,8 +117,8 @@ int CaptureAndEncodePreview(int maxWidth, int quality, if (quality < 1) quality = 1; if (quality > 100) quality = 100; - int srcW = GetSystemMetrics(SM_CXSCREEN); - int srcH = GetSystemMetrics(SM_CYSCREEN); + int srcW = GetSystemMetrics(SM_CXVIRTUALSCREEN); + int srcH = GetSystemMetrics(SM_CYVIRTUALSCREEN); if (srcW <= 0 || srcH <= 0) return SCREEN_PREVIEW_CAPTURE_FAILED; // 等比缩放,禁止放大 @@ -131,7 +131,7 @@ int CaptureAndEncodePreview(int maxWidth, int quality, if (targetW < 2) targetW = 2; if (targetH < 2) targetH = 2; - Bitmap* bmp = GrabPrimaryScaled(targetW, targetH); + Bitmap* bmp = GrabDesktopScaled(targetW, targetH); if (!bmp) return SCREEN_PREVIEW_CAPTURE_FAILED; CLSID clsid; diff --git a/client/ScreenPreview.h b/client/ScreenPreview.h index 471cde6..53aca6e 100644 --- a/client/ScreenPreview.h +++ b/client/ScreenPreview.h @@ -1,11 +1,11 @@ // ScreenPreview.h -// 屏幕预览:抓主屏 → 等比缩放 → JPEG 编码,供 COMMAND_SCREEN_PREVIEW_REQ 响应使用。 +// 屏幕预览:抓整个虚拟桌面 → 等比缩放 → JPEG 编码,供 COMMAND_SCREEN_PREVIEW_REQ 响应使用。 #pragma once #include -// 抓取主屏并编码成 JPEG。 -// maxWidth 服务端期望的宽度;客户端按源屏宽度等比缩放,不会强制放大 +// 抓取整个虚拟桌面并编码成 JPEG。 +// maxWidth 服务端期望的宽度;客户端按源图宽度等比缩放,不会强制放大 // quality JPEG 质量 1..100(建议 40..85) // out 编码后的 JPEG 字节流 // outWidth 实际编码图宽 diff --git a/client/ScreenSpy.cpp b/client/ScreenSpy.cpp index 2694f36..37ae6ec 100644 --- a/client/ScreenSpy.cpp +++ b/client/ScreenSpy.cpp @@ -13,8 +13,8 @@ ////////////////////////////////////////////////////////////////////// CScreenSpy::CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk, int gop, BOOL all, int level, - RECT rc, BOOL switchScreen, HWND hwnd, bool dynamicFg) : - ScreenCapture(ulbiBitCount, algo, all, level, rc, switchScreen) + RECT rc, BOOL switchScreen, HWND hwnd, bool dynamicFg, BOOL forceVirtual) : + ScreenCapture(ulbiBitCount, algo, all, level, rc, switchScreen, forceVirtual) { m_hTargetWnd = hwnd; m_bDynamicForeground = dynamicFg; @@ -24,7 +24,9 @@ CScreenSpy::CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk, int gop, BOOL // 也走这里:两条路径必须落到同一块屏上,否则下一次重建会换屏。 // 窗口捕获(含动态前景)不参与屏索引:下面的窗口矩形会覆盖基类的显示器几何, // 此时若提交,一次窗口重建就会把索引往前推一格,取消窗口捕获后画面就换屏了。 - if (!hwnd && !m_bDynamicForeground && m_bEnableMultiScreen && m_nScreenCount > 0) + // 强制虚拟桌面同理:那是服务端本次会话的显式选择,不应影响人类的屏轮转顺序。 + if (!hwnd && !m_bDynamicForeground && !m_bForceVirtualDesktop && + m_bEnableMultiScreen && m_nScreenCount > 0) CommitMonitorIndex(m_nMonitorIndex); // 窗口捕获模式:用 DWM 真实边界覆盖基类的全屏尺寸,并缓存阴影偏移量 diff --git a/client/ScreenSpy.h b/client/ScreenSpy.h index 4e7b50d..3102001 100644 --- a/client/ScreenSpy.h +++ b/client/ScreenSpy.h @@ -107,7 +107,8 @@ protected: public: CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk = FALSE, int gop = DEFAULT_GOP, BOOL all = FALSE, - int level = LEVEL_H264_SOFT, RECT rc = {0}, BOOL switchScreen = TRUE, HWND hwnd = NULL, bool dynamicFg = false); + int level = LEVEL_H264_SOFT, RECT rc = {0}, BOOL switchScreen = TRUE, HWND hwnd = NULL, bool dynamicFg = false, + BOOL forceVirtual = FALSE); virtual HWND GetTargetWindow() const override { return m_hTargetWnd; } diff --git a/common/commands.h b/common/commands.h index 10cae94..d5bf0f5 100644 --- a/common/commands.h +++ b/common/commands.h @@ -140,6 +140,7 @@ inline int isValid_10s() #define CLIENT_CAP_C2C_RECV_ACK 0x0010 // 作为 C2C 目标端时,落盘完成后回执(TOKEN_C2C_RECV_RESULT,目前仅 Linux) #define CLIENT_CAP_PROTO_SWITCH 0x0020 // 支持服务端远程切换传输协议(COMMAND_SET_PROTO,仅 Windows) #define CLIENT_CAP_EMBEDDED_ANCHOR 0x0040 // 当前靠「生成时内嵌的 TLS 信任锚」在加密(按运行时状态置位,不是能力声明) +#define CLIENT_CAP_VIRTUAL_DESKTOP 0x0080 // 支持按 COMMAND_SCREEN_SPY 参数 buffer[11](wire 索引 12)强制抓取整个虚拟桌面(多显示器远程控制) #define TALK_DLG_MAXLEN 1024 // 最大输入字符长度 @@ -1188,6 +1189,8 @@ typedef struct LOGIN_INFOR { caps |= CLIENT_CAP_C2C_PATHS; // 协议切换(COMMAND_SET_PROTO)只有 Windows 客户端实现了命令处理与入口持久化。 caps |= CLIENT_CAP_PROTO_SWITCH; + // 多显示器远程控制:抓整个虚拟桌面,截图与注入共用同一坐标系。 + caps |= CLIENT_CAP_VIRTUAL_DESKTOP; #elif defined(__linux__) && !defined(__ANDROID__) // Linux 客户端作为 C2C 目标端时会回执落盘结果(common/FileTransferV2.h)。 // macOS 暂未接线(无全局主连接句柄),不声明——服务端就不会为它空等回执。 diff --git a/server/2015Remote/McpServer.cpp b/server/2015Remote/McpServer.cpp index 5501a91..029b8fa 100644 --- a/server/2015Remote/McpServer.cpp +++ b/server/2015Remote/McpServer.cpp @@ -2343,7 +2343,7 @@ std::string BuildToolsListResult(const Json::Value& id) { { Json::Value tool(Json::objectValue); tool["name"] = "remote_open"; - tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与物理屏幕分辨率(screen_w/screen_h)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;多显示器暂不支持。"; + tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与抓屏帧尺寸(screen_w/screen_h,可能是等比缩放后的整屏/整虚拟桌面像素)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;Windows 多显示器需要新版客户端(整虚拟桌面抓屏),老客户端会被拒绝(Linux 截屏/注入天然使用整个 X root,不受此限)。"; tool["inputSchema"] = BuildRemoteOpenInputSchema(); tool["outputSchema"] = BuildRemoteOpenOutputSchema(); tools.append(tool); @@ -2720,8 +2720,9 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V std::string expired = ResFieldUtf8(ctx, cp, RES_EXPIRED_DATE); if (!expired.empty()) host["expiredDate"] = expired; - // 心跳格式 "N:W*H":N=显示器数。多显示器时 remote_open/remote_* 会拒(-32005), - // 把 N 直接给出,AI 不必自己拆 screen 字符串。老客户端没有冒号 → 0 表示未知。 + // 心跳格式 "N:W*H":N=显示器数。Windows 客户端多显示器且不支持整虚拟桌面抓屏时 + // remote_open/remote_* 会拒(-32005),把 N 直接给出,AI 不必自己拆 screen 字符串。 + // 老客户端没有冒号 → 0 表示未知。 int monitorCount = 0; std::string res = ResFieldUtf8(ctx, cp, RES_RESOLUTION); size_t colon = res.find(':'); @@ -2743,15 +2744,17 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V // McpFileTransfer …)不在这里重复——get_server_status 的 switches 已经完整暴露。 // 每一项都对应一处真实门控,改这里时要一起看向下派发:screenPreview→get_screenshot // (SupportsScreenPreview)、remoteControl→remote_*(ResolveScreenCtrlSessionHost 的 - // 平台门 + remote_open 的单显示器门)、fileTransferV2→download_file/upload_file、 + // 平台门 + remote_open 的多显示器门)、fileTransferV2→download_file/upload_file、 // c2cPaths→copy_files_c2c 源端、registry→list_registry。 CString clientType = ctx->GetAdditionalData(RES_CLIENT_TYPE); Json::Value caps(Json::objectValue); caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK caps["screenPreview"] = ctx->SupportsScreenPreview(); - // 远程控制只实现于 Windows / Linux,且阶段一要求单显示器;monitorCount=0(老客户端 - // 没报格式)无法判定,与 remote_open 一样按放行处理。 - caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && monitorCount <= 1; + // 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源: + // Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。 + // monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。 + bool virtDesktopOk = monitorCount <= 1 || !IsWindowsClient(ctx) || ctx->SupportsVirtualDesktop(); + caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && virtDesktopOk; caps["fileTransferV2"] = ctx->SupportsFileV2(); caps["c2cPaths"] = ctx->SupportsC2CPaths(); caps["registry"] = IsWindowsClient(ctx); @@ -4684,11 +4687,11 @@ Json::Value BuildRemoteOpenOutputSchema() { props["session_id"] = sid; Json::Value sw(Json::objectValue); sw["type"] = "integer"; - sw["description"] = u8"物理屏幕宽度(虚拟桌面像素),用于把归一化坐标 0..1 映射为像素"; + sw["description"] = u8"抓屏帧宽度(发送帧像素:整屏或整虚拟桌面,可能已等比缩放),用于把归一化坐标 0..1 映射为像素"; props["screen_w"] = sw; Json::Value sh(Json::objectValue); sh["type"] = "integer"; - sh["description"] = u8"物理屏幕高度(虚拟桌面像素)"; + sh["description"] = u8"抓屏帧高度(发送帧像素)"; props["screen_h"] = sh; Json::Value schema(Json::objectValue); schema["type"] = "object"; @@ -4736,7 +4739,7 @@ Json::Value BuildRemoteCloseOutputSchema() { return schema; } -// tools/call:remote_open(建立隐藏屏幕子连接 + 取物理分辨率) +// tools/call:remote_open(建立隐藏屏幕子连接 + 取抓屏帧尺寸) std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) { CMcpServer& mcp = CMcpServer::Instance(); mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec); @@ -4747,19 +4750,20 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2 if (!ResolveScreenCtrlSessionHost(id, args, parent, devId, ctx, errJson)) return errJson; - // 多显示器门(§7.2 / §12.1):Observe 抓主屏,Act 注入在虚拟桌面空间,两者仅单显示器时 - // 重合。客户端心跳 RES_RESOLUTION = "N:W*H"(N=显示器数);N>1 时归一化坐标会系统性 - // 偏移,阶段一直接拒绝。这是能力限制(改参数也修不好),故报 -32005 而非参数校验错误 -32012。 - // 无冒号(老客户端格式)则无法判定,放行不拦。 + // 多显示器门:截图与注入必须共用同一坐标系。Windows 新客户端(CLIENT_CAP_VIRTUAL_DESKTOP) + // 会按 COMMAND_SCREEN_SPY 参数 buffer[11](wire 索引 12)抓整个虚拟桌面,可放行; + // 老 Windows 客户端只能抓单块屏,N>1 时归一化坐标会系统性偏移,拒绝并提示升级。 + // 非 Windows(如 Linux)不要求该能力位:其截屏取整个 X root + // (linux/ScreenHandler.h 用 XDisplayWidth/Height),注入用 root 绝对坐标, + // 多输出下天然同源。无冒号(老客户端格式)无法判定,放行不拦。 { CString res = ctx->GetAdditionalData(RES_RESOLUTION); int colon = res.Find(':'); if (colon > 0) { - CString countStr = res.Left(colon); - int monitorCount = atoi(countStr); - if (monitorCount > 1) + int monitorCount = atoi(res.Left(colon)); + if (monitorCount > 1 && IsWindowsClient(ctx) && !ctx->SupportsVirtualDesktop()) return BuildError(id, -32005, - "Multi-monitor hosts are not supported yet (phase 1): monitor count " + + "Multi-monitor hosts require a Windows client with virtual desktop capture (upgrade the client): monitor count " + std::to_string(monitorCount)); } } @@ -4783,7 +4787,8 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2 return BuildError(id, -32003, "Device busy: a remote control session is already active for this host"); // 建立隐藏屏幕子连接(发 COMMAND_SCREEN_SPY、标记 web-triggered)。 - if (!WebService().StartRemoteDesktop(devId)) { + // 新客户端额外要求抓整个虚拟桌面,保证截图与注入坐标系一致。 + if (!WebService().StartRemoteDesktop(devId, ctx->SupportsVirtualDesktop())) { mcp.CloseScreenCtrlSession(devId, sessionId); // StartRemoteDesktop 先置 web-triggered 再 Send2Client,发送失败需一并清理, // 否则该设备残留 web-triggered 标记、后续人类 web 观看会误走隐藏会话路径。 @@ -5296,7 +5301,7 @@ Json::Value BuildRemoteMouseOutputSchema() { return schema; } -// tools/call:remote_mouse(鼠标注入;归一化坐标 → 物理像素) +// tools/call:remote_mouse(鼠标注入;归一化坐标 → 发送帧像素) std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) { CMcpServer& mcp = CMcpServer::Instance(); mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec); @@ -5350,14 +5355,14 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy if (!GetNormCoord(args, "y2", ny2, coordErr)) return BuildError(id, -32602, coordErr); } - // 校验 session_id/busy 并置 busy,同时取物理分辨率(screenW/H>0 由会话就绪保证)。 + // 校验 session_id/busy 并置 busy,同时取抓屏帧尺寸(screenW/H>0 由会话就绪保证)。 context* subCtx = nullptr; int screenW = 0, screenH = 0; int r = mcp.BeginScreenCtrlAction(devId, sessionId, subCtx, screenW, screenH); if (r == 2) return BuildError(id, -32003, "Device busy: another injection is in flight for this session"); if (r == 1) return BuildError(id, -32002, "Remote control session not found, not ready, or closed: " + sessionId); - // 归一化坐标 → 物理像素(round + clamp 到 [0, screen-1]) + // 归一化坐标 → 发送帧像素(round + clamp 到 [0, screen-1];客户端 PointConversion 再换算为物理坐标) auto toPxX = [&](double n) -> int { int p = (int)(n * screenW + 0.5); return p < 0 ? 0 : (p > screenW - 1 ? screenW - 1 : p); }; auto toPxY = [&](double n) -> int { int p = (int)(n * screenH + 0.5); return p < 0 ? 0 : (p > screenH - 1 ? screenH - 1 : p); }; int px = toPxX(nx), py = toPxY(ny); @@ -5404,7 +5409,7 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy if (!ok) return BuildError(id, -32004, "Failed to send mouse injection"); - // 审计(不可关闭;含归一化坐标与换算后的物理像素) + // 审计(不可关闭;含归一化坐标与换算后的帧像素) if (parent) { std::string audit = "host " + std::to_string(devId) + " remote-mouse: session " + sessionId + " action=" + action; diff --git a/server/2015Remote/WebService.cpp b/server/2015Remote/WebService.cpp index 151ee4d..5085546 100644 --- a/server/2015Remote/WebService.cpp +++ b/server/2015Remote/WebService.cpp @@ -2389,7 +2389,7 @@ void CWebService::NotifyClipboardText(uint64_t device_id, const std::string& tex } } -bool CWebService::StartRemoteDesktop(uint64_t device_id) { +bool CWebService::StartRemoteDesktop(uint64_t device_id, bool forceVirtualDesktop) { if (!m_pParentDlg) return false; context* ctx = m_pParentDlg->FindHost(device_id); @@ -2439,6 +2439,11 @@ bool CWebService::StartRemoteDesktop(uint64_t device_id) { bToken[1] = 0; // DXGI mode: 0=GDI bToken[2] = ALGORITHM_H264; // H264 algorithm bToken[3] = 1; // Multi-screen: true + // 参数 buffer[11](wire 索引 12):强制抓整个虚拟桌面。客户端把 token 剥掉命令字节后 + // 按 buffer[i]=bToken[i+1] 索引,且 buffer[3..10](wire [4..11])是 HWND, + // 不能占用。老客户端会忽略该字节,服务端只在客户端声明 + // CLIENT_CAP_VIRTUAL_DESKTOP 时置位。 + bToken[12] = forceVirtualDesktop ? 1 : 0; return ctx->Send2Client(bToken, sizeof(bToken)) != FALSE; } diff --git a/server/2015Remote/WebService.h b/server/2015Remote/WebService.h index 01c6cbb..29e411c 100644 --- a/server/2015Remote/WebService.h +++ b/server/2015Remote/WebService.h @@ -179,8 +179,10 @@ public: // Get count of web clients watching a device int GetWebClientCount(uint64_t device_id); - // Start remote desktop session for web viewing - bool StartRemoteDesktop(uint64_t device_id); + // Start remote desktop session for web viewing. + // forceVirtualDesktop=true 时要求客户端抓整个虚拟桌面(MCP 远程控制): + // 多显示器下截图与鼠标注入必须共用同一坐标系,Web 观看仍保持原有轮转行为。 + bool StartRemoteDesktop(uint64_t device_id, bool forceVirtualDesktop = false); // Stop remote desktop session void StopRemoteDesktop(uint64_t device_id); diff --git a/server/2015Remote/context.h b/server/2015Remote/context.h index 6d2cf37..d437d59 100644 --- a/server/2015Remote/context.h +++ b/server/2015Remote/context.h @@ -105,6 +105,15 @@ public: return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_SCREEN_PREVIEW) != 0; } + // 检查客户端是否支持强制抓取整个虚拟桌面(COMMAND_SCREEN_SPY 参数 buffer[11], + // 即 wire 索引 12;wire [4..11] 是 HWND,不能占用)。 + // 多显示器远程控制依赖它:截图与注入共用虚拟桌面坐标系。 + bool SupportsVirtualDesktop() const { + CString caps = GetClientData(ONLINELIST_CAPABILITIES); + if (caps.IsEmpty()) return false; + return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_VIRTUAL_DESKTOP) != 0; + } + // 检查客户端是否支持远程协议切换(COMMAND_SET_PROTO)。老客户端没有这个命令, // 收到也会走 default 丢弃,服务端据此把「协议切换」菜单置灰。 bool SupportsProtoSwitch() const { -- 2.43.0