Feature: Capture the virtual desktop for multi-monitor remote control

The MCP Observe/Act loop requires screenshots and mouse injection to share
one coordinate space. Previously the client sent only the primary monitor
while injection used virtual-desktop coordinates, so multi-monitor hosts
were rejected outright.

- Client: COMMAND_SCREEN_SPY param buffer[11] (wire index 12) now forces
  whole-virtual-desktop GDI capture; preview JPEGs also cover the virtual
  desktop including negative origins.
- Server: advertise CLIENT_CAP_VIRTUAL_DESKTOP, gate multi-monitor remote
  control on that capability, and pass the flag when starting the session.
  Linux stays exempt (X root capture/injection are already consistent).
This commit was merged in pull request #7.
This commit is contained in:
yuanyuanxiang committed 2026-11-10 08:21:55 +01:00
1 parent 5bb24d714a
commit 5eb44307ee
11 files changed
+93 -48

No files matched your search

+28 -23
View File
@@ -2343,7 +2343,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
{
Json::Value tool(Json::objectValue);
tool["name"] = "remote_open";
tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与物理屏幕分辨率(screen_w/screen_h)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;多显示器暂不支持。";
tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与抓屏帧尺寸(screen_w/screen_h,可能是等比缩放后的整屏/整虚拟桌面像素)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;Windows 多显示器需要新版客户端(整虚拟桌面抓屏),老客户端会被拒绝(Linux 截屏/注入天然使用整个 X root,不受此限)。";
tool["inputSchema"] = BuildRemoteOpenInputSchema();
tool["outputSchema"] = BuildRemoteOpenOutputSchema();
tools.append(tool);
@@ -2720,8 +2720,9 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
std::string expired = ResFieldUtf8(ctx, cp, RES_EXPIRED_DATE);
if (!expired.empty()) host["expiredDate"] = expired;
// 心跳格式 "N:W*H":N=显示器数。多显示器时 remote_open/remote_* 会拒(-32005),
// 把 N 直接给出,AI 不必自己拆 screen 字符串。老客户端没有冒号 → 0 表示未知。
// 心跳格式 "N:W*H":N=显示器数。Windows 客户端多显示器且不支持整虚拟桌面抓屏时
// remote_open/remote_* 会拒(-32005),把 N 直接给出,AI 不必自己拆 screen 字符串。
// 老客户端没有冒号 → 0 表示未知。
int monitorCount = 0;
std::string res = ResFieldUtf8(ctx, cp, RES_RESOLUTION);
size_t colon = res.find(':');
@@ -2743,15 +2744,17 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
// McpFileTransfer …)不在这里重复——get_server_status 的 switches 已经完整暴露。
// 每一项都对应一处真实门控,改这里时要一起看向下派发:screenPreview→get_screenshot
// (SupportsScreenPreview)、remoteControl→remote_*(ResolveScreenCtrlSessionHost 的
// 平台门 + remote_open 的单显示器门)、fileTransferV2→download_file/upload_file、
// 平台门 + remote_open 的多显示器门)、fileTransferV2→download_file/upload_file、
// c2cPaths→copy_files_c2c 源端、registry→list_registry。
CString clientType = ctx->GetAdditionalData(RES_CLIENT_TYPE);
Json::Value caps(Json::objectValue);
caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK
caps["screenPreview"] = ctx->SupportsScreenPreview();
// 远程控制只实现于 Windows / Linux,且阶段一要求单显示器;monitorCount=0(老客户端
// 没报格式)无法判定,与 remote_open 一样按放行处理。
caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && monitorCount <= 1;
// 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源:
// Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。
// monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。
bool virtDesktopOk = monitorCount <= 1 || !IsWindowsClient(ctx) || ctx->SupportsVirtualDesktop();
caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && virtDesktopOk;
caps["fileTransferV2"] = ctx->SupportsFileV2();
caps["c2cPaths"] = ctx->SupportsC2CPaths();
caps["registry"] = IsWindowsClient(ctx);
@@ -4684,11 +4687,11 @@ Json::Value BuildRemoteOpenOutputSchema() {
props["session_id"] = sid;
Json::Value sw(Json::objectValue);
sw["type"] = "integer";
sw["description"] = u8"物理屏幕宽度(虚拟桌面像素),用于把归一化坐标 0..1 映射为像素";
sw["description"] = u8"抓屏帧宽度(发送帧像素:整屏或整虚拟桌面,可能已等比缩放),用于把归一化坐标 0..1 映射为像素";
props["screen_w"] = sw;
Json::Value sh(Json::objectValue);
sh["type"] = "integer";
sh["description"] = u8"物理屏幕高度(虚拟桌面像素)";
sh["description"] = u8"抓屏帧高度(发送帧像素)";
props["screen_h"] = sh;
Json::Value schema(Json::objectValue);
schema["type"] = "object";
@@ -4736,7 +4739,7 @@ Json::Value BuildRemoteCloseOutputSchema() {
return schema;
}
// tools/call:remote_open(建立隐藏屏幕子连接 + 取物理分辨率)
// tools/call:remote_open(建立隐藏屏幕子连接 + 取抓屏帧尺寸)
std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
CMcpServer& mcp = CMcpServer::Instance();
mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec);
@@ -4747,19 +4750,20 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2
if (!ResolveScreenCtrlSessionHost(id, args, parent, devId, ctx, errJson))
return errJson;
// 多显示器门(§7.2 / §12.1):Observe 抓主屏,Act 注入在虚拟桌面空间,两者仅单显示器时
// 重合。客户端心跳 RES_RESOLUTION = "N:W*H"(N=显示器数);N>1 时归一化坐标会系统性
// 偏移,阶段一直接拒绝。这是能力限制(改参数也修不好),故报 -32005 而非参数校验错误 -32012。
// 无冒号(老客户端格式)则无法判定,放行不拦。
// 多显示器门:截图与注入必须共用同一坐标系。Windows 新客户端(CLIENT_CAP_VIRTUAL_DESKTOP)
// 会按 COMMAND_SCREEN_SPY 参数 buffer[11](wire 索引 12)抓整个虚拟桌面,可放行;
// 老 Windows 客户端只能抓单块屏,N>1 时归一化坐标会系统性偏移,拒绝并提示升级。
// 非 Windows(如 Linux)不要求该能力位:其截屏取整个 X root
// (linux/ScreenHandler.h 用 XDisplayWidth/Height),注入用 root 绝对坐标,
// 多输出下天然同源。无冒号(老客户端格式)无法判定,放行不拦。
{
CString res = ctx->GetAdditionalData(RES_RESOLUTION);
int colon = res.Find(':');
if (colon > 0) {
CString countStr = res.Left(colon);
int monitorCount = atoi(countStr);
if (monitorCount > 1)
int monitorCount = atoi(res.Left(colon));
if (monitorCount > 1 && IsWindowsClient(ctx) && !ctx->SupportsVirtualDesktop())
return BuildError(id, -32005,
"Multi-monitor hosts are not supported yet (phase 1): monitor count " +
"Multi-monitor hosts require a Windows client with virtual desktop capture (upgrade the client): monitor count " +
std::to_string(monitorCount));
}
}
@@ -4783,7 +4787,8 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2
return BuildError(id, -32003, "Device busy: a remote control session is already active for this host");
// 建立隐藏屏幕子连接(发 COMMAND_SCREEN_SPY、标记 web-triggered)。
if (!WebService().StartRemoteDesktop(devId)) {
// 新客户端额外要求抓整个虚拟桌面,保证截图与注入坐标系一致。
if (!WebService().StartRemoteDesktop(devId, ctx->SupportsVirtualDesktop())) {
mcp.CloseScreenCtrlSession(devId, sessionId);
// StartRemoteDesktop 先置 web-triggered 再 Send2Client,发送失败需一并清理,
// 否则该设备残留 web-triggered 标记、后续人类 web 观看会误走隐藏会话路径。
@@ -5296,7 +5301,7 @@ Json::Value BuildRemoteMouseOutputSchema() {
return schema;
}
// tools/call:remote_mouse(鼠标注入;归一化坐标 → 物理像素)
// tools/call:remote_mouse(鼠标注入;归一化坐标 → 发送帧像素)
std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
CMcpServer& mcp = CMcpServer::Instance();
mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec);
@@ -5350,14 +5355,14 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy
if (!GetNormCoord(args, "y2", ny2, coordErr)) return BuildError(id, -32602, coordErr);
}
// 校验 session_id/busy 并置 busy,同时取物理分辨率(screenW/H>0 由会话就绪保证)。
// 校验 session_id/busy 并置 busy,同时取抓屏帧尺寸(screenW/H>0 由会话就绪保证)。
context* subCtx = nullptr;
int screenW = 0, screenH = 0;
int r = mcp.BeginScreenCtrlAction(devId, sessionId, subCtx, screenW, screenH);
if (r == 2) return BuildError(id, -32003, "Device busy: another injection is in flight for this session");
if (r == 1) return BuildError(id, -32002, "Remote control session not found, not ready, or closed: " + sessionId);
// 归一化坐标 → 物理像素(round + clamp 到 [0, screen-1])
// 归一化坐标 → 发送帧像素(round + clamp 到 [0, screen-1];客户端 PointConversion 再换算为物理坐标)
auto toPxX = [&](double n) -> int { int p = (int)(n * screenW + 0.5); return p < 0 ? 0 : (p > screenW - 1 ? screenW - 1 : p); };
auto toPxY = [&](double n) -> int { int p = (int)(n * screenH + 0.5); return p < 0 ? 0 : (p > screenH - 1 ? screenH - 1 : p); };
int px = toPxX(nx), py = toPxY(ny);
@@ -5404,7 +5409,7 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy
if (!ok)
return BuildError(id, -32004, "Failed to send mouse injection");
// 审计(不可关闭;含归一化坐标与换算后的物理像素)
// 审计(不可关闭;含归一化坐标与换算后的帧像素)
if (parent) {
std::string audit = "host " + std::to_string(devId) + " remote-mouse: session " + sessionId
+ " action=" + action;