Feature: Capture the virtual desktop for multi-monitor remote control
The MCP Observe/Act loop requires screenshots and mouse injection to share one coordinate space. Previously the client sent only the primary monitor while injection used virtual-desktop coordinates, so multi-monitor hosts were rejected outright. - Client: COMMAND_SCREEN_SPY param buffer[11] (wire index 12) now forces whole-virtual-desktop GDI capture; preview JPEGs also cover the virtual desktop including negative origins. - Server: advertise CLIENT_CAP_VIRTUAL_DESKTOP, gate multi-monitor remote control on that capability, and pass the flag when starting the session. Linux stays exempt (X root capture/injection are already consistent).
This commit was merged in pull request #7.
This commit is contained in:
1 parent
5bb24d714a
commit
5eb44307ee
11 files changed
+93
-48
No files matched your search
@@ -2343,7 +2343,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
|
||||
{
|
||||
Json::Value tool(Json::objectValue);
|
||||
tool["name"] = "remote_open";
|
||||
tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与物理屏幕分辨率(screen_w/screen_h)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;多显示器暂不支持。";
|
||||
tool["description"] = u8"在指定在线 Windows 或 Linux 主机上建立远程控制会话(隐藏屏幕子连接)并返回 session_id 与抓屏帧尺寸(screen_w/screen_h,可能是等比缩放后的整屏/整虚拟桌面像素)。后续 remote_mouse/remote_keyboard 用归一化坐标(0..1)注入输入,用毕须 remote_close。Linux 需要 X11/Xorg(或 XWayland)桌面,纯 Wayland 会话无法抓屏/注入;Windows 多显示器需要新版客户端(整虚拟桌面抓屏),老客户端会被拒绝(Linux 截屏/注入天然使用整个 X root,不受此限)。";
|
||||
tool["inputSchema"] = BuildRemoteOpenInputSchema();
|
||||
tool["outputSchema"] = BuildRemoteOpenOutputSchema();
|
||||
tools.append(tool);
|
||||
@@ -2720,8 +2720,9 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
|
||||
std::string expired = ResFieldUtf8(ctx, cp, RES_EXPIRED_DATE);
|
||||
if (!expired.empty()) host["expiredDate"] = expired;
|
||||
|
||||
// 心跳格式 "N:W*H":N=显示器数。多显示器时 remote_open/remote_* 会拒(-32005),
|
||||
// 把 N 直接给出,AI 不必自己拆 screen 字符串。老客户端没有冒号 → 0 表示未知。
|
||||
// 心跳格式 "N:W*H":N=显示器数。Windows 客户端多显示器且不支持整虚拟桌面抓屏时
|
||||
// remote_open/remote_* 会拒(-32005),把 N 直接给出,AI 不必自己拆 screen 字符串。
|
||||
// 老客户端没有冒号 → 0 表示未知。
|
||||
int monitorCount = 0;
|
||||
std::string res = ResFieldUtf8(ctx, cp, RES_RESOLUTION);
|
||||
size_t colon = res.find(':');
|
||||
@@ -2743,15 +2744,17 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
|
||||
// McpFileTransfer …)不在这里重复——get_server_status 的 switches 已经完整暴露。
|
||||
// 每一项都对应一处真实门控,改这里时要一起看向下派发:screenPreview→get_screenshot
|
||||
// (SupportsScreenPreview)、remoteControl→remote_*(ResolveScreenCtrlSessionHost 的
|
||||
// 平台门 + remote_open 的单显示器门)、fileTransferV2→download_file/upload_file、
|
||||
// 平台门 + remote_open 的多显示器门)、fileTransferV2→download_file/upload_file、
|
||||
// c2cPaths→copy_files_c2c 源端、registry→list_registry。
|
||||
CString clientType = ctx->GetAdditionalData(RES_CLIENT_TYPE);
|
||||
Json::Value caps(Json::objectValue);
|
||||
caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK
|
||||
caps["screenPreview"] = ctx->SupportsScreenPreview();
|
||||
// 远程控制只实现于 Windows / Linux,且阶段一要求单显示器;monitorCount=0(老客户端
|
||||
// 没报格式)无法判定,与 remote_open 一样按放行处理。
|
||||
caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && monitorCount <= 1;
|
||||
// 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源:
|
||||
// Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。
|
||||
// monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。
|
||||
bool virtDesktopOk = monitorCount <= 1 || !IsWindowsClient(ctx) || ctx->SupportsVirtualDesktop();
|
||||
caps["remoteControl"] = (IsWindowsClient(ctx) || clientType == "LNX") && virtDesktopOk;
|
||||
caps["fileTransferV2"] = ctx->SupportsFileV2();
|
||||
caps["c2cPaths"] = ctx->SupportsC2CPaths();
|
||||
caps["registry"] = IsWindowsClient(ctx);
|
||||
@@ -4684,11 +4687,11 @@ Json::Value BuildRemoteOpenOutputSchema() {
|
||||
props["session_id"] = sid;
|
||||
Json::Value sw(Json::objectValue);
|
||||
sw["type"] = "integer";
|
||||
sw["description"] = u8"物理屏幕宽度(虚拟桌面像素),用于把归一化坐标 0..1 映射为像素";
|
||||
sw["description"] = u8"抓屏帧宽度(发送帧像素:整屏或整虚拟桌面,可能已等比缩放),用于把归一化坐标 0..1 映射为像素";
|
||||
props["screen_w"] = sw;
|
||||
Json::Value sh(Json::objectValue);
|
||||
sh["type"] = "integer";
|
||||
sh["description"] = u8"物理屏幕高度(虚拟桌面像素)";
|
||||
sh["description"] = u8"抓屏帧高度(发送帧像素)";
|
||||
props["screen_h"] = sh;
|
||||
Json::Value schema(Json::objectValue);
|
||||
schema["type"] = "object";
|
||||
@@ -4736,7 +4739,7 @@ Json::Value BuildRemoteCloseOutputSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
// tools/call:remote_open(建立隐藏屏幕子连接 + 取物理分辨率)
|
||||
// tools/call:remote_open(建立隐藏屏幕子连接 + 取抓屏帧尺寸)
|
||||
std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
|
||||
CMcpServer& mcp = CMcpServer::Instance();
|
||||
mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec);
|
||||
@@ -4747,19 +4750,20 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2
|
||||
if (!ResolveScreenCtrlSessionHost(id, args, parent, devId, ctx, errJson))
|
||||
return errJson;
|
||||
|
||||
// 多显示器门(§7.2 / §12.1):Observe 抓主屏,Act 注入在虚拟桌面空间,两者仅单显示器时
|
||||
// 重合。客户端心跳 RES_RESOLUTION = "N:W*H"(N=显示器数);N>1 时归一化坐标会系统性
|
||||
// 偏移,阶段一直接拒绝。这是能力限制(改参数也修不好),故报 -32005 而非参数校验错误 -32012。
|
||||
// 无冒号(老客户端格式)则无法判定,放行不拦。
|
||||
// 多显示器门:截图与注入必须共用同一坐标系。Windows 新客户端(CLIENT_CAP_VIRTUAL_DESKTOP)
|
||||
// 会按 COMMAND_SCREEN_SPY 参数 buffer[11](wire 索引 12)抓整个虚拟桌面,可放行;
|
||||
// 老 Windows 客户端只能抓单块屏,N>1 时归一化坐标会系统性偏移,拒绝并提示升级。
|
||||
// 非 Windows(如 Linux)不要求该能力位:其截屏取整个 X root
|
||||
// (linux/ScreenHandler.h 用 XDisplayWidth/Height),注入用 root 绝对坐标,
|
||||
// 多输出下天然同源。无冒号(老客户端格式)无法判定,放行不拦。
|
||||
{
|
||||
CString res = ctx->GetAdditionalData(RES_RESOLUTION);
|
||||
int colon = res.Find(':');
|
||||
if (colon > 0) {
|
||||
CString countStr = res.Left(colon);
|
||||
int monitorCount = atoi(countStr);
|
||||
if (monitorCount > 1)
|
||||
int monitorCount = atoi(res.Left(colon));
|
||||
if (monitorCount > 1 && IsWindowsClient(ctx) && !ctx->SupportsVirtualDesktop())
|
||||
return BuildError(id, -32005,
|
||||
"Multi-monitor hosts are not supported yet (phase 1): monitor count " +
|
||||
"Multi-monitor hosts require a Windows client with virtual desktop capture (upgrade the client): monitor count " +
|
||||
std::to_string(monitorCount));
|
||||
}
|
||||
}
|
||||
@@ -4783,7 +4787,8 @@ std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2
|
||||
return BuildError(id, -32003, "Device busy: a remote control session is already active for this host");
|
||||
|
||||
// 建立隐藏屏幕子连接(发 COMMAND_SCREEN_SPY、标记 web-triggered)。
|
||||
if (!WebService().StartRemoteDesktop(devId)) {
|
||||
// 新客户端额外要求抓整个虚拟桌面,保证截图与注入坐标系一致。
|
||||
if (!WebService().StartRemoteDesktop(devId, ctx->SupportsVirtualDesktop())) {
|
||||
mcp.CloseScreenCtrlSession(devId, sessionId);
|
||||
// StartRemoteDesktop 先置 web-triggered 再 Send2Client,发送失败需一并清理,
|
||||
// 否则该设备残留 web-triggered 标记、后续人类 web 观看会误走隐藏会话路径。
|
||||
@@ -5296,7 +5301,7 @@ Json::Value BuildRemoteMouseOutputSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
// tools/call:remote_mouse(鼠标注入;归一化坐标 → 物理像素)
|
||||
// tools/call:remote_mouse(鼠标注入;归一化坐标 → 发送帧像素)
|
||||
std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
|
||||
CMcpServer& mcp = CMcpServer::Instance();
|
||||
mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec);
|
||||
@@ -5350,14 +5355,14 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy
|
||||
if (!GetNormCoord(args, "y2", ny2, coordErr)) return BuildError(id, -32602, coordErr);
|
||||
}
|
||||
|
||||
// 校验 session_id/busy 并置 busy,同时取物理分辨率(screenW/H>0 由会话就绪保证)。
|
||||
// 校验 session_id/busy 并置 busy,同时取抓屏帧尺寸(screenW/H>0 由会话就绪保证)。
|
||||
context* subCtx = nullptr;
|
||||
int screenW = 0, screenH = 0;
|
||||
int r = mcp.BeginScreenCtrlAction(devId, sessionId, subCtx, screenW, screenH);
|
||||
if (r == 2) return BuildError(id, -32003, "Device busy: another injection is in flight for this session");
|
||||
if (r == 1) return BuildError(id, -32002, "Remote control session not found, not ready, or closed: " + sessionId);
|
||||
|
||||
// 归一化坐标 → 物理像素(round + clamp 到 [0, screen-1])
|
||||
// 归一化坐标 → 发送帧像素(round + clamp 到 [0, screen-1];客户端 PointConversion 再换算为物理坐标)
|
||||
auto toPxX = [&](double n) -> int { int p = (int)(n * screenW + 0.5); return p < 0 ? 0 : (p > screenW - 1 ? screenW - 1 : p); };
|
||||
auto toPxY = [&](double n) -> int { int p = (int)(n * screenH + 0.5); return p < 0 ? 0 : (p > screenH - 1 ? screenH - 1 : p); };
|
||||
int px = toPxX(nx), py = toPxY(ny);
|
||||
@@ -5404,7 +5409,7 @@ std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy
|
||||
if (!ok)
|
||||
return BuildError(id, -32004, "Failed to send mouse injection");
|
||||
|
||||
// 审计(不可关闭;含归一化坐标与换算后的物理像素)
|
||||
// 审计(不可关闭;含归一化坐标与换算后的帧像素)
|
||||
if (parent) {
|
||||
std::string audit = "host " + std::to_string(devId) + " remote-mouse: session " + sessionId
|
||||
+ " action=" + action;
|
||||
|
||||
Reference in new issue
Block a user