From 18259b8526062db16c8cfddffc78dddc87b7842e Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Tue, 25 Aug 2026 13:38:09 +0200 Subject: [PATCH] Feature: Add remote_mouse MCP tool Add the remote_mouse tool to the MCP remote control surface, injecting mouse events (move / down / up / click / right_click / middle_click / drag / scroll) into an established remote_open session. Normalized 0..1 coordinates are mapped to physical pixels via the session's captured resolution and clamped to screen bounds. Scroll follows the existing web console's wheel sign convention (positive delta scrolls down) and is vertical-only. Extract BuildMouseMsg64 into WebService.h as a shared helper, reused by both the web console's HandleMouse and the new MCP handler so the two injection paths cannot drift. Co-Authored-By: deepseek-v4-pro --- server/2015Remote/McpServer.cpp | 296 +++++++++++++++++++++++++++++++ server/2015Remote/WebService.cpp | 41 ++--- server/2015Remote/WebService.h | 15 ++ 3 files changed, 321 insertions(+), 31 deletions(-) diff --git a/server/2015Remote/McpServer.cpp b/server/2015Remote/McpServer.cpp index a234ea2..29d47fd 100644 --- a/server/2015Remote/McpServer.cpp +++ b/server/2015Remote/McpServer.cpp @@ -1134,9 +1134,12 @@ Json::Value BuildRemoteCloseInputSchema(); Json::Value BuildRemoteCloseOutputSchema(); Json::Value BuildRemoteKeyboardInputSchema(); Json::Value BuildRemoteKeyboardOutputSchema(); +Json::Value BuildRemoteMouseInputSchema(); +Json::Value BuildRemoteMouseOutputSchema(); std::string BuildRemoteOpen(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent); std::string BuildRemoteClose(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent); std::string BuildRemoteKeyboard(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent); +std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent); // tools/list std::string BuildToolsListResult(const Json::Value& id) { @@ -1368,6 +1371,14 @@ std::string BuildToolsListResult(const Json::Value& id) { tool["outputSchema"] = BuildRemoteKeyboardOutputSchema(); tools.append(tool); } + { + Json::Value tool(Json::objectValue); + tool["name"] = "remote_mouse"; + tool["description"] = u8"向已建立的远程控制会话注入鼠标事件。action 可选 move / down / up / click / right_click / middle_click / drag / scroll；坐标 x/y（及 drag 的 x2/y2）为归一化 0..1 浮点。button 可选 left/middle/right（默认 left）；click 的 clicks 可选 1/2/3（默认 1）；scroll 的 delta 正=向下滚、负=向上滚。"; + tool["inputSchema"] = BuildRemoteMouseInputSchema(); + tool["outputSchema"] = BuildRemoteMouseOutputSchema(); + tools.append(tool); + } } result["tools"] = tools; @@ -3113,6 +3124,290 @@ std::string BuildRemoteKeyboard(const Json::Value& id, const Json::Value& args, return BuildResult(id, result); } +// ===== P5：remote_mouse（鼠标注入，move/down/up/click/right_click/middle_click/drag/scroll）===== + +// 读取归一化坐标（0..1 浮点，接受 JSON number 或数字字符串），越界钳到 [0,1]。 +static bool GetNormCoord(const Json::Value& args, const char* key, double& out, std::string& err) { + if (!args.isMember(key)) { err = std::string("Missing required parameter: ") + key; return false; } + const Json::Value& v = args[key]; + if (v.isNumeric()) { + out = v.asDouble(); + } else if (v.isString()) { + const std::string s = v.asString(); + if (s.empty()) { err = std::string("Invalid ") + key + ": expected a number in 0..1"; return false; } + char* end = nullptr; + double d = strtod(s.c_str(), &end); + if (end == s.c_str() || *end != '\0') { err = std::string("Invalid ") + key + ": expected a number in 0..1"; return false; } + out = d; + } else { + err = std::string("Invalid ") + key + ": expected a number in 0..1"; + return false; + } + if (out != out) { // NaN（如字符串 "nan"）会绕过钳制并令 (int)(n*screen) 未定义行为 + err = std::string("Invalid ") + key + ": not a finite number"; + return false; + } + if (out < 0.0) out = 0.0; + if (out > 1.0) out = 1.0; + return true; +} + +// 读取可选整数参数（JSON number 或数字字符串）；缺失/非法返回 false（不写 out）。 +static bool GetIntArg(const Json::Value& args, const char* key, int& out) { + if (!args.isMember(key)) return false; + const Json::Value& v = args[key]; + if (v.isNumeric()) { out = v.asInt(); return true; } + if (v.isString()) { + const std::string s = v.asString(); + if (IsDigits(s) || (!s.empty() && s[0] == '-' && IsDigits(s.substr(1)))) { + out = atoi(s.c_str()); + return true; + } + } + return false; +} + +// button 字符串 → 0=左/1=中/2=右；空或 left 默认左。 +static bool ParseButton(const std::string& s, int& out) { + if (s.empty() || s == "left") { out = 0; return true; } + if (s == "middle") { out = 1; return true; } + if (s == "right") { out = 2; return true; } + return false; +} + +// button → 按下/抬起消息 + 按下 wParam（对齐 WebService::HandleMouse）。 +static void ButtonMessages(int button, UINT& downMsg, UINT& upMsg, uint64_t& downWParam) { + if (button == 1) { downMsg = WM_MBUTTONDOWN; upMsg = WM_MBUTTONUP; downWParam = MK_MBUTTON; } + else if (button == 2) { downMsg = WM_RBUTTONDOWN; upMsg = WM_RBUTTONUP; downWParam = MK_RBUTTON; } + else { downMsg = WM_LBUTTONDOWN; upMsg = WM_LBUTTONUP; downWParam = MK_LBUTTON; } +} + +Json::Value BuildRemoteMouseInputSchema() { + Json::Value props(Json::objectValue); + Json::Value idProp(Json::objectValue); + idProp["type"] = "string"; + idProp["description"] = u8"主机 id，取 list_online_hosts / search_hosts 返回的 id 字段"; + props["id"] = idProp; + + Json::Value sid(Json::objectValue); + sid["type"] = "string"; + sid["description"] = u8"remote_open 返回的 session_id"; + props["session_id"] = sid; + + Json::Value act(Json::objectValue); + act["type"] = "string"; + act["description"] = u8"动作：move / down / up / click / right_click / middle_click / drag / scroll"; + Json::Value actEnum(Json::arrayValue); + actEnum.append("move"); actEnum.append("down"); actEnum.append("up"); actEnum.append("click"); + actEnum.append("right_click"); actEnum.append("middle_click"); actEnum.append("drag"); actEnum.append("scroll"); + act["enum"] = actEnum; + props["action"] = act; + + Json::Value xProp(Json::objectValue); + xProp["type"] = "number"; + xProp["description"] = u8"归一化 X 坐标（0=屏幕最左，1=最右）"; + props["x"] = xProp; + + Json::Value yProp(Json::objectValue); + yProp["type"] = "number"; + yProp["description"] = u8"归一化 Y 坐标（0=屏幕最上，1=最下）"; + props["y"] = yProp; + + Json::Value x2Prop(Json::objectValue); + x2Prop["type"] = "number"; + x2Prop["description"] = u8"拖拽终点 X（仅 drag）"; + props["x2"] = x2Prop; + + Json::Value y2Prop(Json::objectValue); + y2Prop["type"] = "number"; + y2Prop["description"] = u8"拖拽终点 Y（仅 drag）"; + props["y2"] = y2Prop; + + Json::Value btn(Json::objectValue); + btn["type"] = "string"; + btn["description"] = u8"按键：left / middle / right（默认 left）"; + Json::Value btnEnum(Json::arrayValue); + btnEnum.append("left"); btnEnum.append("middle"); btnEnum.append("right"); + btn["enum"] = btnEnum; + props["button"] = btn; + + Json::Value clk(Json::objectValue); + clk["type"] = "integer"; + clk["description"] = u8"点击次数 1/2/3（仅 click，默认 1）"; + props["clicks"] = clk; + + Json::Value del(Json::objectValue); + del["type"] = "integer"; + del["description"] = u8"滚动量（仅 scroll；正=向下滚，负=向上滚，与 Web 控制台一致；仅垂直滚轮，忽略 button）"; + props["delta"] = del; + + Json::Value schema(Json::objectValue); + schema["type"] = "object"; + schema["properties"] = props; + Json::Value required(Json::arrayValue); + required.append("id"); + required.append("session_id"); + required.append("action"); + required.append("x"); + required.append("y"); + schema["required"] = required; + return schema; +} + +Json::Value BuildRemoteMouseOutputSchema() { + Json::Value props(Json::objectValue); + Json::Value schema(Json::objectValue); + schema["type"] = "object"; + schema["properties"] = props; + return schema; +} + +// tools/call：remote_mouse（鼠标注入；归一化坐标 → 物理像素） +std::string BuildRemoteMouse(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) { + CMcpServer& mcp = CMcpServer::Instance(); + mcp.SweepIdleScreenCtrl(kScreenCtrlIdleTimeoutSec); + + uint64_t devId = 0; + context* ctx = nullptr; + std::string errJson; + if (!ResolveScreenCtrlSessionHost(id, args, parent, devId, ctx, errJson)) + return errJson; + + std::string sessionId = GetStringArg(args, "session_id"); + if (sessionId.empty()) + return BuildError(id, -32602, "Missing required parameter: session_id"); + + std::string action = GetStringArg(args, "action"); + if (action.empty()) + return BuildError(id, -32602, "Missing required parameter: action"); + + // ---- 纯参数解析（不依赖会话分辨率；BeginScreenCtrlAction 之后再无提前返回）---- + bool knownAction = + action == "move" || action == "down" || action == "up" || action == "click" || + action == "right_click" || action == "middle_click" || action == "drag" || action == "scroll"; + if (!knownAction) + return BuildError(id, -32602, "Unknown action: " + action); + + int button = 0; + std::string buttonStr = GetStringArg(args, "button"); + if (!ParseButton(buttonStr, button)) + return BuildError(id, -32602, "Invalid button: " + buttonStr + " (expected left/middle/right)"); + + int clicks = 1; + if (args.isMember("clicks")) { + if (!GetIntArg(args, "clicks", clicks)) + return BuildError(id, -32602, "clicks must be an integer 1..3"); + if (clicks < 1 || clicks > 3) + return BuildError(id, -32602, "clicks must be 1, 2 or 3"); + } + + int delta = 0; + if (args.isMember("delta") && !GetIntArg(args, "delta", delta)) + return BuildError(id, -32602, "delta must be an integer"); + + int notches = 0; // scroll 实际注入的滚动档数（= clamp(delta, -10, 10)），审计用 + + double nx = 0, ny = 0, nx2 = 0, ny2 = 0; + std::string coordErr; + if (!GetNormCoord(args, "x", nx, coordErr)) return BuildError(id, -32602, coordErr); + if (!GetNormCoord(args, "y", ny, coordErr)) return BuildError(id, -32602, coordErr); + if (action == "drag") { + if (!GetNormCoord(args, "x2", nx2, coordErr)) return BuildError(id, -32602, coordErr); + if (!GetNormCoord(args, "y2", ny2, coordErr)) return BuildError(id, -32602, coordErr); + } + + // 校验 session_id/busy 并置 busy，同时取物理分辨率（screenW/H>0 由会话就绪保证）。 + context* subCtx = nullptr; + int screenW = 0, screenH = 0; + int r = mcp.BeginScreenCtrlAction(devId, sessionId, subCtx, screenW, screenH); + if (r == 2) return BuildError(id, -32003, "Device busy: another injection is in flight for this session"); + if (r == 1) return BuildError(id, -32002, "Remote control session not found, not ready, or closed: " + sessionId); + + // 归一化坐标 → 物理像素（round + clamp 到 [0, screen-1]） + auto toPxX = [&](double n) -> int { int p = (int)(n * screenW + 0.5); return p < 0 ? 0 : (p > screenW - 1 ? screenW - 1 : p); }; + auto toPxY = [&](double n) -> int { int p = (int)(n * screenH + 0.5); return p < 0 ? 0 : (p > screenH - 1 ? screenH - 1 : p); }; + int px = toPxX(nx), py = toPxY(ny); + int px2 = 0, py2 = 0; + + UINT downMsg, upMsg; uint64_t downWParam; + ButtonMessages(button, downMsg, upMsg, downWParam); + + // 组装注入批次（按发送顺序；每个 MSG64 由客户端 ProcessCommand 分发到 SendInput） + std::vector batch; + if (action == "move") { + batch.push_back(BuildMouseMsg64(px, py, WM_MOUSEMOVE, 0)); + } else if (action == "down") { + batch.push_back(BuildMouseMsg64(px, py, downMsg, downWParam)); + } else if (action == "up") { + batch.push_back(BuildMouseMsg64(px, py, upMsg, 0)); + } else if (action == "click" || action == "right_click" || action == "middle_click") { + int b = (action == "right_click") ? 2 : (action == "middle_click" ? 1 : button); + UINT dMsg, uMsg; uint64_t dW; + ButtonMessages(b, dMsg, uMsg, dW); + for (int i = 0; i < clicks; ++i) { + batch.push_back(BuildMouseMsg64(px, py, dMsg, dW)); + batch.push_back(BuildMouseMsg64(px, py, uMsg, 0)); + } + } else if (action == "drag") { + px2 = toPxX(nx2); py2 = toPxY(ny2); + batch.push_back(BuildMouseMsg64(px, py, downMsg, downWParam)); + batch.push_back(BuildMouseMsg64(px2, py2, WM_MOUSEMOVE, 0)); + batch.push_back(BuildMouseMsg64(px2, py2, upMsg, 0)); + } else if (action == "scroll") { + // delta 正=向下滚（与 Web 控制台一致）；WM_MOUSEWHEEL 负值=向下滚，故取反。 + // 客户端对 WM_MOUSEWHEEL 会先 SetCursorPos(x,y) 再 MOUSEEVENTF_WHEEL，单条消息即可定位+滚动。 + notches = delta; + if (notches > 10) notches = 10; + if (notches < -10) notches = -10; + short wheelDelta = (short)(-notches * 120); + batch.push_back(BuildMouseMsg64(px, py, WM_MOUSEWHEEL, MAKEWPARAM(0, wheelDelta))); + } + + // 组包 [COMMAND_SCREEN_CONTROL][MSG64*N]，经屏幕子连接发送（主连接无效，设计 P2）。 + const int len = (int)(1 + batch.size() * sizeof(MSG64)); + std::vector packet((size_t)len); + packet[0] = COMMAND_SCREEN_CONTROL; + memcpy(packet.data() + 1, batch.data(), batch.size() * sizeof(MSG64)); + bool ok = subCtx->Send2Client(packet.data(), (ULONG)len) != FALSE; + + mcp.EndScreenCtrlAction(devId, sessionId); + + if (!ok) + return BuildError(id, -32004, "Failed to send mouse injection"); + + // 审计（不可关闭；含归一化坐标与换算后的物理像素） + if (parent) { + std::string audit = "host " + std::to_string(devId) + " remote-mouse: session " + sessionId + + " action=" + action; + char cbuf[96]; + sprintf(cbuf, " norm=(%.3f,%.3f) px=(%d,%d)", nx, ny, px, py); + audit += cbuf; + if (action == "drag") { + sprintf(cbuf, " -> norm2=(%.3f,%.3f) px2=(%d,%d)", nx2, ny2, px2, py2); + audit += cbuf; + } + if (action == "scroll") + audit += " notches=" + std::to_string(notches); + if (action == "click" || action == "right_click" || action == "middle_click") + audit += " clicks=" + std::to_string(clicks); + parent->PostMessageA(WM_SHOWERRORMSG, + (WPARAM)new CString(ToAnsi(audit, 936).c_str()), + (LPARAM)new CString(_TR("MCP远程控制"))); + } + + Json::Value result(Json::objectValue); + result["structuredContent"] = Json::Value(Json::objectValue); + Json::Value content(Json::arrayValue); + Json::Value item(Json::objectValue); + item["type"] = "text"; + item["text"] = std::string(u8"ok"); + content.append(item); + result["content"] = content; + result["isError"] = false; + + return BuildResult(id, result); +} + // tools/call 分派 std::string BuildToolsCall(const Json::Value& root, CMy2015RemoteDlg* parent) { const Json::Value& id = root["id"]; @@ -3144,6 +3439,7 @@ std::string BuildToolsCall(const Json::Value& root, CMy2015RemoteDlg* parent) { if (toolName == "remote_open") return BuildRemoteOpen(id, args, parent); if (toolName == "remote_close") return BuildRemoteClose(id, args, parent); if (toolName == "remote_keyboard") return BuildRemoteKeyboard(id, args, parent); + if (toolName == "remote_mouse") return BuildRemoteMouse(id, args, parent); return BuildError(id, -32602, "Unknown tool: " + (toolName.empty() ? std::string("(empty)") : toolName)); diff --git a/server/2015Remote/WebService.cpp b/server/2015Remote/WebService.cpp index b969159..2b98056 100644 --- a/server/2015Remote/WebService.cpp +++ b/server/2015Remote/WebService.cpp @@ -799,34 +799,18 @@ void CWebService::HandleMouse(void* ws_ptr, const std::string& msg) { return; } - // Build MSG64 structure - MSG64 msg64; - memset(&msg64, 0, sizeof(MSG64)); - msg64.pt.x = x; - msg64.pt.y = y; - msg64.lParam = MAKELPARAM(x, y); - msg64.time = GetTickCount(); + // Build MSG64 structure（共享 helper，见 WebService.h；与 MCP remote_mouse 复用） + MSG64 msg64 = BuildMouseMsg64(x, y, 0, 0); // 默认 WM_NULL；未知 button 时仍发送（保持原行为） // Map type and button to Windows message if (type == "down") { - if (button == 0) { - msg64.message = WM_LBUTTONDOWN; - msg64.wParam = MK_LBUTTON; - } else if (button == 1) { - msg64.message = WM_MBUTTONDOWN; - msg64.wParam = MK_MBUTTON; - } else if (button == 2) { - msg64.message = WM_RBUTTONDOWN; - msg64.wParam = MK_RBUTTON; - } + if (button == 0) { msg64.message = WM_LBUTTONDOWN; msg64.wParam = MK_LBUTTON; } + else if (button == 1) { msg64.message = WM_MBUTTONDOWN; msg64.wParam = MK_MBUTTON; } + else if (button == 2) { msg64.message = WM_RBUTTONDOWN; msg64.wParam = MK_RBUTTON; } } else if (type == "up") { - if (button == 0) { - msg64.message = WM_LBUTTONUP; - } else if (button == 1) { - msg64.message = WM_MBUTTONUP; - } else if (button == 2) { - msg64.message = WM_RBUTTONUP; - } + if (button == 0) { msg64.message = WM_LBUTTONUP; } + else if (button == 1) { msg64.message = WM_MBUTTONUP; } + else if (button == 2) { msg64.message = WM_RBUTTONUP; } } else if (type == "move") { msg64.message = WM_MOUSEMOVE; } else if (type == "wheel") { @@ -845,13 +829,8 @@ void CWebService::HandleMouse(void* ws_ptr, const std::string& msg) { if (clientType != GetClientType(CLIENT_TYPE_MACOS) && clientType != "macOS") { return; // Skip dblclick for non-macOS clients } - if (button == 0) { - msg64.message = WM_LBUTTONDBLCLK; - msg64.wParam = MK_LBUTTON; - } else if (button == 2) { - msg64.message = WM_RBUTTONDBLCLK; - msg64.wParam = MK_RBUTTON; - } + if (button == 0) { msg64.message = WM_LBUTTONDBLCLK; msg64.wParam = MK_LBUTTON; } + else if (button == 2) { msg64.message = WM_RBUTTONDBLCLK; msg64.wParam = MK_RBUTTON; } } else { return; // Unknown type } diff --git a/server/2015Remote/WebService.h b/server/2015Remote/WebService.h index 2fdeeed..f312822 100644 --- a/server/2015Remote/WebService.h +++ b/server/2015Remote/WebService.h @@ -48,6 +48,21 @@ inline MSG64 BuildKeyMsg64(int keyCode, bool isDown, bool isSysKey) { return msg64; } +// 构造单个鼠标注入 MSG64（复用 WebService::HandleMouse 的基座字段逻辑，抽取为共享 helper， +// 供 Web 与 MCP 远程控制两处使用，见 docs/Mcp_RemoteControl_Design.md §6.2）。 +// x/y 为物理像素坐标（已由调用方从归一化坐标映射）；message/wParam 由调用方按动作设定。 +inline MSG64 BuildMouseMsg64(int x, int y, UINT message, uint64_t wParam) { + MSG64 msg64; + memset(&msg64, 0, sizeof(MSG64)); + msg64.pt.x = x; + msg64.pt.y = y; + msg64.lParam = MAKELPARAM(x, y); + msg64.time = GetTickCount(); + msg64.message = message; + msg64.wParam = wParam; + return msg64; +} + // Web client state struct WebClient { std::string token;