Feature: Add native region, window and monitor capture to get_screenshot

New Windows clients advertise CLIENT_CAP_SCREEN_REGION and accept a 24-byte
ScreenPreviewReqEx appended to the existing 8-byte preview request, so
get_screenshot can capture a region, a window or a single monitor at 1:1
resolution instead of only the scaled virtual desktop. Successful v2
responses append an optional ScreenCaptureMeta block reporting the source
rect and scale, letting callers map coordinates back for remote input.
Old clients and old servers keep the v1 byte stream unchanged.

The 8192 px per-side bound only applies to native (1:1) capture, where it
guards bitmap memory; scaled captures are unaffected so ultra-wide virtual
desktops (for example triple 4K) still produce thumbnails and previews.
max_width is clamped to [64,1920] as before instead of rejected, matching
the documented schema.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-10 16:34:10 +01:00
1 parent b2c0d81396
commit 235a8a622c
11 files changed
+668 -97

No files matched your search

+16
View File
@@ -10403,6 +10403,22 @@ void CMy2015RemoteDlg::SendScreenPreviewRequest(context* ctx, WORD reqId, WORD m
ctx->Send2Client((PBYTE)&req, sizeof(req));
}
void CMy2015RemoteDlg::SendScreenPreviewRequestEx(context* ctx, WORD reqId, WORD maxWidth,
BYTE jpegQuality, const ScreenPreviewReqEx& ex)
{
if (!ctx) return;
BYTE buf[sizeof(ScreenPreviewReq) + sizeof(ScreenPreviewReqEx)] = {};
ScreenPreviewReq req;
memset(&req, 0, sizeof(req));
req.cmd = COMMAND_SCREEN_PREVIEW_REQ;
req.reqId = reqId;
req.maxWidth = maxWidth;
req.jpegQuality = jpegQuality;
memcpy(buf, &req, sizeof(req));
memcpy(buf + sizeof(req), &ex, sizeof(ex));
ctx->Send2Client(buf, (int)sizeof(buf));
}
// 收到 TOKEN_SCREEN_PREVIEW_RSP(在主线程)
// wParam: 未使用
// LPARAM: 指向堆分配的 PreviewRspMsg*(含 clientId + 完整响应包:[Header|JPEG]),
+4
View File
@@ -673,6 +673,10 @@ public:
void ChooseScreenPreviewParams(context* ctx, WORD& maxWidth, BYTE& jpegQuality) const;
// 发起预览请求;reqId 应与 m_PreviewReqId 同步
void SendScreenPreviewRequest(context* ctx, WORD reqId, WORD maxWidth, BYTE jpegQuality);
// v2:在 v1 请求后追加 ScreenPreviewReqEx(区域/窗口/显示器 + 原生分辨率标志)。
// 只对声明 CLIENT_CAP_SCREEN_REGION 的客户端使用。
void SendScreenPreviewRequestEx(context* ctx, WORD reqId, WORD maxWidth, BYTE jpegQuality,
const ScreenPreviewReqEx& ex);
// 预览响应规范化:Linux 客户端回原始 BGRA(SCREEN_PREVIEW_FMT_BGRX),在 IO 线程入口
// 统一转成 JPEG,下游(浮窗/列表缩略图/MCP)只处理 JPEG。返回 false 表示畸形包,应丢弃。
static bool NormalizeScreenPreviewPacket(std::vector<BYTE>& pkt);
+208 -15
View File
@@ -1030,6 +1030,9 @@ Json::Value BuildHostDetailOutputSchema() {
platform["description"] = u8"客户端形态:EXE/DLL/MEXE/SC/MDLL(Windows)、LNX(Linux)、MAC、APK";
capProps["platform"] = platform;
capProps["screenPreview"] = boolProp(u8"能否截屏(get_screenshot 的前提)");
capProps["screenRegion"] = boolProp(
u8"能否按区域/窗口/显示器原生分辨率截屏(get_screenshot 的 mode=region/window/monitor 前提);"
u8"仅声明 CLIENT_CAP_SCREEN_REGION 的新 Windows 客户端为 true");
capProps["remoteControl"] = boolProp(
u8"能否建立远程控制会话(remote_open/remote_mouse/remote_keyboard 的前提);"
u8"还要求服务端 Web 远程服务已启动(WebSvrPort>0)");
@@ -1767,6 +1770,24 @@ Json::Value BuildScreenshotOutputSchema() {
imageProp["properties"] = imageProps;
props["image"] = imageProp;
Json::Value capProp(Json::objectValue);
capProp["type"] = "object";
Json::Value capProps(Json::objectValue);
Json::Value capMode(Json::objectValue);
capMode["type"] = "string";
capProps["mode"] = capMode;
const char* capInts[] = { "x", "y", "width", "height", "scale" };
for (const char* f : capInts) {
Json::Value s(Json::objectValue);
s["type"] = "integer";
capProps[f] = s;
}
Json::Value capNative(Json::objectValue);
capNative["type"] = "boolean";
capProps["native"] = capNative;
capProp["properties"] = capProps;
props["capture"] = capProp;
Json::Value schema(Json::objectValue);
schema["type"] = "object";
schema["properties"] = props;
@@ -1826,7 +1847,7 @@ Json::Value BuildGetForegroundWindowOutputSchema() {
return schema;
}
// get_screenshot 的 inputSchema(id 必填,max_width 可选)
// get_screenshot 的 inputSchema(id 必填;mode/region/window/monitor 需新 Windows 客户端)
Json::Value BuildGetScreenshotInputSchema() {
Json::Value props(Json::objectValue);
@@ -1835,9 +1856,53 @@ Json::Value BuildGetScreenshotInputSchema() {
idProp["description"] = u8"主机 id,取 list_online_hosts / search_hosts 返回的 id 字段";
props["id"] = idProp;
Json::Value modeProp(Json::objectValue);
modeProp["type"] = "string";
Json::Value modeEnum(Json::arrayValue);
modeEnum.append("screen");
modeEnum.append("region");
modeEnum.append("window");
modeEnum.append("monitor");
modeProp["enum"] = modeEnum;
modeProp["description"] = u8"抓取范围,默认 screen。screen=整个虚拟桌面(可缩放);region=指定矩形;window=指定窗口;monitor=指定显示器。region/window/monitor 默认按源尺寸 1:1 抓取(原生分辨率,适合 OCR/视觉读字),只有显式传 max_width 才缩放;这三种需要较新的 Windows 客户端(能力位 CLIENT_CAP_SCREEN_REGION),否则返回 -32005。";
props["mode"] = modeProp;
Json::Value xProp(Json::objectValue);
xProp["type"] = "integer";
xProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 X 坐标(可为负;多显示器时以虚拟桌面左上角为原点)。省略按 0 处理。";
props["x"] = xProp;
Json::Value yProp(Json::objectValue);
yProp["type"] = "integer";
yProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 Y 坐标(可为负)。省略按 0 处理。";
props["y"] = yProp;
Json::Value wProp(Json::objectValue);
wProp["type"] = "integer";
wProp["description"] = u8"region 模式必填:矩形宽度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
props["width"] = wProp;
Json::Value hProp(Json::objectValue);
hProp["type"] = "integer";
hProp["description"] = u8"region 模式必填:矩形高度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
props["height"] = hProp;
Json::Value hwndProp(Json::objectValue);
Json::Value hwndTypes(Json::arrayValue);
hwndTypes.append("integer");
hwndTypes.append("string");
hwndProp["type"] = hwndTypes;
hwndProp["description"] = u8"window 模式必填:窗口句柄(整数或 0x 前缀字符串),取 host_list_windows / get_foreground_window 返回的 hwnd。窗口最小化时无法抓取(返回错误),先 activate/restore。";
props["hwnd"] = hwndProp;
Json::Value monProp(Json::objectValue);
monProp["type"] = "integer";
monProp["description"] = u8"monitor 模式:显示器序号,0=主显示器(默认),1..N 为其余显示器(按系统枚举顺序)。序号越界返回错误。";
props["monitor"] = monProp;
Json::Value mwProp(Json::objectValue);
mwProp["type"] = "integer";
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);传 1920 可拿到接近原分辨率(1080p 源屏即原分辨率,4K 源屏最多 1920),供 AI 视觉/OCR 场景提升清晰度。";
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。screen 模式省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);region/window/monitor 模式省略/传 0 时按源尺寸 1:1 抓取(原生分辨率),显式传入才等比缩小。";
props["max_width"] = mwProp;
Json::Value schema(Json::objectValue);
@@ -2131,7 +2196,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
{
Json::Value tool(Json::objectValue);
tool["name"] = "get_screenshot";
tool["description"] = u8"截取指定在线主机的一帧屏幕,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。";
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选
tool["outputSchema"] = BuildScreenshotOutputSchema();
@@ -2855,6 +2920,8 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
Json::Value caps(Json::objectValue);
caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK
caps["screenPreview"] = ctx->SupportsScreenPreview();
// screenRegion → get_screenshot 的 region/window/monitor 模式(SupportsScreenCapture)
caps["screenRegion"] = ctx->SupportsScreenCapture();
// 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源:
// Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。
// monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。
@@ -3198,6 +3265,30 @@ std::string BuildGetActivityHistory(const Json::Value& id, const Json::Value& ar
return BuildResult(id, result);
}
// 读可选整数参数(缺省/ null 时用 defVal);越界/类型错误返回 false 并填好错误响应
static bool GetOptIntInRange(const Json::Value& id, const Json::Value& args, const char* key,
int lo, int hi, int defVal, int& out, std::string& errJson) {
out = defVal;
if (!args.isMember(key) || args[key].isNull()) return true;
if (!args[key].isInt()) {
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": expected an integer");
return false;
}
int n = args[key].asInt();
if (n < lo || n > hi) {
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": out of range [" +
std::to_string(lo) + "," + std::to_string(hi) + "]");
return false;
}
out = n;
return true;
}
// 定义在文件后部(expect_hwnd 解析),这里先声明以便 get_screenshot 复用
static bool GetOptionalWindowArg(const Json::Value& args, const char* key, uint32_t& out,
bool& present, std::string& err);
static std::string Trim(const std::string& s);
// tools/call:get_screenshot(屏幕预览链路,主连接 RPC,不建子链接、不弹框)
std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
uint64_t devId = 0;
@@ -3214,6 +3305,65 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
if (!ctx->SupportsScreenPreview())
return BuildError(id, -32005, "Host does not support screen preview");
// ---- 抓取范围 ----
std::string modeStr = Trim(GetStringArg(args, "mode"));
if (modeStr.empty()) modeStr = "screen";
int mode = -1;
if (modeStr == "screen") mode = SCREEN_CAPTURE_VDESKTOP;
else if (modeStr == "region") mode = SCREEN_CAPTURE_REGION;
else if (modeStr == "window") mode = SCREEN_CAPTURE_WINDOW;
else if (modeStr == "monitor") mode = SCREEN_CAPTURE_MONITOR;
else return BuildError(id, -32602,
"Invalid mode: " + modeStr + " (expect screen|region|window|monitor)");
// 新客户端(CLIENT_CAP_SCREEN_REGION)支持 v2;老客户端只能整屏。
const bool supportsRegion = ctx->SupportsScreenCapture();
if (mode != SCREEN_CAPTURE_VDESKTOP && !supportsRegion)
return BuildError(id, -32005,
"mode=region/window/monitor requires a newer Windows client "
"(screen region capability); use mode=screen instead");
std::string errJson;
int rx = 0, ry = 0, rw = 0, rh = 0, monitorIdx = 0;
uint32_t hwnd = 0;
if (mode == SCREEN_CAPTURE_REGION) {
if (!GetOptIntInRange(id, args, "x", -100000, 100000, 0, rx, errJson)) return errJson;
if (!GetOptIntInRange(id, args, "y", -100000, 100000, 0, ry, errJson)) return errJson;
if (!args.isMember("width") || args["width"].isNull())
return BuildError(id, -32602, "Missing required parameter: width (mode=region)");
if (!args.isMember("height") || args["height"].isNull())
return BuildError(id, -32602, "Missing required parameter: height (mode=region)");
if (!GetOptIntInRange(id, args, "width", 1, 8192, 0, rw, errJson)) return errJson;
if (!GetOptIntInRange(id, args, "height", 1, 8192, 0, rh, errJson)) return errJson;
} else if (mode == SCREEN_CAPTURE_WINDOW) {
bool hasHwnd = false;
std::string oerr;
if (!GetOptionalWindowArg(args, "hwnd", hwnd, hasHwnd, oerr))
return BuildError(id, -32602, oerr);
if (!hasHwnd)
return BuildError(id, -32602, "Missing required parameter: hwnd (mode=window)");
} else if (mode == SCREEN_CAPTURE_MONITOR) {
if (!GetOptIntInRange(id, args, "monitor", 0, 255, 0, monitorIdx, errJson)) return errJson;
}
// max_width:screen 模式下缺省/0 沿用 RTT 自适应缩略图档位;region/window/monitor
// 缺省/0 表示原生分辨率(1:1),显式传入才等比缩放。
bool hasMaxWidth = false;
int maxWidthArg = 0;
if (args.isMember("max_width") && !args["max_width"].isNull()) {
if (!args["max_width"].isInt())
return BuildError(id, -32602, "Invalid max_width: expected an integer");
maxWidthArg = args["max_width"].asInt();
// 与 inputSchema 描述一致:钳制到 [64,1920](老行为),不因越界直接报错
if (maxWidthArg > 0) {
if (maxWidthArg < 64) maxWidthArg = 64;
if (maxWidthArg > 1920) maxWidthArg = 1920;
hasMaxWidth = true;
} else {
maxWidthArg = 0;
}
}
CMcpServer& mcp = CMcpServer::Instance();
if (!mcp.BeginPending(devId, "get_screenshot"))
return BuildError(id, -32003, BusyMessage("Device busy: another request is pending for this host", devId));
@@ -3225,35 +3375,66 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
WORD maxWidth = 0;
BYTE quality = 0;
parent->ChooseScreenPreviewParams(ctx, maxWidth, quality);
if (hasMaxWidth) maxWidth = (WORD)maxWidthArg;
// 可选 max_width:缺省/0 沿用缩略图档位;>0 覆盖宽度(钳制到客户端上限 [64,1920]),
// 供 AI 视觉/OCR 场景请求接近原分辨率的大图。jpegQuality 仍沿用档位自适应值。
if (args.isMember("max_width") && args["max_width"].isInt()) {
int mw = args["max_width"].asInt();
if (mw > 0) {
if (mw < 64) mw = 64;
if (mw > 1920) mw = 1920;
maxWidth = (WORD)mw;
if (mode == SCREEN_CAPTURE_VDESKTOP) {
if (supportsRegion) {
// 新客户端也走 v2:多回一份源矩形元数据,但目标仍是整屏 + 缩放。
ScreenPreviewReqEx ex = {};
ex.mode = SCREEN_CAPTURE_VDESKTOP;
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
} else {
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
}
} else {
ScreenPreviewReqEx ex = {};
ex.mode = (uint8_t)mode;
if (!hasMaxWidth) ex.flags |= SCREEN_CAPTURE_F_NATIVE;
ex.x = rx;
ex.y = ry;
ex.w = rw;
ex.h = rh;
ex.hwnd = hwnd;
ex.monitor = (uint16_t)monitorIdx;
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
}
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
std::vector<BYTE> data;
if (!mcp.WaitPending(devId, data, mcp.ToolTimeoutMs()))
return BuildError(id, -32001, "Timeout waiting for screenshot");
// data = [ScreenPreviewRspHeader][JPEG]
// data = [ScreenPreviewRspHeader][JPEG][可选 ScreenCaptureMeta]
if (data.size() < sizeof(ScreenPreviewRspHeader))
return BuildError(id, -32000, "Invalid screenshot response");
const ScreenPreviewRspHeader* hdr = reinterpret_cast<const ScreenPreviewRspHeader*>(data.data());
if (hdr->status == SCREEN_PREVIEW_BAD_TARGET) {
if (mode == SCREEN_CAPTURE_WINDOW)
return BuildError(id, -32009, "Window hwnd=" + std::to_string(hwnd) +
" is not capturable (not found, minimized, off-screen, "
"or larger than 8192 px per side)");
if (mode == SCREEN_CAPTURE_MONITOR)
return BuildError(id, -32602, "Monitor index " + std::to_string(monitorIdx) +
" is out of range");
if (mode == SCREEN_CAPTURE_REGION)
return BuildError(id, -32602, "Capture region is empty after clipping to the virtual desktop");
return BuildError(id, -32000, "Screen is not capturable");
}
if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG ||
hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) {
return BuildError(id, -32000,
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")");
}
// 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。
// 偏移 = 16 + bytes,可能不是 4 字节对齐,按字节拷贝到局部变量再读。
bool hasMeta = false;
ScreenCaptureMeta meta = {};
if (data.size() >= sizeof(ScreenPreviewRspHeader) + hdr->bytes + sizeof(ScreenCaptureMeta)) {
memcpy(&meta, data.data() + sizeof(ScreenPreviewRspHeader) + hdr->bytes, sizeof(meta));
hasMeta = (meta.magic == SCREEN_CAPTURE_META_MAGIC);
}
std::string b64 = httplib::detail::base64_encode(
std::string((const char*)data.data() + sizeof(ScreenPreviewRspHeader), hdr->bytes));
@@ -3263,9 +3444,21 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
image["height"] = (Json::UInt64)hdr->height;
image["bytes"] = (Json::UInt64)hdr->bytes;
Json::Value capture(Json::objectValue);
capture["mode"] = modeStr;
if (hasMeta) {
capture["x"] = meta.x;
capture["y"] = meta.y;
capture["width"] = meta.w;
capture["height"] = meta.h;
capture["scale"] = (int)meta.scale;
capture["native"] = meta.scale >= 100;
}
Json::Value result(Json::objectValue);
Json::Value structuredContent(Json::objectValue);
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
structuredContent["capture"] = capture;
result["structuredContent"] = structuredContent;
Json::Value content(Json::arrayValue);
+8
View File
@@ -123,6 +123,14 @@ public:
return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_WINDOW_INFO) != 0;
}
// 检查客户端是否支持区域/窗口/显示器原生分辨率截图(COMMAND_SCREEN_PREVIEW_REQ 的
// v2 扩展,CLIENT_CAP_SCREEN_REGION)。未声明时 get_screenshot 只能整屏抓取。
bool SupportsScreenCapture() const {
CString caps = GetClientData(ONLINELIST_CAPABILITIES);
if (caps.IsEmpty()) return false;
return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_SCREEN_REGION) != 0;
}
// 检查客户端是否支持远程协议切换(COMMAND_SET_PROTO)。老客户端没有这个命令,
// 收到也会走 default 丢弃,服务端据此把「协议切换」菜单置灰。
bool SupportsProtoSwitch() const {