Feature: Add native region, window and monitor capture to get_screenshot
New Windows clients advertise CLIENT_CAP_SCREEN_REGION and accept a 24-byte ScreenPreviewReqEx appended to the existing 8-byte preview request, so get_screenshot can capture a region, a window or a single monitor at 1:1 resolution instead of only the scaled virtual desktop. Successful v2 responses append an optional ScreenCaptureMeta block reporting the source rect and scale, letting callers map coordinates back for remote input. Old clients and old servers keep the v1 byte stream unchanged. The 8192 px per-side bound only applies to native (1:1) capture, where it guards bitmap memory; scaled captures are unaffected so ultra-wide virtual desktops (for example triple 4K) still produce thumbnails and previews. max_width is clamped to [64,1920] as before instead of rejected, matching the documented schema. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
b2c0d81396
commit
235a8a622c
11 files changed
+668
-97
No files matched your search
+208
-15
@@ -1030,6 +1030,9 @@ Json::Value BuildHostDetailOutputSchema() {
|
||||
platform["description"] = u8"客户端形态:EXE/DLL/MEXE/SC/MDLL(Windows)、LNX(Linux)、MAC、APK";
|
||||
capProps["platform"] = platform;
|
||||
capProps["screenPreview"] = boolProp(u8"能否截屏(get_screenshot 的前提)");
|
||||
capProps["screenRegion"] = boolProp(
|
||||
u8"能否按区域/窗口/显示器原生分辨率截屏(get_screenshot 的 mode=region/window/monitor 前提);"
|
||||
u8"仅声明 CLIENT_CAP_SCREEN_REGION 的新 Windows 客户端为 true");
|
||||
capProps["remoteControl"] = boolProp(
|
||||
u8"能否建立远程控制会话(remote_open/remote_mouse/remote_keyboard 的前提);"
|
||||
u8"还要求服务端 Web 远程服务已启动(WebSvrPort>0)");
|
||||
@@ -1767,6 +1770,24 @@ Json::Value BuildScreenshotOutputSchema() {
|
||||
imageProp["properties"] = imageProps;
|
||||
props["image"] = imageProp;
|
||||
|
||||
Json::Value capProp(Json::objectValue);
|
||||
capProp["type"] = "object";
|
||||
Json::Value capProps(Json::objectValue);
|
||||
Json::Value capMode(Json::objectValue);
|
||||
capMode["type"] = "string";
|
||||
capProps["mode"] = capMode;
|
||||
const char* capInts[] = { "x", "y", "width", "height", "scale" };
|
||||
for (const char* f : capInts) {
|
||||
Json::Value s(Json::objectValue);
|
||||
s["type"] = "integer";
|
||||
capProps[f] = s;
|
||||
}
|
||||
Json::Value capNative(Json::objectValue);
|
||||
capNative["type"] = "boolean";
|
||||
capProps["native"] = capNative;
|
||||
capProp["properties"] = capProps;
|
||||
props["capture"] = capProp;
|
||||
|
||||
Json::Value schema(Json::objectValue);
|
||||
schema["type"] = "object";
|
||||
schema["properties"] = props;
|
||||
@@ -1826,7 +1847,7 @@ Json::Value BuildGetForegroundWindowOutputSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
// get_screenshot 的 inputSchema(id 必填,max_width 可选)
|
||||
// get_screenshot 的 inputSchema(id 必填;mode/region/window/monitor 需新 Windows 客户端)
|
||||
Json::Value BuildGetScreenshotInputSchema() {
|
||||
Json::Value props(Json::objectValue);
|
||||
|
||||
@@ -1835,9 +1856,53 @@ Json::Value BuildGetScreenshotInputSchema() {
|
||||
idProp["description"] = u8"主机 id,取 list_online_hosts / search_hosts 返回的 id 字段";
|
||||
props["id"] = idProp;
|
||||
|
||||
Json::Value modeProp(Json::objectValue);
|
||||
modeProp["type"] = "string";
|
||||
Json::Value modeEnum(Json::arrayValue);
|
||||
modeEnum.append("screen");
|
||||
modeEnum.append("region");
|
||||
modeEnum.append("window");
|
||||
modeEnum.append("monitor");
|
||||
modeProp["enum"] = modeEnum;
|
||||
modeProp["description"] = u8"抓取范围,默认 screen。screen=整个虚拟桌面(可缩放);region=指定矩形;window=指定窗口;monitor=指定显示器。region/window/monitor 默认按源尺寸 1:1 抓取(原生分辨率,适合 OCR/视觉读字),只有显式传 max_width 才缩放;这三种需要较新的 Windows 客户端(能力位 CLIENT_CAP_SCREEN_REGION),否则返回 -32005。";
|
||||
props["mode"] = modeProp;
|
||||
|
||||
Json::Value xProp(Json::objectValue);
|
||||
xProp["type"] = "integer";
|
||||
xProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 X 坐标(可为负;多显示器时以虚拟桌面左上角为原点)。省略按 0 处理。";
|
||||
props["x"] = xProp;
|
||||
|
||||
Json::Value yProp(Json::objectValue);
|
||||
yProp["type"] = "integer";
|
||||
yProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 Y 坐标(可为负)。省略按 0 处理。";
|
||||
props["y"] = yProp;
|
||||
|
||||
Json::Value wProp(Json::objectValue);
|
||||
wProp["type"] = "integer";
|
||||
wProp["description"] = u8"region 模式必填:矩形宽度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
|
||||
props["width"] = wProp;
|
||||
|
||||
Json::Value hProp(Json::objectValue);
|
||||
hProp["type"] = "integer";
|
||||
hProp["description"] = u8"region 模式必填:矩形高度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
|
||||
props["height"] = hProp;
|
||||
|
||||
Json::Value hwndProp(Json::objectValue);
|
||||
Json::Value hwndTypes(Json::arrayValue);
|
||||
hwndTypes.append("integer");
|
||||
hwndTypes.append("string");
|
||||
hwndProp["type"] = hwndTypes;
|
||||
hwndProp["description"] = u8"window 模式必填:窗口句柄(整数或 0x 前缀字符串),取 host_list_windows / get_foreground_window 返回的 hwnd。窗口最小化时无法抓取(返回错误),先 activate/restore。";
|
||||
props["hwnd"] = hwndProp;
|
||||
|
||||
Json::Value monProp(Json::objectValue);
|
||||
monProp["type"] = "integer";
|
||||
monProp["description"] = u8"monitor 模式:显示器序号,0=主显示器(默认),1..N 为其余显示器(按系统枚举顺序)。序号越界返回错误。";
|
||||
props["monitor"] = monProp;
|
||||
|
||||
Json::Value mwProp(Json::objectValue);
|
||||
mwProp["type"] = "integer";
|
||||
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);传 1920 可拿到接近原分辨率(1080p 源屏即原分辨率,4K 源屏最多 1920),供 AI 视觉/OCR 场景提升清晰度。";
|
||||
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。screen 模式省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);region/window/monitor 模式省略/传 0 时按源尺寸 1:1 抓取(原生分辨率),显式传入才等比缩小。";
|
||||
props["max_width"] = mwProp;
|
||||
|
||||
Json::Value schema(Json::objectValue);
|
||||
@@ -2131,7 +2196,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
|
||||
{
|
||||
Json::Value tool(Json::objectValue);
|
||||
tool["name"] = "get_screenshot";
|
||||
tool["description"] = u8"截取指定在线主机的一帧屏幕,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。";
|
||||
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
|
||||
|
||||
tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选
|
||||
tool["outputSchema"] = BuildScreenshotOutputSchema();
|
||||
@@ -2855,6 +2920,8 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
|
||||
Json::Value caps(Json::objectValue);
|
||||
caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK
|
||||
caps["screenPreview"] = ctx->SupportsScreenPreview();
|
||||
// screenRegion → get_screenshot 的 region/window/monitor 模式(SupportsScreenCapture)
|
||||
caps["screenRegion"] = ctx->SupportsScreenCapture();
|
||||
// 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源:
|
||||
// Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。
|
||||
// monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。
|
||||
@@ -3198,6 +3265,30 @@ std::string BuildGetActivityHistory(const Json::Value& id, const Json::Value& ar
|
||||
return BuildResult(id, result);
|
||||
}
|
||||
|
||||
// 读可选整数参数(缺省/ null 时用 defVal);越界/类型错误返回 false 并填好错误响应
|
||||
static bool GetOptIntInRange(const Json::Value& id, const Json::Value& args, const char* key,
|
||||
int lo, int hi, int defVal, int& out, std::string& errJson) {
|
||||
out = defVal;
|
||||
if (!args.isMember(key) || args[key].isNull()) return true;
|
||||
if (!args[key].isInt()) {
|
||||
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": expected an integer");
|
||||
return false;
|
||||
}
|
||||
int n = args[key].asInt();
|
||||
if (n < lo || n > hi) {
|
||||
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": out of range [" +
|
||||
std::to_string(lo) + "," + std::to_string(hi) + "]");
|
||||
return false;
|
||||
}
|
||||
out = n;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 定义在文件后部(expect_hwnd 解析),这里先声明以便 get_screenshot 复用
|
||||
static bool GetOptionalWindowArg(const Json::Value& args, const char* key, uint32_t& out,
|
||||
bool& present, std::string& err);
|
||||
static std::string Trim(const std::string& s);
|
||||
|
||||
// tools/call:get_screenshot(屏幕预览链路,主连接 RPC,不建子链接、不弹框)
|
||||
std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
|
||||
uint64_t devId = 0;
|
||||
@@ -3214,6 +3305,65 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
if (!ctx->SupportsScreenPreview())
|
||||
return BuildError(id, -32005, "Host does not support screen preview");
|
||||
|
||||
// ---- 抓取范围 ----
|
||||
std::string modeStr = Trim(GetStringArg(args, "mode"));
|
||||
if (modeStr.empty()) modeStr = "screen";
|
||||
int mode = -1;
|
||||
if (modeStr == "screen") mode = SCREEN_CAPTURE_VDESKTOP;
|
||||
else if (modeStr == "region") mode = SCREEN_CAPTURE_REGION;
|
||||
else if (modeStr == "window") mode = SCREEN_CAPTURE_WINDOW;
|
||||
else if (modeStr == "monitor") mode = SCREEN_CAPTURE_MONITOR;
|
||||
else return BuildError(id, -32602,
|
||||
"Invalid mode: " + modeStr + " (expect screen|region|window|monitor)");
|
||||
|
||||
// 新客户端(CLIENT_CAP_SCREEN_REGION)支持 v2;老客户端只能整屏。
|
||||
const bool supportsRegion = ctx->SupportsScreenCapture();
|
||||
if (mode != SCREEN_CAPTURE_VDESKTOP && !supportsRegion)
|
||||
return BuildError(id, -32005,
|
||||
"mode=region/window/monitor requires a newer Windows client "
|
||||
"(screen region capability); use mode=screen instead");
|
||||
|
||||
std::string errJson;
|
||||
int rx = 0, ry = 0, rw = 0, rh = 0, monitorIdx = 0;
|
||||
uint32_t hwnd = 0;
|
||||
if (mode == SCREEN_CAPTURE_REGION) {
|
||||
if (!GetOptIntInRange(id, args, "x", -100000, 100000, 0, rx, errJson)) return errJson;
|
||||
if (!GetOptIntInRange(id, args, "y", -100000, 100000, 0, ry, errJson)) return errJson;
|
||||
if (!args.isMember("width") || args["width"].isNull())
|
||||
return BuildError(id, -32602, "Missing required parameter: width (mode=region)");
|
||||
if (!args.isMember("height") || args["height"].isNull())
|
||||
return BuildError(id, -32602, "Missing required parameter: height (mode=region)");
|
||||
if (!GetOptIntInRange(id, args, "width", 1, 8192, 0, rw, errJson)) return errJson;
|
||||
if (!GetOptIntInRange(id, args, "height", 1, 8192, 0, rh, errJson)) return errJson;
|
||||
} else if (mode == SCREEN_CAPTURE_WINDOW) {
|
||||
bool hasHwnd = false;
|
||||
std::string oerr;
|
||||
if (!GetOptionalWindowArg(args, "hwnd", hwnd, hasHwnd, oerr))
|
||||
return BuildError(id, -32602, oerr);
|
||||
if (!hasHwnd)
|
||||
return BuildError(id, -32602, "Missing required parameter: hwnd (mode=window)");
|
||||
} else if (mode == SCREEN_CAPTURE_MONITOR) {
|
||||
if (!GetOptIntInRange(id, args, "monitor", 0, 255, 0, monitorIdx, errJson)) return errJson;
|
||||
}
|
||||
|
||||
// max_width:screen 模式下缺省/0 沿用 RTT 自适应缩略图档位;region/window/monitor
|
||||
// 缺省/0 表示原生分辨率(1:1),显式传入才等比缩放。
|
||||
bool hasMaxWidth = false;
|
||||
int maxWidthArg = 0;
|
||||
if (args.isMember("max_width") && !args["max_width"].isNull()) {
|
||||
if (!args["max_width"].isInt())
|
||||
return BuildError(id, -32602, "Invalid max_width: expected an integer");
|
||||
maxWidthArg = args["max_width"].asInt();
|
||||
// 与 inputSchema 描述一致:钳制到 [64,1920](老行为),不因越界直接报错
|
||||
if (maxWidthArg > 0) {
|
||||
if (maxWidthArg < 64) maxWidthArg = 64;
|
||||
if (maxWidthArg > 1920) maxWidthArg = 1920;
|
||||
hasMaxWidth = true;
|
||||
} else {
|
||||
maxWidthArg = 0;
|
||||
}
|
||||
}
|
||||
|
||||
CMcpServer& mcp = CMcpServer::Instance();
|
||||
if (!mcp.BeginPending(devId, "get_screenshot"))
|
||||
return BuildError(id, -32003, BusyMessage("Device busy: another request is pending for this host", devId));
|
||||
@@ -3225,35 +3375,66 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
WORD maxWidth = 0;
|
||||
BYTE quality = 0;
|
||||
parent->ChooseScreenPreviewParams(ctx, maxWidth, quality);
|
||||
if (hasMaxWidth) maxWidth = (WORD)maxWidthArg;
|
||||
|
||||
// 可选 max_width:缺省/0 沿用缩略图档位;>0 覆盖宽度(钳制到客户端上限 [64,1920]),
|
||||
// 供 AI 视觉/OCR 场景请求接近原分辨率的大图。jpegQuality 仍沿用档位自适应值。
|
||||
if (args.isMember("max_width") && args["max_width"].isInt()) {
|
||||
int mw = args["max_width"].asInt();
|
||||
if (mw > 0) {
|
||||
if (mw < 64) mw = 64;
|
||||
if (mw > 1920) mw = 1920;
|
||||
maxWidth = (WORD)mw;
|
||||
if (mode == SCREEN_CAPTURE_VDESKTOP) {
|
||||
if (supportsRegion) {
|
||||
// 新客户端也走 v2:多回一份源矩形元数据,但目标仍是整屏 + 缩放。
|
||||
ScreenPreviewReqEx ex = {};
|
||||
ex.mode = SCREEN_CAPTURE_VDESKTOP;
|
||||
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
|
||||
} else {
|
||||
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
|
||||
}
|
||||
} else {
|
||||
ScreenPreviewReqEx ex = {};
|
||||
ex.mode = (uint8_t)mode;
|
||||
if (!hasMaxWidth) ex.flags |= SCREEN_CAPTURE_F_NATIVE;
|
||||
ex.x = rx;
|
||||
ex.y = ry;
|
||||
ex.w = rw;
|
||||
ex.h = rh;
|
||||
ex.hwnd = hwnd;
|
||||
ex.monitor = (uint16_t)monitorIdx;
|
||||
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
|
||||
}
|
||||
|
||||
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
|
||||
|
||||
std::vector<BYTE> data;
|
||||
if (!mcp.WaitPending(devId, data, mcp.ToolTimeoutMs()))
|
||||
return BuildError(id, -32001, "Timeout waiting for screenshot");
|
||||
|
||||
// data = [ScreenPreviewRspHeader][JPEG]
|
||||
// data = [ScreenPreviewRspHeader][JPEG][可选 ScreenCaptureMeta]
|
||||
if (data.size() < sizeof(ScreenPreviewRspHeader))
|
||||
return BuildError(id, -32000, "Invalid screenshot response");
|
||||
|
||||
const ScreenPreviewRspHeader* hdr = reinterpret_cast<const ScreenPreviewRspHeader*>(data.data());
|
||||
if (hdr->status == SCREEN_PREVIEW_BAD_TARGET) {
|
||||
if (mode == SCREEN_CAPTURE_WINDOW)
|
||||
return BuildError(id, -32009, "Window hwnd=" + std::to_string(hwnd) +
|
||||
" is not capturable (not found, minimized, off-screen, "
|
||||
"or larger than 8192 px per side)");
|
||||
if (mode == SCREEN_CAPTURE_MONITOR)
|
||||
return BuildError(id, -32602, "Monitor index " + std::to_string(monitorIdx) +
|
||||
" is out of range");
|
||||
if (mode == SCREEN_CAPTURE_REGION)
|
||||
return BuildError(id, -32602, "Capture region is empty after clipping to the virtual desktop");
|
||||
return BuildError(id, -32000, "Screen is not capturable");
|
||||
}
|
||||
if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG ||
|
||||
hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) {
|
||||
return BuildError(id, -32000,
|
||||
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")");
|
||||
}
|
||||
|
||||
// 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。
|
||||
// 偏移 = 16 + bytes,可能不是 4 字节对齐,按字节拷贝到局部变量再读。
|
||||
bool hasMeta = false;
|
||||
ScreenCaptureMeta meta = {};
|
||||
if (data.size() >= sizeof(ScreenPreviewRspHeader) + hdr->bytes + sizeof(ScreenCaptureMeta)) {
|
||||
memcpy(&meta, data.data() + sizeof(ScreenPreviewRspHeader) + hdr->bytes, sizeof(meta));
|
||||
hasMeta = (meta.magic == SCREEN_CAPTURE_META_MAGIC);
|
||||
}
|
||||
|
||||
std::string b64 = httplib::detail::base64_encode(
|
||||
std::string((const char*)data.data() + sizeof(ScreenPreviewRspHeader), hdr->bytes));
|
||||
|
||||
@@ -3263,9 +3444,21 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
image["height"] = (Json::UInt64)hdr->height;
|
||||
image["bytes"] = (Json::UInt64)hdr->bytes;
|
||||
|
||||
Json::Value capture(Json::objectValue);
|
||||
capture["mode"] = modeStr;
|
||||
if (hasMeta) {
|
||||
capture["x"] = meta.x;
|
||||
capture["y"] = meta.y;
|
||||
capture["width"] = meta.w;
|
||||
capture["height"] = meta.h;
|
||||
capture["scale"] = (int)meta.scale;
|
||||
capture["native"] = meta.scale >= 100;
|
||||
}
|
||||
|
||||
Json::Value result(Json::objectValue);
|
||||
Json::Value structuredContent(Json::objectValue);
|
||||
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
|
||||
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
|
||||
structuredContent["capture"] = capture;
|
||||
result["structuredContent"] = structuredContent;
|
||||
|
||||
Json::Value content(Json::arrayValue);
|
||||
|
||||
Reference in new issue
Block a user