Feature: Add native region, window and monitor capture to get_screenshot
New Windows clients advertise CLIENT_CAP_SCREEN_REGION and accept a 24-byte ScreenPreviewReqEx appended to the existing 8-byte preview request, so get_screenshot can capture a region, a window or a single monitor at 1:1 resolution instead of only the scaled virtual desktop. Successful v2 responses append an optional ScreenCaptureMeta block reporting the source rect and scale, letting callers map coordinates back for remote input. Old clients and old servers keep the v1 byte stream unchanged. The 8192 px per-side bound only applies to native (1:1) capture, where it guards bitmap memory; scaled captures are unaffected so ultra-wide virtual desktops (for example triple 4K) still produce thumbnails and previews. max_width is clamped to [64,1920] as before instead of rejected, matching the documented schema. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
b2c0d81396
commit
235a8a622c
11 files changed
+668
-97
No files matched your search
@@ -1634,6 +1634,13 @@ VOID CKernelManager::OnReceive(PBYTE szBuffer, ULONG ulLength)
|
||||
if (ulLength < sizeof(ScreenPreviewReq)) break;
|
||||
ScreenPreviewReq req;
|
||||
memcpy(&req, szBuffer, sizeof(req));
|
||||
|
||||
// v2 扩展(CLIENT_CAP_SCREEN_REGION):区域/窗口/显示器抓取。老服务端只发 8 字节,
|
||||
// 包长足够才解析,避免把旧包尾巴(压缩层补位)当成扩展字段。
|
||||
const bool hasEx = ulLength >= sizeof(ScreenPreviewReq) + sizeof(ScreenPreviewReqEx);
|
||||
ScreenPreviewReqEx ex = {};
|
||||
if (hasEx) memcpy(&ex, szBuffer + sizeof(ScreenPreviewReq), sizeof(ex));
|
||||
|
||||
// 限流:同一时刻最多 1 个抓屏任务在跑,防御服务端洪泛或异常重发把客户端打爆
|
||||
static std::atomic<int> s_inFlight{0};
|
||||
if (s_inFlight.fetch_add(1) >= 1) {
|
||||
@@ -1641,24 +1648,54 @@ VOID CKernelManager::OnReceive(PBYTE szBuffer, ULONG ulLength)
|
||||
break; // 直接丢弃,让服务端 4s 超时降级为"预览不可用"
|
||||
}
|
||||
// 投递到工作线程,避免阻塞 OnReceive;抓屏 + 编码可能耗几十毫秒
|
||||
std::thread([this, req]() {
|
||||
std::thread([this, req, ex, hasEx]() {
|
||||
struct Guard { ~Guard(){ s_inFlight.fetch_sub(1); } } guard;
|
||||
std::vector<unsigned char> jpg;
|
||||
int w = 0, h = 0;
|
||||
int st = CaptureAndEncodePreview(req.maxWidth, req.jpegQuality, jpg, w, h);
|
||||
CaptureResult cap;
|
||||
int st;
|
||||
if (hasEx) {
|
||||
CaptureTarget target;
|
||||
target.mode = ex.mode;
|
||||
target.native = (ex.flags & SCREEN_CAPTURE_F_NATIVE) != 0;
|
||||
target.maxWidth = req.maxWidth;
|
||||
target.x = ex.x;
|
||||
target.y = ex.y;
|
||||
target.w = ex.w;
|
||||
target.h = ex.h;
|
||||
target.hwnd = ex.hwnd;
|
||||
target.monitor = ex.monitor;
|
||||
st = CaptureAndEncodeScreenshot(target, req.jpegQuality, jpg, cap);
|
||||
} else {
|
||||
st = CaptureAndEncodePreview(req.maxWidth, req.jpegQuality,
|
||||
jpg, cap.outWidth, cap.outHeight);
|
||||
}
|
||||
|
||||
std::vector<BYTE> pkt(sizeof(ScreenPreviewRspHeader) + (st == SCREEN_PREVIEW_OK ? jpg.size() : 0));
|
||||
// v2 成功时在 JPEG 之后追加源矩形元数据(老服务端按 header.bytes 截取,会忽略)
|
||||
const size_t metaBytes = (st == SCREEN_PREVIEW_OK && hasEx) ? sizeof(ScreenCaptureMeta) : 0;
|
||||
const size_t jpgBytes = (st == SCREEN_PREVIEW_OK) ? jpg.size() : 0;
|
||||
std::vector<BYTE> pkt(sizeof(ScreenPreviewRspHeader) + jpgBytes + metaBytes);
|
||||
ScreenPreviewRspHeader* hdr = reinterpret_cast<ScreenPreviewRspHeader*>(pkt.data());
|
||||
memset(hdr, 0, sizeof(*hdr));
|
||||
hdr->token = TOKEN_SCREEN_PREVIEW_RSP;
|
||||
hdr->reqId = req.reqId;
|
||||
hdr->status = (uint8_t)st;
|
||||
hdr->format = SCREEN_PREVIEW_FMT_JPEG;
|
||||
hdr->width = (uint16_t)w;
|
||||
hdr->height = (uint16_t)h;
|
||||
hdr->bytes = (uint32_t)(st == SCREEN_PREVIEW_OK ? jpg.size() : 0);
|
||||
if (st == SCREEN_PREVIEW_OK && !jpg.empty()) {
|
||||
memcpy(pkt.data() + sizeof(*hdr), jpg.data(), jpg.size());
|
||||
hdr->width = (uint16_t)cap.outWidth;
|
||||
hdr->height = (uint16_t)cap.outHeight;
|
||||
hdr->bytes = (uint32_t)jpgBytes;
|
||||
if (jpgBytes) {
|
||||
memcpy(pkt.data() + sizeof(*hdr), jpg.data(), jpgBytes);
|
||||
}
|
||||
if (metaBytes) {
|
||||
ScreenCaptureMeta meta = {};
|
||||
meta.magic = SCREEN_CAPTURE_META_MAGIC;
|
||||
meta.x = cap.x;
|
||||
meta.y = cap.y;
|
||||
meta.w = cap.w;
|
||||
meta.h = cap.h;
|
||||
meta.mode = ex.mode;
|
||||
meta.scale = (uint8_t)cap.scale;
|
||||
memcpy(pkt.data() + sizeof(*hdr) + jpgBytes, &meta, sizeof(meta));
|
||||
}
|
||||
if (m_ClientObject && m_ClientObject->IsConnected()) {
|
||||
m_ClientObject->Send2Server((char*)pkt.data(), (int)pkt.size());
|
||||
|
||||
+167
-58
@@ -46,24 +46,98 @@ int GetJpegEncoderClsid(CLSID& clsid)
|
||||
return -1;
|
||||
}
|
||||
|
||||
// 抓整个虚拟桌面到 24bpp Bitmap,目标尺寸已等比换算。
|
||||
// 虚拟桌面原点不一定是 (0,0):主屏左侧/上方存在显示器时 SM_X/YVIRTUALSCREEN 为负,
|
||||
// 必须作为 StretchBlt 的源点,否则抓到的只是主屏那一块(与注入坐标系不一致)。
|
||||
bool GetVirtualDesktopRect(RECT& rc)
|
||||
{
|
||||
rc.left = GetSystemMetrics(SM_XVIRTUALSCREEN);
|
||||
rc.top = GetSystemMetrics(SM_YVIRTUALSCREEN);
|
||||
rc.right = rc.left + GetSystemMetrics(SM_CXVIRTUALSCREEN);
|
||||
rc.bottom = rc.top + GetSystemMetrics(SM_CYVIRTUALSCREEN);
|
||||
return rc.right > rc.left && rc.bottom > rc.top;
|
||||
}
|
||||
|
||||
BOOL CALLBACK CollectMonitorProc(HMONITOR, HDC, LPRECT rc, LPARAM lp)
|
||||
{
|
||||
reinterpret_cast<std::vector<RECT>*>(lp)->push_back(*rc);
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
// 把请求目标解析成虚拟桌面坐标系里的源矩形。失败返回 false
|
||||
// (区域裁剪为空 / 窗口不存在或已最小化 / 显示器序号越界)。
|
||||
bool ResolveTargetRect(const CaptureTarget& t, RECT& src)
|
||||
{
|
||||
switch (t.mode) {
|
||||
case SCREEN_CAPTURE_VDESKTOP:
|
||||
return GetVirtualDesktopRect(src);
|
||||
|
||||
case SCREEN_CAPTURE_REGION: {
|
||||
if (t.w <= 0 || t.h <= 0) return false;
|
||||
RECT vd = {};
|
||||
if (!GetVirtualDesktopRect(vd)) return false;
|
||||
RECT want = { t.x, t.y, t.x + t.w, t.y + t.h };
|
||||
RECT clipped = {};
|
||||
if (!IntersectRect(&clipped, &want, &vd)) return false;
|
||||
src = clipped;
|
||||
return true;
|
||||
}
|
||||
|
||||
case SCREEN_CAPTURE_WINDOW: {
|
||||
HWND h = reinterpret_cast<HWND>((uintptr_t)t.hwnd);
|
||||
// 最小化窗口没有可抓的像素(GetWindowRect 给的是还原位置),直接报目标无效,
|
||||
// 避免返回一张桌面背景图让调用方误判。
|
||||
if (!h || !::IsWindow(h) || ::IsIconic(h)) return false;
|
||||
RECT wr = {};
|
||||
if (!::GetWindowRect(h, &wr)) return false;
|
||||
RECT vd = {};
|
||||
if (GetVirtualDesktopRect(vd)) {
|
||||
RECT clipped = {};
|
||||
if (!IntersectRect(&clipped, &wr, &vd)) return false;
|
||||
wr = clipped;
|
||||
}
|
||||
if (wr.right <= wr.left || wr.bottom <= wr.top) return false;
|
||||
src = wr;
|
||||
return true;
|
||||
}
|
||||
|
||||
case SCREEN_CAPTURE_MONITOR: {
|
||||
std::vector<RECT> mons;
|
||||
::EnumDisplayMonitors(NULL, NULL, CollectMonitorProc, (LPARAM)&mons);
|
||||
if (mons.empty()) return false;
|
||||
|
||||
// 0 = 主显示器;1..N 按 EnumDisplayMonitors 顺序跳过主显示器编号。
|
||||
POINT origin = { 0, 0 };
|
||||
HMONITOR ph = ::MonitorFromPoint(origin, MONITOR_DEFAULTTOPRIMARY);
|
||||
MONITORINFO mi = { sizeof(mi) };
|
||||
if (!ph || !::GetMonitorInfo(ph, &mi)) return false;
|
||||
const RECT primary = mi.rcMonitor;
|
||||
if (t.monitor == 0) {
|
||||
src = primary;
|
||||
return true;
|
||||
}
|
||||
int idx = 1;
|
||||
for (const RECT& r : mons) {
|
||||
if (::EqualRect(&r, &primary)) continue;
|
||||
if (idx++ == t.monitor) {
|
||||
src = r;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 抓源矩形到 24bpp Bitmap,目标尺寸已换算。
|
||||
// 虚拟桌面原点不一定是 (0,0):主屏左侧/上方存在显示器时为负,直接作为 StretchBlt
|
||||
// 的源点,否则抓到的只是主屏那一块(与注入坐标系不一致)。
|
||||
// 返回新分配的 Bitmap*,失败返回 nullptr。调用者负责 delete。
|
||||
Bitmap* GrabDesktopScaled(int targetW, int targetH)
|
||||
Bitmap* GrabScreenScaled(const RECT& src, int targetW, int targetH)
|
||||
{
|
||||
HDC hScreen = GetDC(NULL);
|
||||
if (!hScreen) return nullptr;
|
||||
|
||||
int srcX = GetSystemMetrics(SM_XVIRTUALSCREEN);
|
||||
int srcY = GetSystemMetrics(SM_YVIRTUALSCREEN);
|
||||
int srcW = GetSystemMetrics(SM_CXVIRTUALSCREEN);
|
||||
int srcH = GetSystemMetrics(SM_CYVIRTUALSCREEN);
|
||||
if (srcW <= 0 || srcH <= 0) {
|
||||
ReleaseDC(NULL, hScreen);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
HDC hMem = CreateCompatibleDC(hScreen);
|
||||
HBITMAP hBmp = CreateCompatibleBitmap(hScreen, targetW, targetH);
|
||||
if (!hMem || !hBmp) {
|
||||
@@ -75,11 +149,13 @@ Bitmap* GrabDesktopScaled(int targetW, int targetH)
|
||||
|
||||
HGDIOBJ oldBmp = SelectObject(hMem, hBmp);
|
||||
|
||||
// 高质量缩放:HALFTONE 内插
|
||||
// 高质量缩放:HALFTONE 内插;1:1 时无影响
|
||||
SetStretchBltMode(hMem, HALFTONE);
|
||||
SetBrushOrgEx(hMem, 0, 0, NULL);
|
||||
BOOL bb = StretchBlt(hMem, 0, 0, targetW, targetH,
|
||||
hScreen, srcX, srcY, srcW, srcH, SRCCOPY | CAPTUREBLT);
|
||||
hScreen, src.left, src.top,
|
||||
src.right - src.left, src.bottom - src.top,
|
||||
SRCCOPY | CAPTUREBLT);
|
||||
|
||||
SelectObject(hMem, oldBmp);
|
||||
|
||||
@@ -102,43 +178,10 @@ Bitmap* GrabDesktopScaled(int targetW, int targetH)
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int CaptureAndEncodePreview(int maxWidth, int quality,
|
||||
std::vector<unsigned char>& out,
|
||||
int& outWidth, int& outHeight)
|
||||
int EncodeJpeg(Bitmap* bmp, int quality, std::vector<unsigned char>& out)
|
||||
{
|
||||
out.clear();
|
||||
outWidth = outHeight = 0;
|
||||
|
||||
if (!g_boot.ok) return SCREEN_PREVIEW_NOT_SUPPORTED;
|
||||
if (maxWidth < 64) maxWidth = 64;
|
||||
if (maxWidth > 1920) maxWidth = 1920;
|
||||
if (quality < 1) quality = 1;
|
||||
if (quality > 100) quality = 100;
|
||||
|
||||
int srcW = GetSystemMetrics(SM_CXVIRTUALSCREEN);
|
||||
int srcH = GetSystemMetrics(SM_CYVIRTUALSCREEN);
|
||||
if (srcW <= 0 || srcH <= 0) return SCREEN_PREVIEW_CAPTURE_FAILED;
|
||||
|
||||
// 等比缩放,禁止放大
|
||||
int targetW = (srcW <= maxWidth) ? srcW : maxWidth;
|
||||
int targetH = (int)((double)srcH * targetW / srcW + 0.5);
|
||||
if (targetH <= 0) targetH = 1;
|
||||
// 偶数对齐,JPEG 编码更高效
|
||||
targetW &= ~1;
|
||||
targetH &= ~1;
|
||||
if (targetW < 2) targetW = 2;
|
||||
if (targetH < 2) targetH = 2;
|
||||
|
||||
Bitmap* bmp = GrabDesktopScaled(targetW, targetH);
|
||||
if (!bmp) return SCREEN_PREVIEW_CAPTURE_FAILED;
|
||||
|
||||
CLSID clsid;
|
||||
if (GetJpegEncoderClsid(clsid) != 0) {
|
||||
delete bmp;
|
||||
return SCREEN_PREVIEW_ENCODE_FAILED;
|
||||
}
|
||||
if (GetJpegEncoderClsid(clsid) != 0) return SCREEN_PREVIEW_ENCODE_FAILED;
|
||||
|
||||
EncoderParameters params;
|
||||
params.Count = 1;
|
||||
@@ -149,14 +192,10 @@ int CaptureAndEncodePreview(int maxWidth, int quality,
|
||||
params.Parameter[0].Value = &q;
|
||||
|
||||
IStream* stream = nullptr;
|
||||
if (FAILED(CreateStreamOnHGlobal(NULL, TRUE, &stream))) {
|
||||
delete bmp;
|
||||
if (FAILED(CreateStreamOnHGlobal(NULL, TRUE, &stream)))
|
||||
return SCREEN_PREVIEW_ENCODE_FAILED;
|
||||
}
|
||||
|
||||
Status st = bmp->Save(stream, &clsid, ¶ms);
|
||||
delete bmp;
|
||||
|
||||
if (st != Ok) {
|
||||
stream->Release();
|
||||
return SCREEN_PREVIEW_ENCODE_FAILED;
|
||||
@@ -181,8 +220,78 @@ int CaptureAndEncodePreview(int maxWidth, int quality,
|
||||
out.assign((unsigned char*)p, (unsigned char*)p + sz);
|
||||
GlobalUnlock(hMem);
|
||||
stream->Release();
|
||||
|
||||
outWidth = targetW;
|
||||
outHeight = targetH;
|
||||
return SCREEN_PREVIEW_OK;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int CaptureAndEncodeScreenshot(const CaptureTarget& target, int quality,
|
||||
std::vector<unsigned char>& out, CaptureResult& result)
|
||||
{
|
||||
out.clear();
|
||||
result = CaptureResult();
|
||||
|
||||
if (!g_boot.ok) return SCREEN_PREVIEW_NOT_SUPPORTED;
|
||||
if (quality < 1) quality = 1;
|
||||
if (quality > 100) quality = 100;
|
||||
|
||||
RECT src = {};
|
||||
if (!ResolveTargetRect(target, src)) return SCREEN_PREVIEW_BAD_TARGET;
|
||||
const int srcW = src.right - src.left;
|
||||
const int srcH = src.bottom - src.top;
|
||||
if (srcW <= 0 || srcH <= 0) return SCREEN_PREVIEW_BAD_TARGET;
|
||||
|
||||
int targetW = 0, targetH = 0;
|
||||
if (target.native) {
|
||||
// 单边 8192 足够覆盖 8K 显示器,同时拦住异常区域把内存打爆。
|
||||
// 只约束 1:1 抓取;缩放路径目标宽 ≤1920,再大的虚拟桌面(如三屏 4K)也安全。
|
||||
if (srcW > 8192 || srcH > 8192) return SCREEN_PREVIEW_BAD_TARGET;
|
||||
targetW = srcW;
|
||||
targetH = srcH;
|
||||
} else {
|
||||
int maxWidth = target.maxWidth;
|
||||
if (maxWidth < 64) maxWidth = 64;
|
||||
if (maxWidth > 1920) maxWidth = 1920;
|
||||
// 等比缩放,禁止放大
|
||||
targetW = (srcW <= maxWidth) ? srcW : maxWidth;
|
||||
targetH = (int)((double)srcH * targetW / srcW + 0.5);
|
||||
if (targetH <= 0) targetH = 1;
|
||||
// 偶数对齐,JPEG 编码更高效
|
||||
targetW &= ~1;
|
||||
targetH &= ~1;
|
||||
if (targetW < 2) targetW = 2;
|
||||
if (targetH < 2) targetH = 2;
|
||||
}
|
||||
|
||||
Bitmap* bmp = GrabScreenScaled(src, targetW, targetH);
|
||||
if (!bmp) return SCREEN_PREVIEW_CAPTURE_FAILED;
|
||||
|
||||
int st = EncodeJpeg(bmp, quality, out);
|
||||
delete bmp;
|
||||
if (st != SCREEN_PREVIEW_OK) return st;
|
||||
|
||||
result.x = src.left;
|
||||
result.y = src.top;
|
||||
result.w = srcW;
|
||||
result.h = srcH;
|
||||
result.scale = (int)((long long)targetW * 100 / srcW);
|
||||
result.outWidth = targetW;
|
||||
result.outHeight = targetH;
|
||||
return SCREEN_PREVIEW_OK;
|
||||
}
|
||||
|
||||
int CaptureAndEncodePreview(int maxWidth, int quality,
|
||||
std::vector<unsigned char>& out,
|
||||
int& outWidth, int& outHeight)
|
||||
{
|
||||
CaptureTarget target;
|
||||
target.mode = SCREEN_CAPTURE_VDESKTOP;
|
||||
target.native = false;
|
||||
target.maxWidth = maxWidth;
|
||||
|
||||
CaptureResult result;
|
||||
int st = CaptureAndEncodeScreenshot(target, quality, out, result);
|
||||
outWidth = result.outWidth;
|
||||
outHeight = result.outHeight;
|
||||
return st;
|
||||
}
|
||||
+26
-8
@@ -1,16 +1,34 @@
|
||||
// ScreenPreview.h
|
||||
// 屏幕预览:抓整个虚拟桌面 → 等比缩放 → JPEG 编码,供 COMMAND_SCREEN_PREVIEW_REQ 响应使用。
|
||||
// 屏幕截图:按目标(虚拟桌面 / 区域 / 窗口 / 显示器)抓取 → 等比缩放 → JPEG 编码,
|
||||
// 供 COMMAND_SCREEN_PREVIEW_REQ 响应使用。
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
// 抓取整个虚拟桌面并编码成 JPEG。
|
||||
// maxWidth 服务端期望的宽度;客户端按源图宽度等比缩放,不会强制放大
|
||||
// quality JPEG 质量 1..100(建议 40..85)
|
||||
// out 编码后的 JPEG 字节流
|
||||
// outWidth 实际编码图宽
|
||||
// outHeight 实际编码图高
|
||||
// 返回 0 表示成功;非 0 见 ScreenPreviewStatus(枚举在 commands.h)
|
||||
// 抓取目标(ScreenCaptureMode 见 common/commands.h)
|
||||
struct CaptureTarget {
|
||||
int mode = 0; // 0=虚拟桌面 1=区域 2=窗口 3=显示器
|
||||
bool native = false; // true = 按源尺寸 1:1 抓取,忽略 maxWidth
|
||||
int maxWidth = 0; // native=false 时的目标宽度上限
|
||||
int x = 0, y = 0; // mode=区域:左上角(虚拟桌面坐标,可为负)
|
||||
int w = 0, h = 0; // mode=区域:宽高(>0)
|
||||
unsigned long hwnd = 0; // mode=窗口
|
||||
int monitor = 0; // mode=显示器:0 = 主显示器
|
||||
};
|
||||
|
||||
// 实际抓取结果;源矩形为裁剪到虚拟桌面之后的坐标,scale 为缩放百分比(100 = 原生)
|
||||
struct CaptureResult {
|
||||
int x = 0, y = 0;
|
||||
int w = 0, h = 0;
|
||||
int scale = 100;
|
||||
int outWidth = 0, outHeight = 0;
|
||||
};
|
||||
|
||||
// 抓取并按需缩放、编码成 JPEG。返回 0 表示成功;非 0 见 ScreenPreviewStatus。
|
||||
int CaptureAndEncodeScreenshot(const CaptureTarget& target, int quality,
|
||||
std::vector<unsigned char>& out, CaptureResult& result);
|
||||
|
||||
// 整虚拟桌面 + 等比缩放到 maxWidth(v1 请求的旧入口)
|
||||
int CaptureAndEncodePreview(int maxWidth, int quality,
|
||||
std::vector<unsigned char>& out,
|
||||
int& outWidth, int& outHeight);
|
||||
+46
-1
@@ -142,6 +142,7 @@ inline int isValid_10s()
|
||||
#define CLIENT_CAP_EMBEDDED_ANCHOR 0x0040 // 当前靠「生成时内嵌的 TLS 信任锚」在加密(按运行时状态置位,不是能力声明)
|
||||
#define CLIENT_CAP_VIRTUAL_DESKTOP 0x0080 // 支持按 COMMAND_SCREEN_SPY 参数 buffer[11](wire 索引 12)强制抓取整个虚拟桌面(多显示器远程控制)
|
||||
#define CLIENT_CAP_WINDOW_INFO 0x0100 // 支持 COMMAND_MCP_WINDOW_INFO(带矩形/前后台/最小化状态的窗口列表)与窗口激活(CMD_WINDOW_ACTIVATE_FLAG)
|
||||
#define CLIENT_CAP_SCREEN_REGION 0x0200 // 支持 COMMAND_SCREEN_PREVIEW_REQ 扩展(区域/窗口/显示器原生分辨率截图)
|
||||
|
||||
#define TALK_DLG_MAXLEN 1024 // 最大输入字符长度
|
||||
|
||||
@@ -524,6 +525,7 @@ enum ScreenPreviewStatus {
|
||||
SCREEN_PREVIEW_CAPTURE_FAILED = 1, // 抓屏失败
|
||||
SCREEN_PREVIEW_ENCODE_FAILED = 2, // 编码失败
|
||||
SCREEN_PREVIEW_NOT_SUPPORTED = 3, // 平台不支持
|
||||
SCREEN_PREVIEW_BAD_TARGET = 4, // 目标无效:区域裁剪后为空/窗口不存在或已最小化/显示器序号越界
|
||||
};
|
||||
|
||||
enum ScreenPreviewFormat {
|
||||
@@ -544,10 +546,51 @@ struct ScreenPreviewRspHeader {
|
||||
uint16_t height; // 实际编码图高
|
||||
uint32_t bytes; // 图像字节数(紧随其后)
|
||||
uint8_t reserved[3];
|
||||
// 后接 data[bytes]
|
||||
// 后接 data[bytes],可选再跟一个 ScreenCaptureMeta
|
||||
}; // 头部 16 字节
|
||||
#pragma pack(pop)
|
||||
|
||||
// 截图目标模式(CLIENT_CAP_SCREEN_REGION)。v1 请求没有 mode 字段,等价于 VDESKTOP。
|
||||
enum ScreenCaptureMode {
|
||||
SCREEN_CAPTURE_VDESKTOP = 0, // 整个虚拟桌面(v1 行为,可缩放)
|
||||
SCREEN_CAPTURE_REGION = 1, // 虚拟桌面坐标系里的矩形
|
||||
SCREEN_CAPTURE_WINDOW = 2, // 按 hwnd 取窗口矩形(最小化窗口无法抓取)
|
||||
SCREEN_CAPTURE_MONITOR = 3, // 显示器序号(0 = 主显示器,其余按 EnumDisplayMonitors 顺序)
|
||||
};
|
||||
|
||||
// v2 请求扩展:紧跟在 ScreenPreviewReq(前 8 字节保持不变)之后。
|
||||
// 新服务端只对声明 CLIENT_CAP_SCREEN_REGION 的客户端发 v2;老客户端 memcpy 前 8 字节后
|
||||
// 忽略追加部分,老服务端永远只发 8 字节,新客户端按包长判断是 v1 还是 v2。
|
||||
#pragma pack(push, 1)
|
||||
struct ScreenPreviewReqEx {
|
||||
uint8_t mode; // ScreenCaptureMode
|
||||
uint8_t flags; // SCREEN_CAPTURE_F_*
|
||||
uint16_t monitor; // mode=MONITOR 时的显示器序号
|
||||
int32_t x, y; // mode=REGION 时的左上角(虚拟桌面坐标,可为负)
|
||||
int32_t w, h; // mode=REGION 时的宽高(>0)
|
||||
uint32_t hwnd; // mode=WINDOW 时的窗口句柄
|
||||
}; // 24 字节
|
||||
#pragma pack(pop)
|
||||
|
||||
#define SCREEN_CAPTURE_F_NATIVE 0x01 // 按源矩形 1:1 抓取,忽略 maxWidth(默认只在 v2 模式下生效)
|
||||
|
||||
// v2 响应可选尾块:紧跟在 JPEG 数据之后,声明实际抓取的源矩形,供调用方换算坐标。
|
||||
// 老服务端只按 header.bytes 取 JPEG,会忽略这段尾块,故可向后兼容。
|
||||
#define SCREEN_CAPTURE_META_MAGIC 0x324D4353u // 'SCM2'
|
||||
#pragma pack(push, 1)
|
||||
struct ScreenCaptureMeta {
|
||||
uint32_t magic; // SCREEN_CAPTURE_META_MAGIC
|
||||
int32_t x, y; // 实际抓取的源矩形左上角(虚拟桌面坐标)
|
||||
int32_t w, h; // 源矩形宽高(缩放前)
|
||||
uint8_t mode; // ScreenCaptureMode
|
||||
uint8_t scale; // 缩放百分比(100 = 原生)
|
||||
uint16_t reserved;
|
||||
}; // 24 字节
|
||||
#pragma pack(pop)
|
||||
|
||||
static_assert(sizeof(ScreenPreviewReqEx) == 24, "ScreenPreviewReqEx must be 24 bytes");
|
||||
static_assert(sizeof(ScreenCaptureMeta) == 24, "ScreenCaptureMeta must be 24 bytes");
|
||||
|
||||
static_assert(sizeof(ScreenPreviewReq) == 8, "ScreenPreviewReq must be 8 bytes");
|
||||
static_assert(sizeof(ScreenPreviewRspHeader) == 16, "ScreenPreviewRspHeader must be 16 bytes");
|
||||
|
||||
@@ -1231,6 +1274,8 @@ typedef struct LOGIN_INFOR {
|
||||
caps |= CLIENT_CAP_VIRTUAL_DESKTOP;
|
||||
// 窗口信息 RPC(矩形/z 序/前后台)与窗口激活。
|
||||
caps |= CLIENT_CAP_WINDOW_INFO;
|
||||
// 区域/窗口/显示器原生分辨率截图(COMMAND_SCREEN_PREVIEW_REQ 的 v2 扩展)。
|
||||
caps |= CLIENT_CAP_SCREEN_REGION;
|
||||
#elif defined(__linux__) && !defined(__ANDROID__)
|
||||
// Linux 客户端作为 C2C 目标端时会回执落盘结果(common/FileTransferV2.h)。
|
||||
// macOS 暂未接线(无全局主连接句柄),不声明——服务端就不会为它空等回执。
|
||||
|
||||
@@ -1063,3 +1063,75 @@ GUI 自动化的失败大多不是「事件没发出去」,而是「事件进
|
||||
列表不完整;`expect_hwnd` 查不到时 `-32009` 的文案区分「窗口可能还在、只是没返回」
|
||||
与「隐藏/已关闭」;`activate` 复核时目标不在返回的这一批里则记 `state=unknown`、
|
||||
`reason=list_truncated`,而不是 `missing`。任何情况下都不要据截断的列表断定窗口已销毁。
|
||||
|
||||
|
||||
## 21. 增补(2026-11-10:区域/窗口/显示器原生分辨率截图)
|
||||
|
||||
### 21.1 问题
|
||||
|
||||
旧的 `get_screenshot` 只有「整个虚拟桌面 + `max_width` 缩放」一种形态。GUI 自动化真正
|
||||
需要的往往是某个窗口或某块 ROI 的原始像素:整屏缩到 1024 后聊天正文、按钮小字都糊了,
|
||||
OCR 与视觉定位同时退化;把 `max_width` 抬到 1920 对 4K 屏仍不够,而且传的依旧是全屏。
|
||||
|
||||
### 21.2 协议
|
||||
|
||||
新增能力位 `CLIENT_CAP_SCREEN_REGION=0x0200`(Windows 客户端 `DefaultCaps()` 置位;
|
||||
Linux 客户端只有 v1 处理分支,不声明)。请求在原有 8 字节 `ScreenPreviewReq` 之后追加
|
||||
24 字节 `ScreenPreviewReqEx`(共 32 字节):
|
||||
|
||||
```
|
||||
u8 mode ScreenCaptureMode: VDESKTOP=0 / REGION=1 / WINDOW=2 / MONITOR=3
|
||||
u8 flags SCREEN_CAPTURE_F_NATIVE=0x01(按源矩形 1:1,忽略 maxWidth)
|
||||
u16 monitor mode=MONITOR 的序号
|
||||
i32 x, y, w, h mode=REGION 的矩形(虚拟桌面坐标)
|
||||
u32 hwnd mode=WINDOW 的窗口句柄
|
||||
```
|
||||
|
||||
响应仍是 `ScreenPreviewRspHeader`(16) + JPEG;v2 成功时客户端在 JPEG 之后追加 24 字节
|
||||
`ScreenCaptureMeta`(magic `'SCM2'`=0x324D4353,`x/y/w/h` + `scale` + `mode`),如实回报
|
||||
实际抓取的源矩形与缩放百分比。老服务端只按 `header.bytes` 取图,会忽略尾块;老客户端
|
||||
忽略请求尾巴、按 v1 抓整屏。服务端只对声明能力位的主机发 v2,两个方向都不需要版本协商。
|
||||
|
||||
### 21.3 抓取语义
|
||||
|
||||
| mode | 参数 | 源矩形 | 默认尺寸 |
|
||||
|---|---|---|---|
|
||||
| `screen` | — | 虚拟桌面 `SM_*VIRTUALSCREEN` | 沿用 RTT 自适应(≤1024),可传 `max_width` |
|
||||
| `region` | `x/y/width/height` | 与虚拟桌面求交后裁剪(`x/y` 可为负) | 1:1 原生分辨率 |
|
||||
| `window` | `hwnd` | `GetWindowRect` 后与虚拟桌面求交;`IsWindow` 且非 `IsIconic` | 1:1 原生分辨率 |
|
||||
| `monitor` | `monitor` | 0=主显示器(`MonitorFromPoint`);1..N 按 `EnumDisplayMonitors` 顺序跳过主屏 | 1:1 原生分辨率 |
|
||||
|
||||
- `region/window/monitor` 只有显式传 `max_width`(钳制到 64..1920) 才等比缩小;缺省即原生分辨率,
|
||||
1:1 抓取单边硬上限 8192 像素(覆盖 8K 显示器,同时拦住异常区域打爆内存)。缩放路径不受此限,
|
||||
整屏同样适用——三屏 4K 这类超宽虚拟桌面仍能正常抓缩略图。
|
||||
- 需要缩放时目标宽高做偶数对齐(JPEG 更省),1:1 时不改尺寸。
|
||||
- 抓屏与注入共用虚拟桌面坐标系(多显示器时原点可为负),`structuredContent.capture`
|
||||
回报的就是实际源矩形,可直接用于换算点击坐标。
|
||||
- 窗口抓取是 GDI 抓屏语义:被其他窗口遮挡的部分截到的是遮挡物,不是 `PrintWindow`
|
||||
式的重绘;需要无遮挡内容时先 `activate` 目标窗口。矩形含非客户区。
|
||||
|
||||
### 21.4 工具参数与返回
|
||||
|
||||
`get_screenshot` 的 `inputSchema` 新增 `mode`(枚举 `screen|region|window|monitor`,默认
|
||||
`screen`)与 `x/y/width/height/hwnd/monitor`;`max_width` 的描述按模式分叉。`hwnd` 直接
|
||||
取 `host_list_windows` / `get_foreground_window` 返回的句柄,同时接受整数与 `0x` 字符串。
|
||||
返回值在 `image` 之外新增 `structuredContent.capture = {mode,x,y,width,height,scale,native}`
|
||||
(老客户端不回尾块时只有 `mode`)。`screen` 模式对新客户端也走 v2,因此同样能拿到源矩形。
|
||||
|
||||
### 21.5 错误
|
||||
|
||||
| 场景 | code | 说明 |
|
||||
|---|---|---|
|
||||
| 客户端无 `CLIENT_CAP_SCREEN_REGION` 却要 region/window/monitor | `-32005` | 换 `mode=screen` 或升级客户端,改别的参数修不好 |
|
||||
| `window` 的 hwnd 不存在、已最小化、完全在虚拟桌面外或单边超过 8192 px | `-32009` | 先 `activate` / `restore`,或重新取 hwnd;超大窗口改用 `region` 分段截取 |
|
||||
| `monitor` 序号越界、`region` 裁剪后为空 | `-32602` | 参数问题 |
|
||||
| 抓屏/编码失败 | `-32000` | 主机侧失败(如无桌面会话) |
|
||||
| 超时 / 设备忙 | `-32001` / `-32003` | 同其他主连接工具 |
|
||||
|
||||
### 21.6 兼容与代价
|
||||
|
||||
- 全部挂在能力位后面:老客户端、老服务端的字节流逐字节不变;Linux 仍走 v1 全屏。
|
||||
- 客户端对抓屏请求保持单飞(同一时刻 1 个),并投递到工作线程编码,不阻塞接收线程;
|
||||
并发请求直接丢弃,表现为服务端超时。
|
||||
- 原生分辨率意味着更大的 JPEG 与 base64 体积:区域越大、越多窗口细节,单次响应越贵。
|
||||
只截需要的 ROI,比整屏 1920 更省也更快。
|
||||
@@ -280,9 +280,11 @@ MCP 现在能读、能传、能搜,**唯独不能删、不能改名、不能
|
||||
|
||||
### 4.2 窗口级截图
|
||||
|
||||
`COMMAND_SCREEN_WINDOW`(`client/ScreenManager.cpp:894`)接的是**屏幕流**(按窗口标题捕获,空串恢复全屏),不是一次性 JPEG。要做 `get_screenshot(window_title=...)` 需要服务端在屏幕子连接上收若干帧再截一张,本质上是「起一个临时屏幕会话」——比现有 `get_screenshot`(主连接一次 RPC)重得多。
|
||||
|
||||
**建议**:先做「AI 侧的替代方案」——`remote_open` 已能拿全屏,`list_windows` 能给窗口句柄与标题,模型可以据此用 `remote_mouse` 点击。等 `remote_*` 在真实任务里被证明「全屏截图不够用」再投入。
|
||||
**已完成**(2026-11-10,`CLIENT_CAP_SCREEN_REGION`):`get_screenshot` 新增 `mode=window`
|
||||
(按 hwnd 原生分辨率抓取)以及 `mode=region` / `mode=monitor`,走主连接 v2 RPC + 客户端
|
||||
`GetWindowRect` / `StretchBlt`,一次性返回 JPEG 并附源矩形元数据,不需要为单张截图起
|
||||
屏幕流子会话。`COMMAND_SCREEN_WINDOW` 仍适用于需要**持续**窗口视频流的场景,与一次性
|
||||
JPEG 不冲突;详见 `Mcp_Design.md §21`。
|
||||
|
||||
### 4.3 代理隧道(价值最高的「新维度」)
|
||||
|
||||
@@ -509,7 +511,7 @@ P10-C(`3d388f8`)加工具时索性去掉数字——避免每加一个工具
|
||||
| ~~`list_windows` 无 `pid` 过滤~~ | — | **已完成**(P10-A,`bb55f52`):`list_processes` / `list_windows` 都加了 `name` + `pid` |
|
||||
| `search_file` 无深度/大小上限参数 | `McpServer.cpp:8050` | 客户端固定深度 32 / 10000 条,服务端可加 `max_depth` 但需客户端支持;暂不做(P12) |
|
||||
| ~~`search_file` 的描述仍写着「会栈溢出」~~ | — | **已完成**(P10-A,`bb55f52`):描述改为「超过 260 字节的路径会被静默跳过(既不记录也不继续向下递归)」,`tools/mcp_smoke.ps1` 的两处注释改为「MAX_PATH 的条目会被客户端静默跳过(8f718d9 起不再栈溢出)」 |
|
||||
| `get_screenshot` 无区域裁剪 | `McpServer.cpp:2085` | 有 `max_width`;ROI 需客户端改动,收益有限,不做 |
|
||||
| ~~`get_screenshot` 无区域裁剪~~ | — | **已完成**(2026-11-10):新增 `mode=region/window/monitor`,默认原生分辨率并返回 `capture` 源矩形元数据;见 `Mcp_Design.md §21` |
|
||||
| 审计不含调用参数 | `Mcp_Design.md §10.8` | 有意设计(防泄漏),维持 |
|
||||
| `chat_wait` 会取走 inbox | `McpInternal.cpp:431` | 已正确标注非幂等,无需改 |
|
||||
|
||||
|
||||
+69
-2
@@ -463,6 +463,10 @@
|
||||
"description": "能否截屏(get_screenshot 的前提)",
|
||||
"type": "boolean"
|
||||
},
|
||||
"screenRegion": {
|
||||
"description": "能否按区域/窗口/显示器原生分辨率截屏(get_screenshot 的 mode=region/window/monitor 前提);仅声明 CLIENT_CAP_SCREEN_REGION 的新 Windows 客户端为 true",
|
||||
"type": "boolean"
|
||||
},
|
||||
"windowInfo": {
|
||||
"description": "是否支持窗口信息(get_foreground_window、host_list_windows 的坐标/前后台字段、operate_window/host_control_window 的 activate);仅声明 CLIENT_CAP_WINDOW_INFO 的新客户端为 true",
|
||||
"type": "boolean"
|
||||
@@ -1044,7 +1048,7 @@
|
||||
},
|
||||
{
|
||||
"name": "get_screenshot",
|
||||
"description": "截取指定在线主机的一帧屏幕,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。",
|
||||
"description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。",
|
||||
"gate": null,
|
||||
"platforms": [
|
||||
"win",
|
||||
@@ -1053,12 +1057,49 @@
|
||||
"impl": "mfc",
|
||||
"inputSchema": {
|
||||
"properties": {
|
||||
"height": {
|
||||
"description": "region 模式必填:矩形高度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。",
|
||||
"type": "integer"
|
||||
},
|
||||
"hwnd": {
|
||||
"description": "window 模式必填:窗口句柄(整数或 0x 前缀字符串),取 host_list_windows / get_foreground_window 返回的 hwnd。窗口最小化时无法抓取(返回错误),先 activate/restore。",
|
||||
"type": [
|
||||
"integer",
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"description": "主机 id,取 list_online_hosts / search_hosts 返回的 id 字段",
|
||||
"type": "string"
|
||||
},
|
||||
"max_width": {
|
||||
"description": "期望图片最大宽度(像素,钳制到 64~1920)。省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);传 1920 可拿到接近原分辨率(1080p 源屏即原分辨率,4K 源屏最多 1920),供 AI 视觉/OCR 场景提升清晰度。",
|
||||
"description": "期望图片最大宽度(像素,钳制到 64~1920)。screen 模式省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);region/window/monitor 模式省略/传 0 时按源尺寸 1:1 抓取(原生分辨率),显式传入才等比缩小。",
|
||||
"type": "integer"
|
||||
},
|
||||
"mode": {
|
||||
"description": "抓取范围,默认 screen。screen=整个虚拟桌面(可缩放);region=指定矩形;window=指定窗口;monitor=指定显示器。region/window/monitor 默认按源尺寸 1:1 抓取(原生分辨率,适合 OCR/视觉读字),只有显式传 max_width 才缩放;这三种需要较新的 Windows 客户端(能力位 CLIENT_CAP_SCREEN_REGION),否则返回 -32005。",
|
||||
"enum": [
|
||||
"screen",
|
||||
"region",
|
||||
"window",
|
||||
"monitor"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"monitor": {
|
||||
"description": "monitor 模式:显示器序号,0=主显示器(默认),1..N 为其余显示器(按系统枚举顺序)。序号越界返回错误。",
|
||||
"type": "integer"
|
||||
},
|
||||
"width": {
|
||||
"description": "region 模式必填:矩形宽度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。",
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
"description": "region 模式:矩形左上角的虚拟桌面 X 坐标(可为负;多显示器时以虚拟桌面左上角为原点)。省略按 0 处理。",
|
||||
"type": "integer"
|
||||
},
|
||||
"y": {
|
||||
"description": "region 模式:矩形左上角的虚拟桌面 Y 坐标(可为负)。省略按 0 处理。",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
@@ -1069,6 +1110,32 @@
|
||||
},
|
||||
"outputSchema": {
|
||||
"properties": {
|
||||
"capture": {
|
||||
"properties": {
|
||||
"height": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mode": {
|
||||
"type": "string"
|
||||
},
|
||||
"native": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"scale": {
|
||||
"type": "integer"
|
||||
},
|
||||
"width": {
|
||||
"type": "integer"
|
||||
},
|
||||
"x": {
|
||||
"type": "integer"
|
||||
},
|
||||
"y": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"image": {
|
||||
"properties": {
|
||||
"bytes": {
|
||||
|
||||
@@ -10403,6 +10403,22 @@ void CMy2015RemoteDlg::SendScreenPreviewRequest(context* ctx, WORD reqId, WORD m
|
||||
ctx->Send2Client((PBYTE)&req, sizeof(req));
|
||||
}
|
||||
|
||||
void CMy2015RemoteDlg::SendScreenPreviewRequestEx(context* ctx, WORD reqId, WORD maxWidth,
|
||||
BYTE jpegQuality, const ScreenPreviewReqEx& ex)
|
||||
{
|
||||
if (!ctx) return;
|
||||
BYTE buf[sizeof(ScreenPreviewReq) + sizeof(ScreenPreviewReqEx)] = {};
|
||||
ScreenPreviewReq req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
req.cmd = COMMAND_SCREEN_PREVIEW_REQ;
|
||||
req.reqId = reqId;
|
||||
req.maxWidth = maxWidth;
|
||||
req.jpegQuality = jpegQuality;
|
||||
memcpy(buf, &req, sizeof(req));
|
||||
memcpy(buf + sizeof(req), &ex, sizeof(ex));
|
||||
ctx->Send2Client(buf, (int)sizeof(buf));
|
||||
}
|
||||
|
||||
// 收到 TOKEN_SCREEN_PREVIEW_RSP(在主线程)
|
||||
// wParam: 未使用
|
||||
// LPARAM: 指向堆分配的 PreviewRspMsg*(含 clientId + 完整响应包:[Header|JPEG]),
|
||||
|
||||
@@ -673,6 +673,10 @@ public:
|
||||
void ChooseScreenPreviewParams(context* ctx, WORD& maxWidth, BYTE& jpegQuality) const;
|
||||
// 发起预览请求;reqId 应与 m_PreviewReqId 同步
|
||||
void SendScreenPreviewRequest(context* ctx, WORD reqId, WORD maxWidth, BYTE jpegQuality);
|
||||
// v2:在 v1 请求后追加 ScreenPreviewReqEx(区域/窗口/显示器 + 原生分辨率标志)。
|
||||
// 只对声明 CLIENT_CAP_SCREEN_REGION 的客户端使用。
|
||||
void SendScreenPreviewRequestEx(context* ctx, WORD reqId, WORD maxWidth, BYTE jpegQuality,
|
||||
const ScreenPreviewReqEx& ex);
|
||||
// 预览响应规范化:Linux 客户端回原始 BGRA(SCREEN_PREVIEW_FMT_BGRX),在 IO 线程入口
|
||||
// 统一转成 JPEG,下游(浮窗/列表缩略图/MCP)只处理 JPEG。返回 false 表示畸形包,应丢弃。
|
||||
static bool NormalizeScreenPreviewPacket(std::vector<BYTE>& pkt);
|
||||
|
||||
+208
-15
@@ -1030,6 +1030,9 @@ Json::Value BuildHostDetailOutputSchema() {
|
||||
platform["description"] = u8"客户端形态:EXE/DLL/MEXE/SC/MDLL(Windows)、LNX(Linux)、MAC、APK";
|
||||
capProps["platform"] = platform;
|
||||
capProps["screenPreview"] = boolProp(u8"能否截屏(get_screenshot 的前提)");
|
||||
capProps["screenRegion"] = boolProp(
|
||||
u8"能否按区域/窗口/显示器原生分辨率截屏(get_screenshot 的 mode=region/window/monitor 前提);"
|
||||
u8"仅声明 CLIENT_CAP_SCREEN_REGION 的新 Windows 客户端为 true");
|
||||
capProps["remoteControl"] = boolProp(
|
||||
u8"能否建立远程控制会话(remote_open/remote_mouse/remote_keyboard 的前提);"
|
||||
u8"还要求服务端 Web 远程服务已启动(WebSvrPort>0)");
|
||||
@@ -1767,6 +1770,24 @@ Json::Value BuildScreenshotOutputSchema() {
|
||||
imageProp["properties"] = imageProps;
|
||||
props["image"] = imageProp;
|
||||
|
||||
Json::Value capProp(Json::objectValue);
|
||||
capProp["type"] = "object";
|
||||
Json::Value capProps(Json::objectValue);
|
||||
Json::Value capMode(Json::objectValue);
|
||||
capMode["type"] = "string";
|
||||
capProps["mode"] = capMode;
|
||||
const char* capInts[] = { "x", "y", "width", "height", "scale" };
|
||||
for (const char* f : capInts) {
|
||||
Json::Value s(Json::objectValue);
|
||||
s["type"] = "integer";
|
||||
capProps[f] = s;
|
||||
}
|
||||
Json::Value capNative(Json::objectValue);
|
||||
capNative["type"] = "boolean";
|
||||
capProps["native"] = capNative;
|
||||
capProp["properties"] = capProps;
|
||||
props["capture"] = capProp;
|
||||
|
||||
Json::Value schema(Json::objectValue);
|
||||
schema["type"] = "object";
|
||||
schema["properties"] = props;
|
||||
@@ -1826,7 +1847,7 @@ Json::Value BuildGetForegroundWindowOutputSchema() {
|
||||
return schema;
|
||||
}
|
||||
|
||||
// get_screenshot 的 inputSchema(id 必填,max_width 可选)
|
||||
// get_screenshot 的 inputSchema(id 必填;mode/region/window/monitor 需新 Windows 客户端)
|
||||
Json::Value BuildGetScreenshotInputSchema() {
|
||||
Json::Value props(Json::objectValue);
|
||||
|
||||
@@ -1835,9 +1856,53 @@ Json::Value BuildGetScreenshotInputSchema() {
|
||||
idProp["description"] = u8"主机 id,取 list_online_hosts / search_hosts 返回的 id 字段";
|
||||
props["id"] = idProp;
|
||||
|
||||
Json::Value modeProp(Json::objectValue);
|
||||
modeProp["type"] = "string";
|
||||
Json::Value modeEnum(Json::arrayValue);
|
||||
modeEnum.append("screen");
|
||||
modeEnum.append("region");
|
||||
modeEnum.append("window");
|
||||
modeEnum.append("monitor");
|
||||
modeProp["enum"] = modeEnum;
|
||||
modeProp["description"] = u8"抓取范围,默认 screen。screen=整个虚拟桌面(可缩放);region=指定矩形;window=指定窗口;monitor=指定显示器。region/window/monitor 默认按源尺寸 1:1 抓取(原生分辨率,适合 OCR/视觉读字),只有显式传 max_width 才缩放;这三种需要较新的 Windows 客户端(能力位 CLIENT_CAP_SCREEN_REGION),否则返回 -32005。";
|
||||
props["mode"] = modeProp;
|
||||
|
||||
Json::Value xProp(Json::objectValue);
|
||||
xProp["type"] = "integer";
|
||||
xProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 X 坐标(可为负;多显示器时以虚拟桌面左上角为原点)。省略按 0 处理。";
|
||||
props["x"] = xProp;
|
||||
|
||||
Json::Value yProp(Json::objectValue);
|
||||
yProp["type"] = "integer";
|
||||
yProp["description"] = u8"region 模式:矩形左上角的虚拟桌面 Y 坐标(可为负)。省略按 0 处理。";
|
||||
props["y"] = yProp;
|
||||
|
||||
Json::Value wProp(Json::objectValue);
|
||||
wProp["type"] = "integer";
|
||||
wProp["description"] = u8"region 模式必填:矩形宽度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
|
||||
props["width"] = wProp;
|
||||
|
||||
Json::Value hProp(Json::objectValue);
|
||||
hProp["type"] = "integer";
|
||||
hProp["description"] = u8"region 模式必填:矩形高度(像素,1~8192)。超出虚拟桌面的部分会被裁剪。";
|
||||
props["height"] = hProp;
|
||||
|
||||
Json::Value hwndProp(Json::objectValue);
|
||||
Json::Value hwndTypes(Json::arrayValue);
|
||||
hwndTypes.append("integer");
|
||||
hwndTypes.append("string");
|
||||
hwndProp["type"] = hwndTypes;
|
||||
hwndProp["description"] = u8"window 模式必填:窗口句柄(整数或 0x 前缀字符串),取 host_list_windows / get_foreground_window 返回的 hwnd。窗口最小化时无法抓取(返回错误),先 activate/restore。";
|
||||
props["hwnd"] = hwndProp;
|
||||
|
||||
Json::Value monProp(Json::objectValue);
|
||||
monProp["type"] = "integer";
|
||||
monProp["description"] = u8"monitor 模式:显示器序号,0=主显示器(默认),1..N 为其余显示器(按系统枚举顺序)。序号越界返回错误。";
|
||||
props["monitor"] = monProp;
|
||||
|
||||
Json::Value mwProp(Json::objectValue);
|
||||
mwProp["type"] = "integer";
|
||||
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);传 1920 可拿到接近原分辨率(1080p 源屏即原分辨率,4K 源屏最多 1920),供 AI 视觉/OCR 场景提升清晰度。";
|
||||
mwProp["description"] = u8"期望图片最大宽度(像素,钳制到 64~1920)。screen 模式省略或传 0 时沿用 RTT 自适应缩略图档位(最大 1024);region/window/monitor 模式省略/传 0 时按源尺寸 1:1 抓取(原生分辨率),显式传入才等比缩小。";
|
||||
props["max_width"] = mwProp;
|
||||
|
||||
Json::Value schema(Json::objectValue);
|
||||
@@ -2131,7 +2196,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
|
||||
{
|
||||
Json::Value tool(Json::objectValue);
|
||||
tool["name"] = "get_screenshot";
|
||||
tool["description"] = u8"截取指定在线主机的一帧屏幕,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。";
|
||||
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
|
||||
|
||||
tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选
|
||||
tool["outputSchema"] = BuildScreenshotOutputSchema();
|
||||
@@ -2855,6 +2920,8 @@ void AppendHostTechnicalDetail(CMy2015RemoteDlg* parent, uint64_t devId, Json::V
|
||||
Json::Value caps(Json::objectValue);
|
||||
caps["platform"] = ToUtf8(clientType.GetString(), cp); // EXE/DLL/MEXE/SC/MDLL/LNX/MAC/APK
|
||||
caps["screenPreview"] = ctx->SupportsScreenPreview();
|
||||
// screenRegion → get_screenshot 的 region/window/monitor 模式(SupportsScreenCapture)
|
||||
caps["screenRegion"] = ctx->SupportsScreenCapture();
|
||||
// 远程控制只实现于 Windows / Linux;多显示器还要求观察与注入同源:
|
||||
// Windows 需声明整虚拟桌面抓屏能力,Linux 的 root 抓屏/注入天然一致。
|
||||
// monitorCount=0(老客户端没报格式)无法判定,与 remote_open 一样按放行处理。
|
||||
@@ -3198,6 +3265,30 @@ std::string BuildGetActivityHistory(const Json::Value& id, const Json::Value& ar
|
||||
return BuildResult(id, result);
|
||||
}
|
||||
|
||||
// 读可选整数参数(缺省/ null 时用 defVal);越界/类型错误返回 false 并填好错误响应
|
||||
static bool GetOptIntInRange(const Json::Value& id, const Json::Value& args, const char* key,
|
||||
int lo, int hi, int defVal, int& out, std::string& errJson) {
|
||||
out = defVal;
|
||||
if (!args.isMember(key) || args[key].isNull()) return true;
|
||||
if (!args[key].isInt()) {
|
||||
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": expected an integer");
|
||||
return false;
|
||||
}
|
||||
int n = args[key].asInt();
|
||||
if (n < lo || n > hi) {
|
||||
errJson = BuildError(id, -32602, std::string("Invalid ") + key + ": out of range [" +
|
||||
std::to_string(lo) + "," + std::to_string(hi) + "]");
|
||||
return false;
|
||||
}
|
||||
out = n;
|
||||
return true;
|
||||
}
|
||||
|
||||
// 定义在文件后部(expect_hwnd 解析),这里先声明以便 get_screenshot 复用
|
||||
static bool GetOptionalWindowArg(const Json::Value& args, const char* key, uint32_t& out,
|
||||
bool& present, std::string& err);
|
||||
static std::string Trim(const std::string& s);
|
||||
|
||||
// tools/call:get_screenshot(屏幕预览链路,主连接 RPC,不建子链接、不弹框)
|
||||
std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, CMy2015RemoteDlg* parent) {
|
||||
uint64_t devId = 0;
|
||||
@@ -3214,6 +3305,65 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
if (!ctx->SupportsScreenPreview())
|
||||
return BuildError(id, -32005, "Host does not support screen preview");
|
||||
|
||||
// ---- 抓取范围 ----
|
||||
std::string modeStr = Trim(GetStringArg(args, "mode"));
|
||||
if (modeStr.empty()) modeStr = "screen";
|
||||
int mode = -1;
|
||||
if (modeStr == "screen") mode = SCREEN_CAPTURE_VDESKTOP;
|
||||
else if (modeStr == "region") mode = SCREEN_CAPTURE_REGION;
|
||||
else if (modeStr == "window") mode = SCREEN_CAPTURE_WINDOW;
|
||||
else if (modeStr == "monitor") mode = SCREEN_CAPTURE_MONITOR;
|
||||
else return BuildError(id, -32602,
|
||||
"Invalid mode: " + modeStr + " (expect screen|region|window|monitor)");
|
||||
|
||||
// 新客户端(CLIENT_CAP_SCREEN_REGION)支持 v2;老客户端只能整屏。
|
||||
const bool supportsRegion = ctx->SupportsScreenCapture();
|
||||
if (mode != SCREEN_CAPTURE_VDESKTOP && !supportsRegion)
|
||||
return BuildError(id, -32005,
|
||||
"mode=region/window/monitor requires a newer Windows client "
|
||||
"(screen region capability); use mode=screen instead");
|
||||
|
||||
std::string errJson;
|
||||
int rx = 0, ry = 0, rw = 0, rh = 0, monitorIdx = 0;
|
||||
uint32_t hwnd = 0;
|
||||
if (mode == SCREEN_CAPTURE_REGION) {
|
||||
if (!GetOptIntInRange(id, args, "x", -100000, 100000, 0, rx, errJson)) return errJson;
|
||||
if (!GetOptIntInRange(id, args, "y", -100000, 100000, 0, ry, errJson)) return errJson;
|
||||
if (!args.isMember("width") || args["width"].isNull())
|
||||
return BuildError(id, -32602, "Missing required parameter: width (mode=region)");
|
||||
if (!args.isMember("height") || args["height"].isNull())
|
||||
return BuildError(id, -32602, "Missing required parameter: height (mode=region)");
|
||||
if (!GetOptIntInRange(id, args, "width", 1, 8192, 0, rw, errJson)) return errJson;
|
||||
if (!GetOptIntInRange(id, args, "height", 1, 8192, 0, rh, errJson)) return errJson;
|
||||
} else if (mode == SCREEN_CAPTURE_WINDOW) {
|
||||
bool hasHwnd = false;
|
||||
std::string oerr;
|
||||
if (!GetOptionalWindowArg(args, "hwnd", hwnd, hasHwnd, oerr))
|
||||
return BuildError(id, -32602, oerr);
|
||||
if (!hasHwnd)
|
||||
return BuildError(id, -32602, "Missing required parameter: hwnd (mode=window)");
|
||||
} else if (mode == SCREEN_CAPTURE_MONITOR) {
|
||||
if (!GetOptIntInRange(id, args, "monitor", 0, 255, 0, monitorIdx, errJson)) return errJson;
|
||||
}
|
||||
|
||||
// max_width:screen 模式下缺省/0 沿用 RTT 自适应缩略图档位;region/window/monitor
|
||||
// 缺省/0 表示原生分辨率(1:1),显式传入才等比缩放。
|
||||
bool hasMaxWidth = false;
|
||||
int maxWidthArg = 0;
|
||||
if (args.isMember("max_width") && !args["max_width"].isNull()) {
|
||||
if (!args["max_width"].isInt())
|
||||
return BuildError(id, -32602, "Invalid max_width: expected an integer");
|
||||
maxWidthArg = args["max_width"].asInt();
|
||||
// 与 inputSchema 描述一致:钳制到 [64,1920](老行为),不因越界直接报错
|
||||
if (maxWidthArg > 0) {
|
||||
if (maxWidthArg < 64) maxWidthArg = 64;
|
||||
if (maxWidthArg > 1920) maxWidthArg = 1920;
|
||||
hasMaxWidth = true;
|
||||
} else {
|
||||
maxWidthArg = 0;
|
||||
}
|
||||
}
|
||||
|
||||
CMcpServer& mcp = CMcpServer::Instance();
|
||||
if (!mcp.BeginPending(devId, "get_screenshot"))
|
||||
return BuildError(id, -32003, BusyMessage("Device busy: another request is pending for this host", devId));
|
||||
@@ -3225,35 +3375,66 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
WORD maxWidth = 0;
|
||||
BYTE quality = 0;
|
||||
parent->ChooseScreenPreviewParams(ctx, maxWidth, quality);
|
||||
if (hasMaxWidth) maxWidth = (WORD)maxWidthArg;
|
||||
|
||||
// 可选 max_width:缺省/0 沿用缩略图档位;>0 覆盖宽度(钳制到客户端上限 [64,1920]),
|
||||
// 供 AI 视觉/OCR 场景请求接近原分辨率的大图。jpegQuality 仍沿用档位自适应值。
|
||||
if (args.isMember("max_width") && args["max_width"].isInt()) {
|
||||
int mw = args["max_width"].asInt();
|
||||
if (mw > 0) {
|
||||
if (mw < 64) mw = 64;
|
||||
if (mw > 1920) mw = 1920;
|
||||
maxWidth = (WORD)mw;
|
||||
if (mode == SCREEN_CAPTURE_VDESKTOP) {
|
||||
if (supportsRegion) {
|
||||
// 新客户端也走 v2:多回一份源矩形元数据,但目标仍是整屏 + 缩放。
|
||||
ScreenPreviewReqEx ex = {};
|
||||
ex.mode = SCREEN_CAPTURE_VDESKTOP;
|
||||
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
|
||||
} else {
|
||||
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
|
||||
}
|
||||
} else {
|
||||
ScreenPreviewReqEx ex = {};
|
||||
ex.mode = (uint8_t)mode;
|
||||
if (!hasMaxWidth) ex.flags |= SCREEN_CAPTURE_F_NATIVE;
|
||||
ex.x = rx;
|
||||
ex.y = ry;
|
||||
ex.w = rw;
|
||||
ex.h = rh;
|
||||
ex.hwnd = hwnd;
|
||||
ex.monitor = (uint16_t)monitorIdx;
|
||||
parent->SendScreenPreviewRequestEx(ctx, reqId, maxWidth, quality, ex);
|
||||
}
|
||||
|
||||
parent->SendScreenPreviewRequest(ctx, reqId, maxWidth, quality);
|
||||
|
||||
std::vector<BYTE> data;
|
||||
if (!mcp.WaitPending(devId, data, mcp.ToolTimeoutMs()))
|
||||
return BuildError(id, -32001, "Timeout waiting for screenshot");
|
||||
|
||||
// data = [ScreenPreviewRspHeader][JPEG]
|
||||
// data = [ScreenPreviewRspHeader][JPEG][可选 ScreenCaptureMeta]
|
||||
if (data.size() < sizeof(ScreenPreviewRspHeader))
|
||||
return BuildError(id, -32000, "Invalid screenshot response");
|
||||
|
||||
const ScreenPreviewRspHeader* hdr = reinterpret_cast<const ScreenPreviewRspHeader*>(data.data());
|
||||
if (hdr->status == SCREEN_PREVIEW_BAD_TARGET) {
|
||||
if (mode == SCREEN_CAPTURE_WINDOW)
|
||||
return BuildError(id, -32009, "Window hwnd=" + std::to_string(hwnd) +
|
||||
" is not capturable (not found, minimized, off-screen, "
|
||||
"or larger than 8192 px per side)");
|
||||
if (mode == SCREEN_CAPTURE_MONITOR)
|
||||
return BuildError(id, -32602, "Monitor index " + std::to_string(monitorIdx) +
|
||||
" is out of range");
|
||||
if (mode == SCREEN_CAPTURE_REGION)
|
||||
return BuildError(id, -32602, "Capture region is empty after clipping to the virtual desktop");
|
||||
return BuildError(id, -32000, "Screen is not capturable");
|
||||
}
|
||||
if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG ||
|
||||
hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) {
|
||||
return BuildError(id, -32000,
|
||||
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")");
|
||||
}
|
||||
|
||||
// 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。
|
||||
// 偏移 = 16 + bytes,可能不是 4 字节对齐,按字节拷贝到局部变量再读。
|
||||
bool hasMeta = false;
|
||||
ScreenCaptureMeta meta = {};
|
||||
if (data.size() >= sizeof(ScreenPreviewRspHeader) + hdr->bytes + sizeof(ScreenCaptureMeta)) {
|
||||
memcpy(&meta, data.data() + sizeof(ScreenPreviewRspHeader) + hdr->bytes, sizeof(meta));
|
||||
hasMeta = (meta.magic == SCREEN_CAPTURE_META_MAGIC);
|
||||
}
|
||||
|
||||
std::string b64 = httplib::detail::base64_encode(
|
||||
std::string((const char*)data.data() + sizeof(ScreenPreviewRspHeader), hdr->bytes));
|
||||
|
||||
@@ -3263,9 +3444,21 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
image["height"] = (Json::UInt64)hdr->height;
|
||||
image["bytes"] = (Json::UInt64)hdr->bytes;
|
||||
|
||||
Json::Value capture(Json::objectValue);
|
||||
capture["mode"] = modeStr;
|
||||
if (hasMeta) {
|
||||
capture["x"] = meta.x;
|
||||
capture["y"] = meta.y;
|
||||
capture["width"] = meta.w;
|
||||
capture["height"] = meta.h;
|
||||
capture["scale"] = (int)meta.scale;
|
||||
capture["native"] = meta.scale >= 100;
|
||||
}
|
||||
|
||||
Json::Value result(Json::objectValue);
|
||||
Json::Value structuredContent(Json::objectValue);
|
||||
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
|
||||
structuredContent["image"] = image; // 元数据(不含 base64,避免在 content 之外重复大 payload)
|
||||
structuredContent["capture"] = capture;
|
||||
result["structuredContent"] = structuredContent;
|
||||
|
||||
Json::Value content(Json::arrayValue);
|
||||
|
||||
@@ -123,6 +123,14 @@ public:
|
||||
return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_WINDOW_INFO) != 0;
|
||||
}
|
||||
|
||||
// 检查客户端是否支持区域/窗口/显示器原生分辨率截图(COMMAND_SCREEN_PREVIEW_REQ 的
|
||||
// v2 扩展,CLIENT_CAP_SCREEN_REGION)。未声明时 get_screenshot 只能整屏抓取。
|
||||
bool SupportsScreenCapture() const {
|
||||
CString caps = GetClientData(ONLINELIST_CAPABILITIES);
|
||||
if (caps.IsEmpty()) return false;
|
||||
return (strtoul(caps.GetString(), nullptr, 16) & CLIENT_CAP_SCREEN_REGION) != 0;
|
||||
}
|
||||
|
||||
// 检查客户端是否支持远程协议切换(COMMAND_SET_PROTO)。老客户端没有这个命令,
|
||||
// 收到也会走 default 丢弃,服务端据此把「协议切换」菜单置灰。
|
||||
bool SupportsProtoSwitch() const {
|
||||
|
||||
Reference in new issue
Block a user