Feature: Add screen preview to the Android client

get_screenshot and the server's online-list and hover thumbnails were
gated to Windows and X11 Linux. The Android client now answers
COMMAND_SCREEN_PREVIEW_REQ on the main connection with
TOKEN_SCREEN_PREVIEW_RSP plus a JPEG, and advertises
CLIENT_CAP_SCREEN_PREVIEW at login only when it can really capture.

Capture goes through AccessibilityService.takeScreenshot (API 30+),
enabled by canTakeScreenshot in the service config. MediaProjection is
deliberately not reused: its VirtualDisplay already feeds the H.264
encoder and Android 14+ refuses a second one from the same projection,
while accessibility capture runs alongside the live stream. The JPEG is
scaled to maxWidth and encoded in Kotlin; native only copies the bytes
and drops a request while one is in flight. Only mode=screen is
supported, and capabilities are re-evaluated on every login so turning
the service off withdraws the capability.

Also fix two review findings: a screenshot callback that lands after
the 3 s timeout now recycles its bitmap instead of leaking it, and the
server's failure hint lists the possible Android causes instead of
asserting disabled accessibility.

Verified on the emulator against a local debug server: get_screenshot
returns 640x1422 and 720x1600 JPEGs, the 120-wide list thumbnail works,
and both MCP tool-table checkers pass with apk added for get_screenshot,
get_activity_history and get_client_log.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-11 20:58:40 +01:00
1 parent 66acde0301
commit f29150d20e
6 files changed
+346 -16

No files matched your search

+171
View File
@@ -226,6 +226,13 @@ static jmethodID g_forceFirstFrameMid = nullptr;
static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String)
static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务
static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
static jclass g_shotClass = nullptr;
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
static jmethodID g_shotCaptureMid = nullptr; // ScreenShot.capture(II)Lcom/yama/client/ScreenShot$Shot;
static jmethodID g_shotDataMid = nullptr; // Shot.getData()[B
static jmethodID g_shotWidthMid = nullptr; // Shot.getWidth()I
static jmethodID g_shotHeightMid = nullptr; // Shot.getHeight()I
// 心跳日志限流:每 60 秒最多打一次
static std::atomic<uint64_t> g_lastHbLogMs{0};
@@ -598,6 +605,118 @@ static void ActivityReportThread()
g_activityReportsInFlight.fetch_sub(1, std::memory_order_relaxed);
}
// ------------------------------------------------------------------ 屏幕预览
// 对应 Windows/Linux 的 COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP。
// Android 的 JPEG 在 Kotlin 侧(ScreenShot,无障碍 takeScreenshot)编码好,这里只搬运;
// 与 Linux 回 BGRX 不同,服务端对 JPEG 直接透传,不再二次编码。
static std::atomic<int> g_previewInFlight(0);
// 抓一帧 JPEG:JNI 调 ScreenShot.capture(maxWidth, quality),把 Shot 的数据/宽高取回来。
// 失败(无障碍没开、系统限流、编码失败)返回 false,调用方回 CAPTURE_FAILED。
static bool CaptureScreenJpeg(uint16_t maxWidth, uint8_t quality,
std::vector<uint8_t>& jpeg, int& w, int& h)
{
jpeg.clear();
w = h = 0;
if (!g_jvm || !g_shotClass || !g_shotCaptureMid || !g_shotDataMid ||
!g_shotWidthMid || !g_shotHeightMid)
return false;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
attached = true;
} else if (st != JNI_OK || !jenv) {
return false;
}
bool ok = false;
jobject shot = jenv->CallStaticObjectMethod(g_shotClass, g_shotCaptureMid,
(jint)maxWidth, (jint)quality);
if (shot && !jenv->ExceptionCheck()) {
jbyteArray arr = (jbyteArray)jenv->CallObjectMethod(shot, g_shotDataMid);
if (!jenv->ExceptionCheck() && arr) {
jint bw = jenv->CallIntMethod(shot, g_shotWidthMid);
jint bh = jenv->CallIntMethod(shot, g_shotHeightMid);
if (!jenv->ExceptionCheck() && bw > 0 && bh > 0) {
jsize len = jenv->GetArrayLength(arr);
if (len > 0) {
jpeg.resize((size_t)len);
jenv->GetByteArrayRegion(arr, 0, len, (jbyte*)jpeg.data());
if (!jenv->ExceptionCheck()) {
w = bw;
h = bh;
ok = true;
}
}
}
jenv->DeleteLocalRef(arr);
}
jenv->DeleteLocalRef(shot);
}
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
if (attached) g_jvm->DetachCurrentThread();
if (!ok) jpeg.clear();
return ok;
}
// 登录前探测:无障碍服务在跑(API 30+)才声明 CLIENT_CAP_SCREEN_PREVIEW,
// 服务端据此放行 get_screenshot。能力位随每次登录刷新,无障碍被关掉重连后会撤回。
static bool ScreenPreviewAvailable()
{
if (!g_jvm || !g_shotClass || !g_shotCanMid) return false;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
attached = true;
} else if (st != JNI_OK || !jenv) {
return false;
}
jboolean can = jenv->CallStaticBooleanMethod(g_shotClass, g_shotCanMid);
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); can = JNI_FALSE; }
if (attached) g_jvm->DetachCurrentThread();
return can == JNI_TRUE;
}
// 抓屏可能耗几十~几百毫秒(系统回调 + 编码),必须离开接收线程;超额请求直接丢弃
// (服务端会超时按「预览不可用」降级,与 Linux/Windows 的限流一致)。
static void ScreenPreviewThread(ScreenPreviewReq req)
{
struct InFlightGuard {
~InFlightGuard() { g_previewInFlight.fetch_sub(1); }
} guard;
std::vector<uint8_t> jpeg;
int w = 0, h = 0;
uint8_t status = CaptureScreenJpeg(req.maxWidth, req.jpegQuality, jpeg, w, h)
? (uint8_t)SCREEN_PREVIEW_OK
: (uint8_t)SCREEN_PREVIEW_CAPTURE_FAILED;
uint32_t bytes = (status == SCREEN_PREVIEW_OK) ? (uint32_t)jpeg.size() : 0;
std::vector<uint8_t> pkt(sizeof(ScreenPreviewRspHeader) + bytes, 0);
ScreenPreviewRspHeader hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.token = TOKEN_SCREEN_PREVIEW_RSP;
hdr.reqId = req.reqId;
hdr.status = status;
hdr.format = SCREEN_PREVIEW_FMT_JPEG; // 失败时 format 无意义,填 JPEG 保持包头合法
hdr.width = (uint16_t)w;
hdr.height = (uint16_t)h;
hdr.bytes = bytes;
memcpy(pkt.data(), &hdr, sizeof(hdr));
if (bytes) memcpy(pkt.data() + sizeof(hdr), jpeg.data(), bytes);
// 与 ActivityReportThread 同一套路:全局指针只取一次快照
IOCPClient* mc = g_pMainClient;
if (mc && mc->IsConnected() && g_bExit == S_CLIENT_NORMAL)
mc->Send2Server((char*)pkt.data(), (ULONG)pkt.size());
LOGI("[Preview] reqId=%u %dx%d %u bytes status=%u", req.reqId, w, h, bytes, status);
}
// ------------------------------------------------------------------ 在线升级
// 载荷布局:[COMMAND_UPDATE:1][APK 大小:8 LE][APK:size][xxh64:8 LE]
// 校验通过后写到 filesDir/update.apk,交给 Kotlin 的 PackageInstaller 会话去装
@@ -851,6 +970,27 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
}
break;
case COMMAND_SCREEN_PREVIEW_REQ: {
// MCP get_screenshot / 在线列表缩略图:主连接回一次 JPEG
if (ulLength < sizeof(ScreenPreviewReq)) break;
ScreenPreviewReq req;
memcpy(&req, szBuffer, sizeof(req));
if (g_previewInFlight.fetch_add(1) >= 1) {
g_previewInFlight.fetch_sub(1); // 已有抓屏在跑,重复请求丢弃
break;
}
LOGI("cmd: COMMAND_SCREEN_PREVIEW_REQ reqId=%u maxWidth=%u quality=%u",
req.reqId, req.maxWidth, req.jpegQuality);
try {
std::thread(ScreenPreviewThread, req).detach();
} catch (const std::exception& e) {
// 线程起不来要把在途计数还回去,否则之后所有预览请求都被当成「忙」丢掉
g_previewInFlight.fetch_sub(1);
LOGE("Screen preview thread failed: %s", e.what());
}
break;
}
case COMMAND_SHELL:
LOGI("COMMAND_SHELL (not implemented)");
break;
@@ -999,6 +1139,10 @@ static void ConnectionThread(uint64_t gen)
#ifdef YAMA_TLS_CHANNEL
RefreshLoginTls(logInfo); // 传输方式可能已热切换,登录前刷新 RES_TLS
#endif
// 屏幕预览能力位按运行时状态刷新:无障碍服务在跑才声明(API 30+ 的 takeScreenshot),
// 服务端据此放行 get_screenshot;无障碍被关掉后重连会撤回,避免服务端白等抓屏
logInfo.SetCaps(LOGIN_INFOR::DefaultCaps() |
(ScreenPreviewAvailable() ? CLIENT_CAP_SCREEN_PREVIEW : 0));
client->SendLoginInfo(logInfo.Speed(clock() - c));
g_lastHeartbeatAckMs.store(GetUnixMs(), std::memory_order_relaxed);
LOGI("Connected & login sent");
@@ -1191,6 +1335,33 @@ Java_com_yama_client_YamaBridge_nativeInit(
} else {
LOGE("CaptureService class not found");
}
// 屏幕预览:返回对象是 ScreenShot$Shot,JNI 签名用 '$' 写嵌套类
jclass shotCls = env->FindClass("com/yama/client/ScreenShot");
if (env->ExceptionCheck()) { env->ExceptionClear(); shotCls = nullptr; }
if (shotCls) {
g_shotClass = (jclass)env->NewGlobalRef(shotCls);
g_shotCanMid = env->GetStaticMethodID(g_shotClass, "canCapture", "()Z");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCanMid = nullptr; }
if (!g_shotCanMid) LOGE("ScreenShot.canCapture not found");
g_shotCaptureMid = env->GetStaticMethodID(g_shotClass, "capture",
"(II)Lcom/yama/client/ScreenShot$Shot;");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCaptureMid = nullptr; }
if (!g_shotCaptureMid) LOGE("ScreenShot.capture not found");
jclass shotResultCls = env->FindClass("com/yama/client/ScreenShot$Shot");
if (env->ExceptionCheck()) { env->ExceptionClear(); shotResultCls = nullptr; }
if (shotResultCls) {
g_shotDataMid = env->GetMethodID(shotResultCls, "getData", "()[B");
g_shotWidthMid = env->GetMethodID(shotResultCls, "getWidth", "()I");
g_shotHeightMid = env->GetMethodID(shotResultCls, "getHeight", "()I");
if (env->ExceptionCheck()) { env->ExceptionClear(); }
if (!g_shotDataMid || !g_shotWidthMid || !g_shotHeightMid)
LOGE("ScreenShot$Shot getters not found");
} else {
LOGE("ScreenShot$Shot class not found");
}
} else {
LOGE("ScreenShot class not found");
}
}
}
@@ -0,0 +1,138 @@
package com.yama.client
import android.accessibilityservice.AccessibilityService
import android.graphics.Bitmap
import android.os.Build
import android.util.Log
import android.view.Display
import java.io.ByteArrayOutputStream
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executor
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
/**
* 屏幕预览(MCP get_screenshot)的 Android 实现:借无障碍服务的
* [AccessibilityService.takeScreenshot](API 30+)抓一帧,等比缩到 maxWidth 以内再编码 JPEG。
*
* 刻意不走 MediaProjection:投屏的 VirtualDisplay 已经被 H.264 编码器占着,Android 14+
* 同一个投影再建第二个 VirtualDisplay 会失败;无障碍截屏与实时投屏互不干扰,用户拒绝投屏
* 授权时也照样能用。代价是 Android 11 以下没有这条路(不声明能力位,服务端直接拒绝)。
*/
object ScreenShot {
private const val TAG = "YAMA"
private const val CAPTURE_TIMEOUT_MS = 3000L
private const val MAX_WIDTH = 1920
/** 抓屏结果:JPEG 数据 + 实际编码后的宽高(服务端据此回报尺寸/换算比例) */
class Shot(val data: ByteArray, val width: Int, val height: Int)
/**
* 当前能否抓屏:API 30+ 且无障碍服务在跑。takeScreenshot 的能力来自服务配置
* (accessibility_service_config.xml 的 canTakeScreenshot),服务没起来就调不了。
*/
@JvmStatic
fun canCapture(): Boolean {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return false
return ControlService.instance != null
}
/**
* 同步抓一帧并编码 JPEG。从 native 工作线程调用(调用方负责 attach JVM),
* 内部用 CountDownLatch 等系统回调。拿不到画面/超时/编码失败一律返回 null,
* native 侧据此回 SCREEN_PREVIEW_CAPTURE_FAILED。
*/
@JvmStatic
fun capture(maxWidth: Int, quality: Int): Shot? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null
val svc = ControlService.instance ?: return null
val latch = CountDownLatch(1)
val holder = arrayOfNulls<Bitmap>(1)
// 超时返回后回调仍可能到达(系统限流时回调会拖到几秒后)。等待方已经放弃,
// 那张整屏 ARGB_8888(1080x2400 约 10 MB)得有人回收。holder 与 abandoned
// 必须在同一把锁下读写:否则回调可能在 countDown 之后、等待方置标志之前
// 看到 false,把位图留在 holder 里,而那边已经超时返回,位图就永久泄漏。
val gate = Any()
val abandoned = AtomicBoolean(false)
// 回调直接在当前线程跑,省掉再起一个线程;Binder 线程回调也安全
val direct = Executor { it.run() }
try {
svc.takeScreenshot(Display.DEFAULT_DISPLAY, direct,
object : AccessibilityService.TakeScreenshotCallback {
override fun onSuccess(result: AccessibilityService.ScreenshotResult) {
val hw = result.hardwareBuffer
var bmp: Bitmap? = null
try {
// 硬件位图引用着 hw,必须先复制成软件位图再关
val hb = Bitmap.wrapHardwareBuffer(hw, result.colorSpace)
bmp = hb?.copy(Bitmap.Config.ARGB_8888, false)
} catch (e: Throwable) {
Log.e(TAG, "takeScreenshot convert failed: $e")
} finally {
hw.close()
}
synchronized(gate) {
holder[0] = bmp
latch.countDown()
// 等待方已超时返回:这张位图归回调自己回收
if (abandoned.get()) {
holder[0]?.recycle()
holder[0] = null
}
}
}
override fun onFailure(errorCode: Int) {
// ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT 等:系统限流/未授权
Log.e(TAG, "takeScreenshot failed: errorCode=$errorCode")
latch.countDown()
}
})
} catch (e: Throwable) {
Log.e(TAG, "takeScreenshot threw: $e")
return null
}
if (!latch.await(CAPTURE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
// 与回调抢同一把锁:回调若已把位图放进 holder 并检查过标志(那时还是 false),
// 这里负责回收;回调若还没进锁,它随后会看到标志并自己回收。
synchronized(gate) {
abandoned.set(true)
val late = holder[0]
holder[0] = null
late?.recycle()
}
Log.e(TAG, "takeScreenshot timeout")
return null
}
val src = holder[0] ?: return null
return try {
encodeJpeg(src, maxWidth, quality)
} finally {
src.recycle()
}
}
/** 等比缩放到 maxWidth 以内(maxWidth<=0 表示不缩放),再压成 JPEG */
private fun encodeJpeg(src: Bitmap, maxWidth: Int, quality: Int): Shot? {
val limit = if (maxWidth in 1..MAX_WIDTH) maxWidth else 0
val scaled = if (limit > 0 && src.width > limit) {
val h = (src.height.toLong() * limit / src.width).toInt().coerceAtLeast(1)
Bitmap.createScaledBitmap(src, limit, h, true)
} else src
try {
val out = ByteArrayOutputStream()
if (!scaled.compress(Bitmap.CompressFormat.JPEG, quality.coerceIn(10, 100), out)) return null
val data = out.toByteArray()
if (data.isEmpty()) return null
Log.i(TAG, "screenshot ${src.width}x${src.height} → ${scaled.width}x${scaled.height}, ${data.size} bytes")
return Shot(data, scaled.width, scaled.height)
} catch (e: Throwable) {
Log.e(TAG, "JPEG encode failed: $e")
return null
} finally {
if (scaled !== src) scaled.recycle()
}
}
}
@@ -4,5 +4,6 @@
android:accessibilityFeedbackType="feedbackGeneric"
android:canPerformGestures="true"
android:canRetrieveWindowContent="true"
android:canTakeScreenshot="true"
android:notificationTimeout="0"
android:settingsActivity="com.yama.client.MainActivity" />
+8 -5
View File
@@ -906,11 +906,12 @@
},
{
"name": "get_activity_history",
"description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。",
"description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;Android 客户端返回前台应用驻留记录(标题为「应用名 (包名)」,需开启无障碍服务);无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。",
"gate": null,
"platforms": [
"win",
"lnx"
"lnx",
"apk"
],
"impl": "mfc",
"inputSchema": {
@@ -1048,11 +1049,12 @@
},
{
"name": "get_screenshot",
"description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。",
"description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。",
"gate": null,
"platforms": [
"win",
"lnx"
"lnx",
"apk"
],
"impl": "mfc",
"inputSchema": {
@@ -1580,7 +1582,8 @@
"platforms": [
"win",
"lnx",
"mac"
"mac",
"apk"
],
"impl": "mfc",
"inputSchema": {
+15 -4
View File
@@ -345,7 +345,8 @@ const char* kServerInstructions =
u8"文件传输、进程列表、日志可用。\n"
u8"apk(APK):支持主机列表(list_online_hosts / list_all_hosts)/ 详情 / 搜索 / "
u8"审计日志 / 服务状态 / 会话管理、运行日志(get_client_log)、活动历史"
u8"(get_activity_history,前台应用停留记录),以及 "
u8"(get_activity_history,前台应用停留记录)、截屏(get_screenshot,只支持 mode=screen,"
u8"要靠无障碍服务 takeScreenshot,Android 11+ 且开启无障碍才行),以及 "
u8"host_set_remark / host_set_watch_level 这两个只改服务端本地数据的工具。其余一律不支持"
u8"(客户端未实现 SYSTEM 与 SHELL)。\n"
u8"copy_files_c2c 是唯一有方向性的工具:源端 src_id 必须是 win,目标端 dst_id 可以是 "
@@ -2197,7 +2198,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
{
Json::Value tool(Json::objectValue);
tool["name"] = "get_screenshot";
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选
tool["outputSchema"] = BuildScreenshotOutputSchema();
@@ -3423,8 +3424,17 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
}
if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG ||
hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) {
// Android 的抓屏依赖无障碍服务(takeScreenshot),失败时给一句可操作的提示
// (kPlatApk 与 CurrentClientPlatform 定义在文件后部,这里直接看 RES_CLIENT_TYPE)。
// 客户端把所有失败(未开无障碍、系统限流、编码/分配失败、回调超时)都归为
// CAPTURE_FAILED,所以提示要列常见原因而不是断言某一条,细节引导去看客户端日志。
std::string hint;
if (hdr->status == SCREEN_PREVIEW_CAPTURE_FAILED && ctx->GetAdditionalData(RES_CLIENT_TYPE) == "APK")
hint = ": Android client could not capture — check the YAMA accessibility service "
"(Android 11+), system screenshot rate limiting, and get_client_log for the "
"capture/encode error";
return BuildError(id, -32000,
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")");
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")" + hint);
}
// 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。
@@ -9942,7 +9952,8 @@ const ToolPlatform kToolPlatforms[] = {
{ "download_file", kPlatWin|kPlatLnx|kPlatMac },
{ "upload_file", kPlatWin|kPlatLnx|kPlatMac },
// ---- win+lnx ---- 远程控制全族 + 需 SCREEN_PREVIEW 的截屏 + macOS 未开放项
{ "get_screenshot", kPlatWin|kPlatLnx },
// Android 用无障碍 takeScreenshot(API 30+,需开无障碍服务),只支持 mode=screen
{ "get_screenshot", kPlatWin|kPlatLnx|kPlatApk },
// 窗口信息 RPC 只在 Windows 客户端实现(CLIENT_CAP_WINDOW_INFO)
{ "get_foreground_window", kPlatWin },
{ "remote_open", kPlatWin|kPlatLnx },
+13 -7
View File
@@ -119,11 +119,15 @@ GATE = {
# 支持面是产品定位的阶梯:win 100% > lnx 46% > mac 28% ≈ apk 16%。
# Windows 是唯一全量支持的平台,check_platform_table.py 会把这条做成不变式。
#
# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp:534-540 明确
# 标注 not implemented,且客户端未声明 SCREEN_PREVIEW,故除服务端本地类外几乎全不支持。
# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp 明确标注 not
# implemented,故除服务端本地类之外,只有少量按需补过分支的能力:运行日志
# (COMMAND_QUERY_LOG)、活动历史(COMMAND_QUERY_ACTIVITY)、截屏
# (COMMAND_SCREEN_PREVIEW_REQ,靠无障碍 takeScreenshot,Android 11+ 且开无障碍时
# 登录才声明 SCREEN_PREVIEW 能力位)。
_APK = ["win", "lnx", "mac", "apk"]
_TRIO = ["win", "lnx", "mac"]
_WL = ["win", "lnx"]
_WLA = ["win", "lnx", "apk"]
_WIN = ["win"]
PLATFORMS = {
@@ -139,10 +143,11 @@ PLATFORMS = {
"host_set_remark": _APK,
"host_set_watch_level": _APK,
# SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了
# SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了;APK 也实现了 QUERY_LOG
# (android/.../main.cpp ClientLogThread → TOKEN_REPORT_LOG)
"list_processes": _TRIO,
"list_files": _TRIO,
"get_client_log": _TRIO,
"get_client_log": _APK,
# list_windows 只有 Windows:服务端把 COMMAND_WSLIST 发在主连接上,Linux/macOS
# 主循环只处理 COMMAND_SYSTEM,WSLIST 只在 SystemManager 子连接里处理 —— 走不到,
@@ -170,15 +175,16 @@ PLATFORMS = {
"remote_mouse": _WL,
"remote_clipboard": _WL,
# 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明
"get_screenshot": _WL,
# 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明,
# Android 仅在无障碍服务在跑时声明(takeScreenshot,只支持 mode=screen)
"get_screenshot": _WLA,
# ---- 仅 Windows ----
"list_services": _WIN,
"control_service": _WIN,
"list_registry": _WIN,
"get_keylog": _WIN,
"get_activity_history": _WL, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp)
"get_activity_history": _WLA, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp);APK 复用同一采样机
"get_foreground_window": _WIN, # 窗口信息 RPC:CLIENT_CAP_WINDOW_INFO,仅 Windows 客户端声明
"search_file": _WIN,
"delete_path": _WIN,