From f29150d20e1c2a17511ee9f4a04f1270eba332fe Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Wed, 11 Nov 2026 13:44:53 +0100 Subject: [PATCH] Feature: Add screen preview to the Android client get_screenshot and the server's online-list and hover thumbnails were gated to Windows and X11 Linux. The Android client now answers COMMAND_SCREEN_PREVIEW_REQ on the main connection with TOKEN_SCREEN_PREVIEW_RSP plus a JPEG, and advertises CLIENT_CAP_SCREEN_PREVIEW at login only when it can really capture. Capture goes through AccessibilityService.takeScreenshot (API 30+), enabled by canTakeScreenshot in the service config. MediaProjection is deliberately not reused: its VirtualDisplay already feeds the H.264 encoder and Android 14+ refuses a second one from the same projection, while accessibility capture runs alongside the live stream. The JPEG is scaled to maxWidth and encoded in Kotlin; native only copies the bytes and drops a request while one is in flight. Only mode=screen is supported, and capabilities are re-evaluated on every login so turning the service off withdraws the capability. Also fix two review findings: a screenshot callback that lands after the 3 s timeout now recycles its bitmap instead of leaking it, and the server's failure hint lists the possible Android causes instead of asserting disabled accessibility. Verified on the emulator against a local debug server: get_screenshot returns 640x1422 and 720x1600 JPEGs, the 120-wide list thumbnail works, and both MCP tool-table checkers pass with apk added for get_screenshot, get_activity_history and get_client_log. Co-Authored-By: deepseek-flash --- android/app/src/main/cpp/main.cpp | 171 ++++++++++++++++++ .../main/java/com/yama/client/ScreenShot.kt | 138 ++++++++++++++ .../res/xml/accessibility_service_config.xml | 1 + mcp/tools.json | 13 +- server/2015Remote/McpServer.cpp | 19 +- tools/mcp_tools/make_tools_json.py | 20 +- 6 files changed, 346 insertions(+), 16 deletions(-) create mode 100644 android/app/src/main/java/com/yama/client/ScreenShot.kt diff --git a/android/app/src/main/cpp/main.cpp b/android/app/src/main/cpp/main.cpp index d8c1e45..a31f710 100644 --- a/android/app/src/main/cpp/main.cpp +++ b/android/app/src/main/cpp/main.cpp @@ -226,6 +226,13 @@ static jmethodID g_forceFirstFrameMid = nullptr; static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String) static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务 static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘 +// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用) +static jclass g_shotClass = nullptr; +static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z +static jmethodID g_shotCaptureMid = nullptr; // ScreenShot.capture(II)Lcom/yama/client/ScreenShot$Shot; +static jmethodID g_shotDataMid = nullptr; // Shot.getData()[B +static jmethodID g_shotWidthMid = nullptr; // Shot.getWidth()I +static jmethodID g_shotHeightMid = nullptr; // Shot.getHeight()I // 心跳日志限流:每 60 秒最多打一次 static std::atomic g_lastHbLogMs{0}; @@ -598,6 +605,118 @@ static void ActivityReportThread() g_activityReportsInFlight.fetch_sub(1, std::memory_order_relaxed); } +// ------------------------------------------------------------------ 屏幕预览 +// 对应 Windows/Linux 的 COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP。 +// Android 的 JPEG 在 Kotlin 侧(ScreenShot,无障碍 takeScreenshot)编码好,这里只搬运; +// 与 Linux 回 BGRX 不同,服务端对 JPEG 直接透传,不再二次编码。 +static std::atomic g_previewInFlight(0); + +// 抓一帧 JPEG:JNI 调 ScreenShot.capture(maxWidth, quality),把 Shot 的数据/宽高取回来。 +// 失败(无障碍没开、系统限流、编码失败)返回 false,调用方回 CAPTURE_FAILED。 +static bool CaptureScreenJpeg(uint16_t maxWidth, uint8_t quality, + std::vector& jpeg, int& w, int& h) +{ + jpeg.clear(); + w = h = 0; + if (!g_jvm || !g_shotClass || !g_shotCaptureMid || !g_shotDataMid || + !g_shotWidthMid || !g_shotHeightMid) + return false; + + JNIEnv* jenv = nullptr; + bool attached = false; + jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6); + if (st == JNI_EDETACHED) { + if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false; + attached = true; + } else if (st != JNI_OK || !jenv) { + return false; + } + + bool ok = false; + jobject shot = jenv->CallStaticObjectMethod(g_shotClass, g_shotCaptureMid, + (jint)maxWidth, (jint)quality); + if (shot && !jenv->ExceptionCheck()) { + jbyteArray arr = (jbyteArray)jenv->CallObjectMethod(shot, g_shotDataMid); + if (!jenv->ExceptionCheck() && arr) { + jint bw = jenv->CallIntMethod(shot, g_shotWidthMid); + jint bh = jenv->CallIntMethod(shot, g_shotHeightMid); + if (!jenv->ExceptionCheck() && bw > 0 && bh > 0) { + jsize len = jenv->GetArrayLength(arr); + if (len > 0) { + jpeg.resize((size_t)len); + jenv->GetByteArrayRegion(arr, 0, len, (jbyte*)jpeg.data()); + if (!jenv->ExceptionCheck()) { + w = bw; + h = bh; + ok = true; + } + } + } + jenv->DeleteLocalRef(arr); + } + jenv->DeleteLocalRef(shot); + } + if (jenv->ExceptionCheck()) jenv->ExceptionClear(); + if (attached) g_jvm->DetachCurrentThread(); + if (!ok) jpeg.clear(); + return ok; +} + +// 登录前探测:无障碍服务在跑(API 30+)才声明 CLIENT_CAP_SCREEN_PREVIEW, +// 服务端据此放行 get_screenshot。能力位随每次登录刷新,无障碍被关掉重连后会撤回。 +static bool ScreenPreviewAvailable() +{ + if (!g_jvm || !g_shotClass || !g_shotCanMid) return false; + JNIEnv* jenv = nullptr; + bool attached = false; + jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6); + if (st == JNI_EDETACHED) { + if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false; + attached = true; + } else if (st != JNI_OK || !jenv) { + return false; + } + jboolean can = jenv->CallStaticBooleanMethod(g_shotClass, g_shotCanMid); + if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); can = JNI_FALSE; } + if (attached) g_jvm->DetachCurrentThread(); + return can == JNI_TRUE; +} + +// 抓屏可能耗几十~几百毫秒(系统回调 + 编码),必须离开接收线程;超额请求直接丢弃 +// (服务端会超时按「预览不可用」降级,与 Linux/Windows 的限流一致)。 +static void ScreenPreviewThread(ScreenPreviewReq req) +{ + struct InFlightGuard { + ~InFlightGuard() { g_previewInFlight.fetch_sub(1); } + } guard; + + std::vector jpeg; + int w = 0, h = 0; + uint8_t status = CaptureScreenJpeg(req.maxWidth, req.jpegQuality, jpeg, w, h) + ? (uint8_t)SCREEN_PREVIEW_OK + : (uint8_t)SCREEN_PREVIEW_CAPTURE_FAILED; + + uint32_t bytes = (status == SCREEN_PREVIEW_OK) ? (uint32_t)jpeg.size() : 0; + std::vector pkt(sizeof(ScreenPreviewRspHeader) + bytes, 0); + ScreenPreviewRspHeader hdr; + memset(&hdr, 0, sizeof(hdr)); + hdr.token = TOKEN_SCREEN_PREVIEW_RSP; + hdr.reqId = req.reqId; + hdr.status = status; + hdr.format = SCREEN_PREVIEW_FMT_JPEG; // 失败时 format 无意义,填 JPEG 保持包头合法 + hdr.width = (uint16_t)w; + hdr.height = (uint16_t)h; + hdr.bytes = bytes; + memcpy(pkt.data(), &hdr, sizeof(hdr)); + if (bytes) memcpy(pkt.data() + sizeof(hdr), jpeg.data(), bytes); + + // 与 ActivityReportThread 同一套路:全局指针只取一次快照 + IOCPClient* mc = g_pMainClient; + if (mc && mc->IsConnected() && g_bExit == S_CLIENT_NORMAL) + mc->Send2Server((char*)pkt.data(), (ULONG)pkt.size()); + LOGI("[Preview] reqId=%u %dx%d %u bytes status=%u", req.reqId, w, h, bytes, status); +} + // ------------------------------------------------------------------ 在线升级 // 载荷布局:[COMMAND_UPDATE:1][APK 大小:8 LE][APK:size][xxh64:8 LE] // 校验通过后写到 filesDir/update.apk,交给 Kotlin 的 PackageInstaller 会话去装 @@ -851,6 +970,27 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength) } break; + case COMMAND_SCREEN_PREVIEW_REQ: { + // MCP get_screenshot / 在线列表缩略图:主连接回一次 JPEG + if (ulLength < sizeof(ScreenPreviewReq)) break; + ScreenPreviewReq req; + memcpy(&req, szBuffer, sizeof(req)); + if (g_previewInFlight.fetch_add(1) >= 1) { + g_previewInFlight.fetch_sub(1); // 已有抓屏在跑,重复请求丢弃 + break; + } + LOGI("cmd: COMMAND_SCREEN_PREVIEW_REQ reqId=%u maxWidth=%u quality=%u", + req.reqId, req.maxWidth, req.jpegQuality); + try { + std::thread(ScreenPreviewThread, req).detach(); + } catch (const std::exception& e) { + // 线程起不来要把在途计数还回去,否则之后所有预览请求都被当成「忙」丢掉 + g_previewInFlight.fetch_sub(1); + LOGE("Screen preview thread failed: %s", e.what()); + } + break; + } + case COMMAND_SHELL: LOGI("COMMAND_SHELL (not implemented)"); break; @@ -999,6 +1139,10 @@ static void ConnectionThread(uint64_t gen) #ifdef YAMA_TLS_CHANNEL RefreshLoginTls(logInfo); // 传输方式可能已热切换,登录前刷新 RES_TLS #endif + // 屏幕预览能力位按运行时状态刷新:无障碍服务在跑才声明(API 30+ 的 takeScreenshot), + // 服务端据此放行 get_screenshot;无障碍被关掉后重连会撤回,避免服务端白等抓屏 + logInfo.SetCaps(LOGIN_INFOR::DefaultCaps() | + (ScreenPreviewAvailable() ? CLIENT_CAP_SCREEN_PREVIEW : 0)); client->SendLoginInfo(logInfo.Speed(clock() - c)); g_lastHeartbeatAckMs.store(GetUnixMs(), std::memory_order_relaxed); LOGI("Connected & login sent"); @@ -1191,6 +1335,33 @@ Java_com_yama_client_YamaBridge_nativeInit( } else { LOGE("CaptureService class not found"); } + // 屏幕预览:返回对象是 ScreenShot$Shot,JNI 签名用 '$' 写嵌套类 + jclass shotCls = env->FindClass("com/yama/client/ScreenShot"); + if (env->ExceptionCheck()) { env->ExceptionClear(); shotCls = nullptr; } + if (shotCls) { + g_shotClass = (jclass)env->NewGlobalRef(shotCls); + g_shotCanMid = env->GetStaticMethodID(g_shotClass, "canCapture", "()Z"); + if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCanMid = nullptr; } + if (!g_shotCanMid) LOGE("ScreenShot.canCapture not found"); + g_shotCaptureMid = env->GetStaticMethodID(g_shotClass, "capture", + "(II)Lcom/yama/client/ScreenShot$Shot;"); + if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCaptureMid = nullptr; } + if (!g_shotCaptureMid) LOGE("ScreenShot.capture not found"); + jclass shotResultCls = env->FindClass("com/yama/client/ScreenShot$Shot"); + if (env->ExceptionCheck()) { env->ExceptionClear(); shotResultCls = nullptr; } + if (shotResultCls) { + g_shotDataMid = env->GetMethodID(shotResultCls, "getData", "()[B"); + g_shotWidthMid = env->GetMethodID(shotResultCls, "getWidth", "()I"); + g_shotHeightMid = env->GetMethodID(shotResultCls, "getHeight", "()I"); + if (env->ExceptionCheck()) { env->ExceptionClear(); } + if (!g_shotDataMid || !g_shotWidthMid || !g_shotHeightMid) + LOGE("ScreenShot$Shot getters not found"); + } else { + LOGE("ScreenShot$Shot class not found"); + } + } else { + LOGE("ScreenShot class not found"); + } } } diff --git a/android/app/src/main/java/com/yama/client/ScreenShot.kt b/android/app/src/main/java/com/yama/client/ScreenShot.kt new file mode 100644 index 0000000..197c580 --- /dev/null +++ b/android/app/src/main/java/com/yama/client/ScreenShot.kt @@ -0,0 +1,138 @@ +package com.yama.client + +import android.accessibilityservice.AccessibilityService +import android.graphics.Bitmap +import android.os.Build +import android.util.Log +import android.view.Display +import java.io.ByteArrayOutputStream +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executor +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +/** + * 屏幕预览(MCP get_screenshot)的 Android 实现:借无障碍服务的 + * [AccessibilityService.takeScreenshot](API 30+)抓一帧,等比缩到 maxWidth 以内再编码 JPEG。 + * + * 刻意不走 MediaProjection:投屏的 VirtualDisplay 已经被 H.264 编码器占着,Android 14+ + * 同一个投影再建第二个 VirtualDisplay 会失败;无障碍截屏与实时投屏互不干扰,用户拒绝投屏 + * 授权时也照样能用。代价是 Android 11 以下没有这条路(不声明能力位,服务端直接拒绝)。 + */ +object ScreenShot { + private const val TAG = "YAMA" + private const val CAPTURE_TIMEOUT_MS = 3000L + private const val MAX_WIDTH = 1920 + + /** 抓屏结果:JPEG 数据 + 实际编码后的宽高(服务端据此回报尺寸/换算比例) */ + class Shot(val data: ByteArray, val width: Int, val height: Int) + + /** + * 当前能否抓屏:API 30+ 且无障碍服务在跑。takeScreenshot 的能力来自服务配置 + * (accessibility_service_config.xml 的 canTakeScreenshot),服务没起来就调不了。 + */ + @JvmStatic + fun canCapture(): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return false + return ControlService.instance != null + } + + /** + * 同步抓一帧并编码 JPEG。从 native 工作线程调用(调用方负责 attach JVM), + * 内部用 CountDownLatch 等系统回调。拿不到画面/超时/编码失败一律返回 null, + * native 侧据此回 SCREEN_PREVIEW_CAPTURE_FAILED。 + */ + @JvmStatic + fun capture(maxWidth: Int, quality: Int): Shot? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null + val svc = ControlService.instance ?: return null + + val latch = CountDownLatch(1) + val holder = arrayOfNulls(1) + // 超时返回后回调仍可能到达(系统限流时回调会拖到几秒后)。等待方已经放弃, + // 那张整屏 ARGB_8888(1080x2400 约 10 MB)得有人回收。holder 与 abandoned + // 必须在同一把锁下读写:否则回调可能在 countDown 之后、等待方置标志之前 + // 看到 false,把位图留在 holder 里,而那边已经超时返回,位图就永久泄漏。 + val gate = Any() + val abandoned = AtomicBoolean(false) + // 回调直接在当前线程跑,省掉再起一个线程;Binder 线程回调也安全 + val direct = Executor { it.run() } + try { + svc.takeScreenshot(Display.DEFAULT_DISPLAY, direct, + object : AccessibilityService.TakeScreenshotCallback { + override fun onSuccess(result: AccessibilityService.ScreenshotResult) { + val hw = result.hardwareBuffer + var bmp: Bitmap? = null + try { + // 硬件位图引用着 hw,必须先复制成软件位图再关 + val hb = Bitmap.wrapHardwareBuffer(hw, result.colorSpace) + bmp = hb?.copy(Bitmap.Config.ARGB_8888, false) + } catch (e: Throwable) { + Log.e(TAG, "takeScreenshot convert failed: $e") + } finally { + hw.close() + } + synchronized(gate) { + holder[0] = bmp + latch.countDown() + // 等待方已超时返回:这张位图归回调自己回收 + if (abandoned.get()) { + holder[0]?.recycle() + holder[0] = null + } + } + } + + override fun onFailure(errorCode: Int) { + // ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT 等:系统限流/未授权 + Log.e(TAG, "takeScreenshot failed: errorCode=$errorCode") + latch.countDown() + } + }) + } catch (e: Throwable) { + Log.e(TAG, "takeScreenshot threw: $e") + return null + } + + if (!latch.await(CAPTURE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { + // 与回调抢同一把锁:回调若已把位图放进 holder 并检查过标志(那时还是 false), + // 这里负责回收;回调若还没进锁,它随后会看到标志并自己回收。 + synchronized(gate) { + abandoned.set(true) + val late = holder[0] + holder[0] = null + late?.recycle() + } + Log.e(TAG, "takeScreenshot timeout") + return null + } + val src = holder[0] ?: return null + return try { + encodeJpeg(src, maxWidth, quality) + } finally { + src.recycle() + } + } + + /** 等比缩放到 maxWidth 以内(maxWidth<=0 表示不缩放),再压成 JPEG */ + private fun encodeJpeg(src: Bitmap, maxWidth: Int, quality: Int): Shot? { + val limit = if (maxWidth in 1..MAX_WIDTH) maxWidth else 0 + val scaled = if (limit > 0 && src.width > limit) { + val h = (src.height.toLong() * limit / src.width).toInt().coerceAtLeast(1) + Bitmap.createScaledBitmap(src, limit, h, true) + } else src + try { + val out = ByteArrayOutputStream() + if (!scaled.compress(Bitmap.CompressFormat.JPEG, quality.coerceIn(10, 100), out)) return null + val data = out.toByteArray() + if (data.isEmpty()) return null + Log.i(TAG, "screenshot ${src.width}x${src.height} → ${scaled.width}x${scaled.height}, ${data.size} bytes") + return Shot(data, scaled.width, scaled.height) + } catch (e: Throwable) { + Log.e(TAG, "JPEG encode failed: $e") + return null + } finally { + if (scaled !== src) scaled.recycle() + } + } +} diff --git a/android/app/src/main/res/xml/accessibility_service_config.xml b/android/app/src/main/res/xml/accessibility_service_config.xml index e8339ab..7f646d0 100644 --- a/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/android/app/src/main/res/xml/accessibility_service_config.xml @@ -4,5 +4,6 @@ android:accessibilityFeedbackType="feedbackGeneric" android:canPerformGestures="true" android:canRetrieveWindowContent="true" + android:canTakeScreenshot="true" android:notificationTimeout="0" android:settingsActivity="com.yama.client.MainActivity" /> diff --git a/mcp/tools.json b/mcp/tools.json index 466c08b..1bd1ec4 100644 --- a/mcp/tools.json +++ b/mcp/tools.json @@ -906,11 +906,12 @@ }, { "name": "get_activity_history", - "description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。", + "description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;Android 客户端返回前台应用驻留记录(标题为「应用名 (包名)」,需开启无障碍服务);无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。", "gate": null, "platforms": [ "win", - "lnx" + "lnx", + "apk" ], "impl": "mfc", "inputSchema": { @@ -1048,11 +1049,12 @@ }, { "name": "get_screenshot", - "description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。", + "description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。", "gate": null, "platforms": [ "win", - "lnx" + "lnx", + "apk" ], "impl": "mfc", "inputSchema": { @@ -1580,7 +1582,8 @@ "platforms": [ "win", "lnx", - "mac" + "mac", + "apk" ], "impl": "mfc", "inputSchema": { diff --git a/server/2015Remote/McpServer.cpp b/server/2015Remote/McpServer.cpp index b67a546..87b1a2e 100644 --- a/server/2015Remote/McpServer.cpp +++ b/server/2015Remote/McpServer.cpp @@ -345,7 +345,8 @@ const char* kServerInstructions = u8"文件传输、进程列表、日志可用。\n" u8"apk(APK):支持主机列表(list_online_hosts / list_all_hosts)/ 详情 / 搜索 / " u8"审计日志 / 服务状态 / 会话管理、运行日志(get_client_log)、活动历史" - u8"(get_activity_history,前台应用停留记录),以及 " + u8"(get_activity_history,前台应用停留记录)、截屏(get_screenshot,只支持 mode=screen," + u8"要靠无障碍服务 takeScreenshot,Android 11+ 且开启无障碍才行),以及 " u8"host_set_remark / host_set_watch_level 这两个只改服务端本地数据的工具。其余一律不支持" u8"(客户端未实现 SYSTEM 与 SHELL)。\n" u8"copy_files_c2c 是唯一有方向性的工具:源端 src_id 必须是 win,目标端 dst_id 可以是 " @@ -2197,7 +2198,7 @@ std::string BuildToolsListResult(const Json::Value& id) { { Json::Value tool(Json::objectValue); tool["name"] = "get_screenshot"; - tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。"; + tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。"; tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选 tool["outputSchema"] = BuildScreenshotOutputSchema(); @@ -3423,8 +3424,17 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C } if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG || hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) { + // Android 的抓屏依赖无障碍服务(takeScreenshot),失败时给一句可操作的提示 + // (kPlatApk 与 CurrentClientPlatform 定义在文件后部,这里直接看 RES_CLIENT_TYPE)。 + // 客户端把所有失败(未开无障碍、系统限流、编码/分配失败、回调超时)都归为 + // CAPTURE_FAILED,所以提示要列常见原因而不是断言某一条,细节引导去看客户端日志。 + std::string hint; + if (hdr->status == SCREEN_PREVIEW_CAPTURE_FAILED && ctx->GetAdditionalData(RES_CLIENT_TYPE) == "APK") + hint = ": Android client could not capture — check the YAMA accessibility service " + "(Android 11+), system screenshot rate limiting, and get_client_log for the " + "capture/encode error"; return BuildError(id, -32000, - "Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")"); + "Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")" + hint); } // 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。 @@ -9942,7 +9952,8 @@ const ToolPlatform kToolPlatforms[] = { { "download_file", kPlatWin|kPlatLnx|kPlatMac }, { "upload_file", kPlatWin|kPlatLnx|kPlatMac }, // ---- win+lnx ---- 远程控制全族 + 需 SCREEN_PREVIEW 的截屏 + macOS 未开放项 - { "get_screenshot", kPlatWin|kPlatLnx }, + // Android 用无障碍 takeScreenshot(API 30+,需开无障碍服务),只支持 mode=screen + { "get_screenshot", kPlatWin|kPlatLnx|kPlatApk }, // 窗口信息 RPC 只在 Windows 客户端实现(CLIENT_CAP_WINDOW_INFO) { "get_foreground_window", kPlatWin }, { "remote_open", kPlatWin|kPlatLnx }, diff --git a/tools/mcp_tools/make_tools_json.py b/tools/mcp_tools/make_tools_json.py index 61b069f..518564d 100644 --- a/tools/mcp_tools/make_tools_json.py +++ b/tools/mcp_tools/make_tools_json.py @@ -119,11 +119,15 @@ GATE = { # 支持面是产品定位的阶梯:win 100% > lnx 46% > mac 28% ≈ apk 16%。 # Windows 是唯一全量支持的平台,check_platform_table.py 会把这条做成不变式。 # -# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp:534-540 明确 -# 标注 not implemented,且客户端未声明 SCREEN_PREVIEW,故除服务端本地类外几乎全不支持。 +# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp 明确标注 not +# implemented,故除服务端本地类之外,只有少量按需补过分支的能力:运行日志 +# (COMMAND_QUERY_LOG)、活动历史(COMMAND_QUERY_ACTIVITY)、截屏 +# (COMMAND_SCREEN_PREVIEW_REQ,靠无障碍 takeScreenshot,Android 11+ 且开无障碍时 +# 登录才声明 SCREEN_PREVIEW 能力位)。 _APK = ["win", "lnx", "mac", "apk"] _TRIO = ["win", "lnx", "mac"] _WL = ["win", "lnx"] +_WLA = ["win", "lnx", "apk"] _WIN = ["win"] PLATFORMS = { @@ -139,10 +143,11 @@ PLATFORMS = { "host_set_remark": _APK, "host_set_watch_level": _APK, - # SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了 + # SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了;APK 也实现了 QUERY_LOG + # (android/.../main.cpp ClientLogThread → TOKEN_REPORT_LOG) "list_processes": _TRIO, "list_files": _TRIO, - "get_client_log": _TRIO, + "get_client_log": _APK, # list_windows 只有 Windows:服务端把 COMMAND_WSLIST 发在主连接上,Linux/macOS # 主循环只处理 COMMAND_SYSTEM,WSLIST 只在 SystemManager 子连接里处理 —— 走不到, @@ -170,15 +175,16 @@ PLATFORMS = { "remote_mouse": _WL, "remote_clipboard": _WL, - # 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明 - "get_screenshot": _WL, + # 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明, + # Android 仅在无障碍服务在跑时声明(takeScreenshot,只支持 mode=screen) + "get_screenshot": _WLA, # ---- 仅 Windows ---- "list_services": _WIN, "control_service": _WIN, "list_registry": _WIN, "get_keylog": _WIN, - "get_activity_history": _WL, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp) + "get_activity_history": _WLA, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp);APK 复用同一采样机 "get_foreground_window": _WIN, # 窗口信息 RPC:CLIENT_CAP_WINDOW_INFO,仅 Windows 客户端声明 "search_file": _WIN, "delete_path": _WIN,