Feature: Add screen preview to the Android client
get_screenshot and the server's online-list and hover thumbnails were gated to Windows and X11 Linux. The Android client now answers COMMAND_SCREEN_PREVIEW_REQ on the main connection with TOKEN_SCREEN_PREVIEW_RSP plus a JPEG, and advertises CLIENT_CAP_SCREEN_PREVIEW at login only when it can really capture. Capture goes through AccessibilityService.takeScreenshot (API 30+), enabled by canTakeScreenshot in the service config. MediaProjection is deliberately not reused: its VirtualDisplay already feeds the H.264 encoder and Android 14+ refuses a second one from the same projection, while accessibility capture runs alongside the live stream. The JPEG is scaled to maxWidth and encoded in Kotlin; native only copies the bytes and drops a request while one is in flight. Only mode=screen is supported, and capabilities are re-evaluated on every login so turning the service off withdraws the capability. Also fix two review findings: a screenshot callback that lands after the 3 s timeout now recycles its bitmap instead of leaking it, and the server's failure hint lists the possible Android causes instead of asserting disabled accessibility. Verified on the emulator against a local debug server: get_screenshot returns 640x1422 and 720x1600 JPEGs, the 120-wide list thumbnail works, and both MCP tool-table checkers pass with apk added for get_screenshot, get_activity_history and get_client_log. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
66acde0301
commit
f29150d20e
6 files changed
+346
-16
No files matched your search
@@ -226,6 +226,13 @@ static jmethodID g_forceFirstFrameMid = nullptr;
|
||||
static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String)
|
||||
static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务
|
||||
static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘
|
||||
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
|
||||
static jclass g_shotClass = nullptr;
|
||||
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
|
||||
static jmethodID g_shotCaptureMid = nullptr; // ScreenShot.capture(II)Lcom/yama/client/ScreenShot$Shot;
|
||||
static jmethodID g_shotDataMid = nullptr; // Shot.getData()[B
|
||||
static jmethodID g_shotWidthMid = nullptr; // Shot.getWidth()I
|
||||
static jmethodID g_shotHeightMid = nullptr; // Shot.getHeight()I
|
||||
|
||||
// 心跳日志限流:每 60 秒最多打一次
|
||||
static std::atomic<uint64_t> g_lastHbLogMs{0};
|
||||
@@ -598,6 +605,118 @@ static void ActivityReportThread()
|
||||
g_activityReportsInFlight.fetch_sub(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 屏幕预览
|
||||
// 对应 Windows/Linux 的 COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP。
|
||||
// Android 的 JPEG 在 Kotlin 侧(ScreenShot,无障碍 takeScreenshot)编码好,这里只搬运;
|
||||
// 与 Linux 回 BGRX 不同,服务端对 JPEG 直接透传,不再二次编码。
|
||||
static std::atomic<int> g_previewInFlight(0);
|
||||
|
||||
// 抓一帧 JPEG:JNI 调 ScreenShot.capture(maxWidth, quality),把 Shot 的数据/宽高取回来。
|
||||
// 失败(无障碍没开、系统限流、编码失败)返回 false,调用方回 CAPTURE_FAILED。
|
||||
static bool CaptureScreenJpeg(uint16_t maxWidth, uint8_t quality,
|
||||
std::vector<uint8_t>& jpeg, int& w, int& h)
|
||||
{
|
||||
jpeg.clear();
|
||||
w = h = 0;
|
||||
if (!g_jvm || !g_shotClass || !g_shotCaptureMid || !g_shotDataMid ||
|
||||
!g_shotWidthMid || !g_shotHeightMid)
|
||||
return false;
|
||||
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ok = false;
|
||||
jobject shot = jenv->CallStaticObjectMethod(g_shotClass, g_shotCaptureMid,
|
||||
(jint)maxWidth, (jint)quality);
|
||||
if (shot && !jenv->ExceptionCheck()) {
|
||||
jbyteArray arr = (jbyteArray)jenv->CallObjectMethod(shot, g_shotDataMid);
|
||||
if (!jenv->ExceptionCheck() && arr) {
|
||||
jint bw = jenv->CallIntMethod(shot, g_shotWidthMid);
|
||||
jint bh = jenv->CallIntMethod(shot, g_shotHeightMid);
|
||||
if (!jenv->ExceptionCheck() && bw > 0 && bh > 0) {
|
||||
jsize len = jenv->GetArrayLength(arr);
|
||||
if (len > 0) {
|
||||
jpeg.resize((size_t)len);
|
||||
jenv->GetByteArrayRegion(arr, 0, len, (jbyte*)jpeg.data());
|
||||
if (!jenv->ExceptionCheck()) {
|
||||
w = bw;
|
||||
h = bh;
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
jenv->DeleteLocalRef(arr);
|
||||
}
|
||||
jenv->DeleteLocalRef(shot);
|
||||
}
|
||||
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
if (!ok) jpeg.clear();
|
||||
return ok;
|
||||
}
|
||||
|
||||
// 登录前探测:无障碍服务在跑(API 30+)才声明 CLIENT_CAP_SCREEN_PREVIEW,
|
||||
// 服务端据此放行 get_screenshot。能力位随每次登录刷新,无障碍被关掉重连后会撤回。
|
||||
static bool ScreenPreviewAvailable()
|
||||
{
|
||||
if (!g_jvm || !g_shotClass || !g_shotCanMid) return false;
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return false;
|
||||
}
|
||||
jboolean can = jenv->CallStaticBooleanMethod(g_shotClass, g_shotCanMid);
|
||||
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); can = JNI_FALSE; }
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
return can == JNI_TRUE;
|
||||
}
|
||||
|
||||
// 抓屏可能耗几十~几百毫秒(系统回调 + 编码),必须离开接收线程;超额请求直接丢弃
|
||||
// (服务端会超时按「预览不可用」降级,与 Linux/Windows 的限流一致)。
|
||||
static void ScreenPreviewThread(ScreenPreviewReq req)
|
||||
{
|
||||
struct InFlightGuard {
|
||||
~InFlightGuard() { g_previewInFlight.fetch_sub(1); }
|
||||
} guard;
|
||||
|
||||
std::vector<uint8_t> jpeg;
|
||||
int w = 0, h = 0;
|
||||
uint8_t status = CaptureScreenJpeg(req.maxWidth, req.jpegQuality, jpeg, w, h)
|
||||
? (uint8_t)SCREEN_PREVIEW_OK
|
||||
: (uint8_t)SCREEN_PREVIEW_CAPTURE_FAILED;
|
||||
|
||||
uint32_t bytes = (status == SCREEN_PREVIEW_OK) ? (uint32_t)jpeg.size() : 0;
|
||||
std::vector<uint8_t> pkt(sizeof(ScreenPreviewRspHeader) + bytes, 0);
|
||||
ScreenPreviewRspHeader hdr;
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
hdr.token = TOKEN_SCREEN_PREVIEW_RSP;
|
||||
hdr.reqId = req.reqId;
|
||||
hdr.status = status;
|
||||
hdr.format = SCREEN_PREVIEW_FMT_JPEG; // 失败时 format 无意义,填 JPEG 保持包头合法
|
||||
hdr.width = (uint16_t)w;
|
||||
hdr.height = (uint16_t)h;
|
||||
hdr.bytes = bytes;
|
||||
memcpy(pkt.data(), &hdr, sizeof(hdr));
|
||||
if (bytes) memcpy(pkt.data() + sizeof(hdr), jpeg.data(), bytes);
|
||||
|
||||
// 与 ActivityReportThread 同一套路:全局指针只取一次快照
|
||||
IOCPClient* mc = g_pMainClient;
|
||||
if (mc && mc->IsConnected() && g_bExit == S_CLIENT_NORMAL)
|
||||
mc->Send2Server((char*)pkt.data(), (ULONG)pkt.size());
|
||||
LOGI("[Preview] reqId=%u %dx%d %u bytes status=%u", req.reqId, w, h, bytes, status);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 在线升级
|
||||
// 载荷布局:[COMMAND_UPDATE:1][APK 大小:8 LE][APK:size][xxh64:8 LE]
|
||||
// 校验通过后写到 filesDir/update.apk,交给 Kotlin 的 PackageInstaller 会话去装
|
||||
@@ -851,6 +970,27 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
|
||||
}
|
||||
break;
|
||||
|
||||
case COMMAND_SCREEN_PREVIEW_REQ: {
|
||||
// MCP get_screenshot / 在线列表缩略图:主连接回一次 JPEG
|
||||
if (ulLength < sizeof(ScreenPreviewReq)) break;
|
||||
ScreenPreviewReq req;
|
||||
memcpy(&req, szBuffer, sizeof(req));
|
||||
if (g_previewInFlight.fetch_add(1) >= 1) {
|
||||
g_previewInFlight.fetch_sub(1); // 已有抓屏在跑,重复请求丢弃
|
||||
break;
|
||||
}
|
||||
LOGI("cmd: COMMAND_SCREEN_PREVIEW_REQ reqId=%u maxWidth=%u quality=%u",
|
||||
req.reqId, req.maxWidth, req.jpegQuality);
|
||||
try {
|
||||
std::thread(ScreenPreviewThread, req).detach();
|
||||
} catch (const std::exception& e) {
|
||||
// 线程起不来要把在途计数还回去,否则之后所有预览请求都被当成「忙」丢掉
|
||||
g_previewInFlight.fetch_sub(1);
|
||||
LOGE("Screen preview thread failed: %s", e.what());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_SHELL:
|
||||
LOGI("COMMAND_SHELL (not implemented)");
|
||||
break;
|
||||
@@ -999,6 +1139,10 @@ static void ConnectionThread(uint64_t gen)
|
||||
#ifdef YAMA_TLS_CHANNEL
|
||||
RefreshLoginTls(logInfo); // 传输方式可能已热切换,登录前刷新 RES_TLS
|
||||
#endif
|
||||
// 屏幕预览能力位按运行时状态刷新:无障碍服务在跑才声明(API 30+ 的 takeScreenshot),
|
||||
// 服务端据此放行 get_screenshot;无障碍被关掉后重连会撤回,避免服务端白等抓屏
|
||||
logInfo.SetCaps(LOGIN_INFOR::DefaultCaps() |
|
||||
(ScreenPreviewAvailable() ? CLIENT_CAP_SCREEN_PREVIEW : 0));
|
||||
client->SendLoginInfo(logInfo.Speed(clock() - c));
|
||||
g_lastHeartbeatAckMs.store(GetUnixMs(), std::memory_order_relaxed);
|
||||
LOGI("Connected & login sent");
|
||||
@@ -1191,6 +1335,33 @@ Java_com_yama_client_YamaBridge_nativeInit(
|
||||
} else {
|
||||
LOGE("CaptureService class not found");
|
||||
}
|
||||
// 屏幕预览:返回对象是 ScreenShot$Shot,JNI 签名用 '$' 写嵌套类
|
||||
jclass shotCls = env->FindClass("com/yama/client/ScreenShot");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); shotCls = nullptr; }
|
||||
if (shotCls) {
|
||||
g_shotClass = (jclass)env->NewGlobalRef(shotCls);
|
||||
g_shotCanMid = env->GetStaticMethodID(g_shotClass, "canCapture", "()Z");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCanMid = nullptr; }
|
||||
if (!g_shotCanMid) LOGE("ScreenShot.canCapture not found");
|
||||
g_shotCaptureMid = env->GetStaticMethodID(g_shotClass, "capture",
|
||||
"(II)Lcom/yama/client/ScreenShot$Shot;");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCaptureMid = nullptr; }
|
||||
if (!g_shotCaptureMid) LOGE("ScreenShot.capture not found");
|
||||
jclass shotResultCls = env->FindClass("com/yama/client/ScreenShot$Shot");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); shotResultCls = nullptr; }
|
||||
if (shotResultCls) {
|
||||
g_shotDataMid = env->GetMethodID(shotResultCls, "getData", "()[B");
|
||||
g_shotWidthMid = env->GetMethodID(shotResultCls, "getWidth", "()I");
|
||||
g_shotHeightMid = env->GetMethodID(shotResultCls, "getHeight", "()I");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); }
|
||||
if (!g_shotDataMid || !g_shotWidthMid || !g_shotHeightMid)
|
||||
LOGE("ScreenShot$Shot getters not found");
|
||||
} else {
|
||||
LOGE("ScreenShot$Shot class not found");
|
||||
}
|
||||
} else {
|
||||
LOGE("ScreenShot class not found");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.accessibilityservice.AccessibilityService
|
||||
import android.graphics.Bitmap
|
||||
import android.os.Build
|
||||
import android.util.Log
|
||||
import android.view.Display
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.Executor
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
/**
|
||||
* 屏幕预览(MCP get_screenshot)的 Android 实现:借无障碍服务的
|
||||
* [AccessibilityService.takeScreenshot](API 30+)抓一帧,等比缩到 maxWidth 以内再编码 JPEG。
|
||||
*
|
||||
* 刻意不走 MediaProjection:投屏的 VirtualDisplay 已经被 H.264 编码器占着,Android 14+
|
||||
* 同一个投影再建第二个 VirtualDisplay 会失败;无障碍截屏与实时投屏互不干扰,用户拒绝投屏
|
||||
* 授权时也照样能用。代价是 Android 11 以下没有这条路(不声明能力位,服务端直接拒绝)。
|
||||
*/
|
||||
object ScreenShot {
|
||||
private const val TAG = "YAMA"
|
||||
private const val CAPTURE_TIMEOUT_MS = 3000L
|
||||
private const val MAX_WIDTH = 1920
|
||||
|
||||
/** 抓屏结果:JPEG 数据 + 实际编码后的宽高(服务端据此回报尺寸/换算比例) */
|
||||
class Shot(val data: ByteArray, val width: Int, val height: Int)
|
||||
|
||||
/**
|
||||
* 当前能否抓屏:API 30+ 且无障碍服务在跑。takeScreenshot 的能力来自服务配置
|
||||
* (accessibility_service_config.xml 的 canTakeScreenshot),服务没起来就调不了。
|
||||
*/
|
||||
@JvmStatic
|
||||
fun canCapture(): Boolean {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return false
|
||||
return ControlService.instance != null
|
||||
}
|
||||
|
||||
/**
|
||||
* 同步抓一帧并编码 JPEG。从 native 工作线程调用(调用方负责 attach JVM),
|
||||
* 内部用 CountDownLatch 等系统回调。拿不到画面/超时/编码失败一律返回 null,
|
||||
* native 侧据此回 SCREEN_PREVIEW_CAPTURE_FAILED。
|
||||
*/
|
||||
@JvmStatic
|
||||
fun capture(maxWidth: Int, quality: Int): Shot? {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null
|
||||
val svc = ControlService.instance ?: return null
|
||||
|
||||
val latch = CountDownLatch(1)
|
||||
val holder = arrayOfNulls<Bitmap>(1)
|
||||
// 超时返回后回调仍可能到达(系统限流时回调会拖到几秒后)。等待方已经放弃,
|
||||
// 那张整屏 ARGB_8888(1080x2400 约 10 MB)得有人回收。holder 与 abandoned
|
||||
// 必须在同一把锁下读写:否则回调可能在 countDown 之后、等待方置标志之前
|
||||
// 看到 false,把位图留在 holder 里,而那边已经超时返回,位图就永久泄漏。
|
||||
val gate = Any()
|
||||
val abandoned = AtomicBoolean(false)
|
||||
// 回调直接在当前线程跑,省掉再起一个线程;Binder 线程回调也安全
|
||||
val direct = Executor { it.run() }
|
||||
try {
|
||||
svc.takeScreenshot(Display.DEFAULT_DISPLAY, direct,
|
||||
object : AccessibilityService.TakeScreenshotCallback {
|
||||
override fun onSuccess(result: AccessibilityService.ScreenshotResult) {
|
||||
val hw = result.hardwareBuffer
|
||||
var bmp: Bitmap? = null
|
||||
try {
|
||||
// 硬件位图引用着 hw,必须先复制成软件位图再关
|
||||
val hb = Bitmap.wrapHardwareBuffer(hw, result.colorSpace)
|
||||
bmp = hb?.copy(Bitmap.Config.ARGB_8888, false)
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "takeScreenshot convert failed: $e")
|
||||
} finally {
|
||||
hw.close()
|
||||
}
|
||||
synchronized(gate) {
|
||||
holder[0] = bmp
|
||||
latch.countDown()
|
||||
// 等待方已超时返回:这张位图归回调自己回收
|
||||
if (abandoned.get()) {
|
||||
holder[0]?.recycle()
|
||||
holder[0] = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun onFailure(errorCode: Int) {
|
||||
// ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT 等:系统限流/未授权
|
||||
Log.e(TAG, "takeScreenshot failed: errorCode=$errorCode")
|
||||
latch.countDown()
|
||||
}
|
||||
})
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "takeScreenshot threw: $e")
|
||||
return null
|
||||
}
|
||||
|
||||
if (!latch.await(CAPTURE_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
// 与回调抢同一把锁:回调若已把位图放进 holder 并检查过标志(那时还是 false),
|
||||
// 这里负责回收;回调若还没进锁,它随后会看到标志并自己回收。
|
||||
synchronized(gate) {
|
||||
abandoned.set(true)
|
||||
val late = holder[0]
|
||||
holder[0] = null
|
||||
late?.recycle()
|
||||
}
|
||||
Log.e(TAG, "takeScreenshot timeout")
|
||||
return null
|
||||
}
|
||||
val src = holder[0] ?: return null
|
||||
return try {
|
||||
encodeJpeg(src, maxWidth, quality)
|
||||
} finally {
|
||||
src.recycle()
|
||||
}
|
||||
}
|
||||
|
||||
/** 等比缩放到 maxWidth 以内(maxWidth<=0 表示不缩放),再压成 JPEG */
|
||||
private fun encodeJpeg(src: Bitmap, maxWidth: Int, quality: Int): Shot? {
|
||||
val limit = if (maxWidth in 1..MAX_WIDTH) maxWidth else 0
|
||||
val scaled = if (limit > 0 && src.width > limit) {
|
||||
val h = (src.height.toLong() * limit / src.width).toInt().coerceAtLeast(1)
|
||||
Bitmap.createScaledBitmap(src, limit, h, true)
|
||||
} else src
|
||||
try {
|
||||
val out = ByteArrayOutputStream()
|
||||
if (!scaled.compress(Bitmap.CompressFormat.JPEG, quality.coerceIn(10, 100), out)) return null
|
||||
val data = out.toByteArray()
|
||||
if (data.isEmpty()) return null
|
||||
Log.i(TAG, "screenshot ${src.width}x${src.height} → ${scaled.width}x${scaled.height}, ${data.size} bytes")
|
||||
return Shot(data, scaled.width, scaled.height)
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "JPEG encode failed: $e")
|
||||
return null
|
||||
} finally {
|
||||
if (scaled !== src) scaled.recycle()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,5 +4,6 @@
|
||||
android:accessibilityFeedbackType="feedbackGeneric"
|
||||
android:canPerformGestures="true"
|
||||
android:canRetrieveWindowContent="true"
|
||||
android:canTakeScreenshot="true"
|
||||
android:notificationTimeout="0"
|
||||
android:settingsActivity="com.yama.client.MainActivity" />
|
||||
+8
-5
@@ -906,11 +906,12 @@
|
||||
},
|
||||
{
|
||||
"name": "get_activity_history",
|
||||
"description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。",
|
||||
"description": "获取指定在线主机的历史活动记录。Windows 与有桌面的 Linux 客户端返回前台窗口驻留记录,每行一条「[时间] [标题] 时长」;Android 客户端返回前台应用驻留记录(标题为「应用名 (包名)」,需开启无障碍服务);无桌面的 Linux 客户端返回登录记录与 shell 命令历史,每行带「[登录]」或「[命令]」标签(首行为「[来源]」说明)。主连接 RPC。结果按 limit/offset 分页(默认 100 条、上限 1000),新在前:offset=0 就是最近 limit 条,想看更早的再往后翻。total 为本机现有记录总数,hasMore 表示还有更早的记录;窗口后端只保留最近 500 条,所以 total 最多约 501(含进行中的当前窗口)。",
|
||||
"gate": null,
|
||||
"platforms": [
|
||||
"win",
|
||||
"lnx"
|
||||
"lnx",
|
||||
"apk"
|
||||
],
|
||||
"impl": "mfc",
|
||||
"inputSchema": {
|
||||
@@ -1048,11 +1049,12 @@
|
||||
},
|
||||
{
|
||||
"name": "get_screenshot",
|
||||
"description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。",
|
||||
"description": "截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。",
|
||||
"gate": null,
|
||||
"platforms": [
|
||||
"win",
|
||||
"lnx"
|
||||
"lnx",
|
||||
"apk"
|
||||
],
|
||||
"impl": "mfc",
|
||||
"inputSchema": {
|
||||
@@ -1580,7 +1582,8 @@
|
||||
"platforms": [
|
||||
"win",
|
||||
"lnx",
|
||||
"mac"
|
||||
"mac",
|
||||
"apk"
|
||||
],
|
||||
"impl": "mfc",
|
||||
"inputSchema": {
|
||||
|
||||
@@ -345,7 +345,8 @@ const char* kServerInstructions =
|
||||
u8"文件传输、进程列表、日志可用。\n"
|
||||
u8"apk(APK):支持主机列表(list_online_hosts / list_all_hosts)/ 详情 / 搜索 / "
|
||||
u8"审计日志 / 服务状态 / 会话管理、运行日志(get_client_log)、活动历史"
|
||||
u8"(get_activity_history,前台应用停留记录),以及 "
|
||||
u8"(get_activity_history,前台应用停留记录)、截屏(get_screenshot,只支持 mode=screen,"
|
||||
u8"要靠无障碍服务 takeScreenshot,Android 11+ 且开启无障碍才行),以及 "
|
||||
u8"host_set_remark / host_set_watch_level 这两个只改服务端本地数据的工具。其余一律不支持"
|
||||
u8"(客户端未实现 SYSTEM 与 SHELL)。\n"
|
||||
u8"copy_files_c2c 是唯一有方向性的工具:源端 src_id 必须是 win,目标端 dst_id 可以是 "
|
||||
@@ -2197,7 +2198,7 @@ std::string BuildToolsListResult(const Json::Value& id) {
|
||||
{
|
||||
Json::Value tool(Json::objectValue);
|
||||
tool["name"] = "get_screenshot";
|
||||
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
|
||||
tool["description"] = u8"截取指定在线主机的一帧画面,返回 base64 编码的 JPEG 图片(image 对象含 mimeType/data/width/height)。支持 Windows 与带 X11 桌面的 Linux 客户端(能力位 CLIENT_CAP_SCREEN_PREVIEW),主连接 RPC、一次性返回。Android 客户端用无障碍服务 takeScreenshot 抓屏(Android 11+ 且开启无障碍服务,此时登录会声明该能力位;只支持 mode=screen)。mode 可选 screen(默认,整虚拟桌面)/region(指定矩形)/window(指定 hwnd)/monitor(指定显示器);后三种默认按源尺寸 1:1 原生分辨率抓取(适合 OCR/视觉读字),需要较新的 Windows 客户端(CLIENT_CAP_SCREEN_REGION,否则 -32005),结果在 structuredContent.capture 里回报实际源矩形与缩放比例。";
|
||||
|
||||
tool["inputSchema"] = BuildGetScreenshotInputSchema(); // { id } 必填 + max_width 可选
|
||||
tool["outputSchema"] = BuildScreenshotOutputSchema();
|
||||
@@ -3423,8 +3424,17 @@ std::string BuildGetScreenshot(const Json::Value& id, const Json::Value& args, C
|
||||
}
|
||||
if (hdr->status != SCREEN_PREVIEW_OK || hdr->format != SCREEN_PREVIEW_FMT_JPEG ||
|
||||
hdr->bytes == 0 || data.size() < sizeof(ScreenPreviewRspHeader) + hdr->bytes) {
|
||||
// Android 的抓屏依赖无障碍服务(takeScreenshot),失败时给一句可操作的提示
|
||||
// (kPlatApk 与 CurrentClientPlatform 定义在文件后部,这里直接看 RES_CLIENT_TYPE)。
|
||||
// 客户端把所有失败(未开无障碍、系统限流、编码/分配失败、回调超时)都归为
|
||||
// CAPTURE_FAILED,所以提示要列常见原因而不是断言某一条,细节引导去看客户端日志。
|
||||
std::string hint;
|
||||
if (hdr->status == SCREEN_PREVIEW_CAPTURE_FAILED && ctx->GetAdditionalData(RES_CLIENT_TYPE) == "APK")
|
||||
hint = ": Android client could not capture — check the YAMA accessibility service "
|
||||
"(Android 11+), system screenshot rate limiting, and get_client_log for the "
|
||||
"capture/encode error";
|
||||
return BuildError(id, -32000,
|
||||
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")");
|
||||
"Screenshot capture failed (status " + std::to_string((int)hdr->status) + ")" + hint);
|
||||
}
|
||||
|
||||
// 可选尾块:实际抓取的源矩形 + 缩放比例(老客户端不回)。
|
||||
@@ -9942,7 +9952,8 @@ const ToolPlatform kToolPlatforms[] = {
|
||||
{ "download_file", kPlatWin|kPlatLnx|kPlatMac },
|
||||
{ "upload_file", kPlatWin|kPlatLnx|kPlatMac },
|
||||
// ---- win+lnx ---- 远程控制全族 + 需 SCREEN_PREVIEW 的截屏 + macOS 未开放项
|
||||
{ "get_screenshot", kPlatWin|kPlatLnx },
|
||||
// Android 用无障碍 takeScreenshot(API 30+,需开无障碍服务),只支持 mode=screen
|
||||
{ "get_screenshot", kPlatWin|kPlatLnx|kPlatApk },
|
||||
// 窗口信息 RPC 只在 Windows 客户端实现(CLIENT_CAP_WINDOW_INFO)
|
||||
{ "get_foreground_window", kPlatWin },
|
||||
{ "remote_open", kPlatWin|kPlatLnx },
|
||||
|
||||
@@ -119,11 +119,15 @@ GATE = {
|
||||
# 支持面是产品定位的阶梯:win 100% > lnx 46% > mac 28% ≈ apk 16%。
|
||||
# Windows 是唯一全量支持的平台,check_platform_table.py 会把这条做成不变式。
|
||||
#
|
||||
# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp:534-540 明确
|
||||
# 标注 not implemented,且客户端未声明 SCREEN_PREVIEW,故除服务端本地类外几乎全不支持。
|
||||
# apk 现状:COMMAND_SYSTEM 与 COMMAND_SHELL 在 android/.../main.cpp 明确标注 not
|
||||
# implemented,故除服务端本地类之外,只有少量按需补过分支的能力:运行日志
|
||||
# (COMMAND_QUERY_LOG)、活动历史(COMMAND_QUERY_ACTIVITY)、截屏
|
||||
# (COMMAND_SCREEN_PREVIEW_REQ,靠无障碍 takeScreenshot,Android 11+ 且开无障碍时
|
||||
# 登录才声明 SCREEN_PREVIEW 能力位)。
|
||||
_APK = ["win", "lnx", "mac", "apk"]
|
||||
_TRIO = ["win", "lnx", "mac"]
|
||||
_WL = ["win", "lnx"]
|
||||
_WLA = ["win", "lnx", "apk"]
|
||||
_WIN = ["win"]
|
||||
|
||||
PLATFORMS = {
|
||||
@@ -139,10 +143,11 @@ PLATFORMS = {
|
||||
"host_set_remark": _APK,
|
||||
"host_set_watch_level": _APK,
|
||||
|
||||
# SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了
|
||||
# SYSTEM / LIST_DRIVE / QUERY_LOG:三个桌面平台都实现了;APK 也实现了 QUERY_LOG
|
||||
# (android/.../main.cpp ClientLogThread → TOKEN_REPORT_LOG)
|
||||
"list_processes": _TRIO,
|
||||
"list_files": _TRIO,
|
||||
"get_client_log": _TRIO,
|
||||
"get_client_log": _APK,
|
||||
|
||||
# list_windows 只有 Windows:服务端把 COMMAND_WSLIST 发在主连接上,Linux/macOS
|
||||
# 主循环只处理 COMMAND_SYSTEM,WSLIST 只在 SystemManager 子连接里处理 —— 走不到,
|
||||
@@ -170,15 +175,16 @@ PLATFORMS = {
|
||||
"remote_mouse": _WL,
|
||||
"remote_clipboard": _WL,
|
||||
|
||||
# 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明
|
||||
"get_screenshot": _WL,
|
||||
# 需要 CLIENT_CAP_SCREEN_PREVIEW:Windows 恒有,Linux 仅在探测到 X11 桌面时声明,
|
||||
# Android 仅在无障碍服务在跑时声明(takeScreenshot,只支持 mode=screen)
|
||||
"get_screenshot": _WLA,
|
||||
|
||||
# ---- 仅 Windows ----
|
||||
"list_services": _WIN,
|
||||
"control_service": _WIN,
|
||||
"list_registry": _WIN,
|
||||
"get_keylog": _WIN,
|
||||
"get_activity_history": _WL, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp)
|
||||
"get_activity_history": _WLA, # Windows 前台窗口;Linux 桌面/无桌面均有实现(linux/main.cpp);APK 复用同一采样机
|
||||
"get_foreground_window": _WIN, # 窗口信息 RPC:CLIENT_CAP_WINDOW_INFO,仅 Windows 客户端声明
|
||||
"search_file": _WIN,
|
||||
"delete_path": _WIN,
|
||||
|
||||
Reference in new issue
Block a user