Feature: Add screen preview to the Android client

get_screenshot and the server's online-list and hover thumbnails were
gated to Windows and X11 Linux. The Android client now answers
COMMAND_SCREEN_PREVIEW_REQ on the main connection with
TOKEN_SCREEN_PREVIEW_RSP plus a JPEG, and advertises
CLIENT_CAP_SCREEN_PREVIEW at login only when it can really capture.

Capture goes through AccessibilityService.takeScreenshot (API 30+),
enabled by canTakeScreenshot in the service config. MediaProjection is
deliberately not reused: its VirtualDisplay already feeds the H.264
encoder and Android 14+ refuses a second one from the same projection,
while accessibility capture runs alongside the live stream. The JPEG is
scaled to maxWidth and encoded in Kotlin; native only copies the bytes
and drops a request while one is in flight. Only mode=screen is
supported, and capabilities are re-evaluated on every login so turning
the service off withdraws the capability.

Also fix two review findings: a screenshot callback that lands after
the 3 s timeout now recycles its bitmap instead of leaking it, and the
server's failure hint lists the possible Android causes instead of
asserting disabled accessibility.

Verified on the emulator against a local debug server: get_screenshot
returns 640x1422 and 720x1600 JPEGs, the 120-wide list thumbnail works,
and both MCP tool-table checkers pass with apk added for get_screenshot,
get_activity_history and get_client_log.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-11 20:58:40 +01:00
1 parent 66acde0301
commit f29150d20e
6 files changed
+346 -16

No files matched your search

+171
View File
@@ -226,6 +226,13 @@ static jmethodID g_forceFirstFrameMid = nullptr;
static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String)
static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务
static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
static jclass g_shotClass = nullptr;
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
static jmethodID g_shotCaptureMid = nullptr; // ScreenShot.capture(II)Lcom/yama/client/ScreenShot$Shot;
static jmethodID g_shotDataMid = nullptr; // Shot.getData()[B
static jmethodID g_shotWidthMid = nullptr; // Shot.getWidth()I
static jmethodID g_shotHeightMid = nullptr; // Shot.getHeight()I
// 心跳日志限流:每 60 秒最多打一次
static std::atomic<uint64_t> g_lastHbLogMs{0};
@@ -598,6 +605,118 @@ static void ActivityReportThread()
g_activityReportsInFlight.fetch_sub(1, std::memory_order_relaxed);
}
// ------------------------------------------------------------------ 屏幕预览
// 对应 Windows/Linux 的 COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP。
// Android 的 JPEG 在 Kotlin 侧(ScreenShot,无障碍 takeScreenshot)编码好,这里只搬运;
// 与 Linux 回 BGRX 不同,服务端对 JPEG 直接透传,不再二次编码。
static std::atomic<int> g_previewInFlight(0);
// 抓一帧 JPEG:JNI 调 ScreenShot.capture(maxWidth, quality),把 Shot 的数据/宽高取回来。
// 失败(无障碍没开、系统限流、编码失败)返回 false,调用方回 CAPTURE_FAILED。
static bool CaptureScreenJpeg(uint16_t maxWidth, uint8_t quality,
std::vector<uint8_t>& jpeg, int& w, int& h)
{
jpeg.clear();
w = h = 0;
if (!g_jvm || !g_shotClass || !g_shotCaptureMid || !g_shotDataMid ||
!g_shotWidthMid || !g_shotHeightMid)
return false;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
attached = true;
} else if (st != JNI_OK || !jenv) {
return false;
}
bool ok = false;
jobject shot = jenv->CallStaticObjectMethod(g_shotClass, g_shotCaptureMid,
(jint)maxWidth, (jint)quality);
if (shot && !jenv->ExceptionCheck()) {
jbyteArray arr = (jbyteArray)jenv->CallObjectMethod(shot, g_shotDataMid);
if (!jenv->ExceptionCheck() && arr) {
jint bw = jenv->CallIntMethod(shot, g_shotWidthMid);
jint bh = jenv->CallIntMethod(shot, g_shotHeightMid);
if (!jenv->ExceptionCheck() && bw > 0 && bh > 0) {
jsize len = jenv->GetArrayLength(arr);
if (len > 0) {
jpeg.resize((size_t)len);
jenv->GetByteArrayRegion(arr, 0, len, (jbyte*)jpeg.data());
if (!jenv->ExceptionCheck()) {
w = bw;
h = bh;
ok = true;
}
}
}
jenv->DeleteLocalRef(arr);
}
jenv->DeleteLocalRef(shot);
}
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
if (attached) g_jvm->DetachCurrentThread();
if (!ok) jpeg.clear();
return ok;
}
// 登录前探测:无障碍服务在跑(API 30+)才声明 CLIENT_CAP_SCREEN_PREVIEW,
// 服务端据此放行 get_screenshot。能力位随每次登录刷新,无障碍被关掉重连后会撤回。
static bool ScreenPreviewAvailable()
{
if (!g_jvm || !g_shotClass || !g_shotCanMid) return false;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
attached = true;
} else if (st != JNI_OK || !jenv) {
return false;
}
jboolean can = jenv->CallStaticBooleanMethod(g_shotClass, g_shotCanMid);
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); can = JNI_FALSE; }
if (attached) g_jvm->DetachCurrentThread();
return can == JNI_TRUE;
}
// 抓屏可能耗几十~几百毫秒(系统回调 + 编码),必须离开接收线程;超额请求直接丢弃
// (服务端会超时按「预览不可用」降级,与 Linux/Windows 的限流一致)。
static void ScreenPreviewThread(ScreenPreviewReq req)
{
struct InFlightGuard {
~InFlightGuard() { g_previewInFlight.fetch_sub(1); }
} guard;
std::vector<uint8_t> jpeg;
int w = 0, h = 0;
uint8_t status = CaptureScreenJpeg(req.maxWidth, req.jpegQuality, jpeg, w, h)
? (uint8_t)SCREEN_PREVIEW_OK
: (uint8_t)SCREEN_PREVIEW_CAPTURE_FAILED;
uint32_t bytes = (status == SCREEN_PREVIEW_OK) ? (uint32_t)jpeg.size() : 0;
std::vector<uint8_t> pkt(sizeof(ScreenPreviewRspHeader) + bytes, 0);
ScreenPreviewRspHeader hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.token = TOKEN_SCREEN_PREVIEW_RSP;
hdr.reqId = req.reqId;
hdr.status = status;
hdr.format = SCREEN_PREVIEW_FMT_JPEG; // 失败时 format 无意义,填 JPEG 保持包头合法
hdr.width = (uint16_t)w;
hdr.height = (uint16_t)h;
hdr.bytes = bytes;
memcpy(pkt.data(), &hdr, sizeof(hdr));
if (bytes) memcpy(pkt.data() + sizeof(hdr), jpeg.data(), bytes);
// 与 ActivityReportThread 同一套路:全局指针只取一次快照
IOCPClient* mc = g_pMainClient;
if (mc && mc->IsConnected() && g_bExit == S_CLIENT_NORMAL)
mc->Send2Server((char*)pkt.data(), (ULONG)pkt.size());
LOGI("[Preview] reqId=%u %dx%d %u bytes status=%u", req.reqId, w, h, bytes, status);
}
// ------------------------------------------------------------------ 在线升级
// 载荷布局:[COMMAND_UPDATE:1][APK 大小:8 LE][APK:size][xxh64:8 LE]
// 校验通过后写到 filesDir/update.apk,交给 Kotlin 的 PackageInstaller 会话去装
@@ -851,6 +970,27 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
}
break;
case COMMAND_SCREEN_PREVIEW_REQ: {
// MCP get_screenshot / 在线列表缩略图:主连接回一次 JPEG
if (ulLength < sizeof(ScreenPreviewReq)) break;
ScreenPreviewReq req;
memcpy(&req, szBuffer, sizeof(req));
if (g_previewInFlight.fetch_add(1) >= 1) {
g_previewInFlight.fetch_sub(1); // 已有抓屏在跑,重复请求丢弃
break;
}
LOGI("cmd: COMMAND_SCREEN_PREVIEW_REQ reqId=%u maxWidth=%u quality=%u",
req.reqId, req.maxWidth, req.jpegQuality);
try {
std::thread(ScreenPreviewThread, req).detach();
} catch (const std::exception& e) {
// 线程起不来要把在途计数还回去,否则之后所有预览请求都被当成「忙」丢掉
g_previewInFlight.fetch_sub(1);
LOGE("Screen preview thread failed: %s", e.what());
}
break;
}
case COMMAND_SHELL:
LOGI("COMMAND_SHELL (not implemented)");
break;
@@ -999,6 +1139,10 @@ static void ConnectionThread(uint64_t gen)
#ifdef YAMA_TLS_CHANNEL
RefreshLoginTls(logInfo); // 传输方式可能已热切换,登录前刷新 RES_TLS
#endif
// 屏幕预览能力位按运行时状态刷新:无障碍服务在跑才声明(API 30+ 的 takeScreenshot),
// 服务端据此放行 get_screenshot;无障碍被关掉后重连会撤回,避免服务端白等抓屏
logInfo.SetCaps(LOGIN_INFOR::DefaultCaps() |
(ScreenPreviewAvailable() ? CLIENT_CAP_SCREEN_PREVIEW : 0));
client->SendLoginInfo(logInfo.Speed(clock() - c));
g_lastHeartbeatAckMs.store(GetUnixMs(), std::memory_order_relaxed);
LOGI("Connected & login sent");
@@ -1191,6 +1335,33 @@ Java_com_yama_client_YamaBridge_nativeInit(
} else {
LOGE("CaptureService class not found");
}
// 屏幕预览:返回对象是 ScreenShot$Shot,JNI 签名用 '$' 写嵌套类
jclass shotCls = env->FindClass("com/yama/client/ScreenShot");
if (env->ExceptionCheck()) { env->ExceptionClear(); shotCls = nullptr; }
if (shotCls) {
g_shotClass = (jclass)env->NewGlobalRef(shotCls);
g_shotCanMid = env->GetStaticMethodID(g_shotClass, "canCapture", "()Z");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCanMid = nullptr; }
if (!g_shotCanMid) LOGE("ScreenShot.canCapture not found");
g_shotCaptureMid = env->GetStaticMethodID(g_shotClass, "capture",
"(II)Lcom/yama/client/ScreenShot$Shot;");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCaptureMid = nullptr; }
if (!g_shotCaptureMid) LOGE("ScreenShot.capture not found");
jclass shotResultCls = env->FindClass("com/yama/client/ScreenShot$Shot");
if (env->ExceptionCheck()) { env->ExceptionClear(); shotResultCls = nullptr; }
if (shotResultCls) {
g_shotDataMid = env->GetMethodID(shotResultCls, "getData", "()[B");
g_shotWidthMid = env->GetMethodID(shotResultCls, "getWidth", "()I");
g_shotHeightMid = env->GetMethodID(shotResultCls, "getHeight", "()I");
if (env->ExceptionCheck()) { env->ExceptionClear(); }
if (!g_shotDataMid || !g_shotWidthMid || !g_shotHeightMid)
LOGE("ScreenShot$Shot getters not found");
} else {
LOGE("ScreenShot$Shot class not found");
}
} else {
LOGE("ScreenShot class not found");
}
}
}