Feature: Add screen preview to the Android client
get_screenshot and the server's online-list and hover thumbnails were gated to Windows and X11 Linux. The Android client now answers COMMAND_SCREEN_PREVIEW_REQ on the main connection with TOKEN_SCREEN_PREVIEW_RSP plus a JPEG, and advertises CLIENT_CAP_SCREEN_PREVIEW at login only when it can really capture. Capture goes through AccessibilityService.takeScreenshot (API 30+), enabled by canTakeScreenshot in the service config. MediaProjection is deliberately not reused: its VirtualDisplay already feeds the H.264 encoder and Android 14+ refuses a second one from the same projection, while accessibility capture runs alongside the live stream. The JPEG is scaled to maxWidth and encoded in Kotlin; native only copies the bytes and drops a request while one is in flight. Only mode=screen is supported, and capabilities are re-evaluated on every login so turning the service off withdraws the capability. Also fix two review findings: a screenshot callback that lands after the 3 s timeout now recycles its bitmap instead of leaking it, and the server's failure hint lists the possible Android causes instead of asserting disabled accessibility. Verified on the emulator against a local debug server: get_screenshot returns 640x1422 and 720x1600 JPEGs, the 120-wide list thumbnail works, and both MCP tool-table checkers pass with apk added for get_screenshot, get_activity_history and get_client_log. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
66acde0301
commit
f29150d20e
6 files changed
+346
-16
No files matched your search
@@ -226,6 +226,13 @@ static jmethodID g_forceFirstFrameMid = nullptr;
|
||||
static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String)
|
||||
static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务
|
||||
static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘
|
||||
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
|
||||
static jclass g_shotClass = nullptr;
|
||||
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
|
||||
static jmethodID g_shotCaptureMid = nullptr; // ScreenShot.capture(II)Lcom/yama/client/ScreenShot$Shot;
|
||||
static jmethodID g_shotDataMid = nullptr; // Shot.getData()[B
|
||||
static jmethodID g_shotWidthMid = nullptr; // Shot.getWidth()I
|
||||
static jmethodID g_shotHeightMid = nullptr; // Shot.getHeight()I
|
||||
|
||||
// 心跳日志限流:每 60 秒最多打一次
|
||||
static std::atomic<uint64_t> g_lastHbLogMs{0};
|
||||
@@ -598,6 +605,118 @@ static void ActivityReportThread()
|
||||
g_activityReportsInFlight.fetch_sub(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 屏幕预览
|
||||
// 对应 Windows/Linux 的 COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP。
|
||||
// Android 的 JPEG 在 Kotlin 侧(ScreenShot,无障碍 takeScreenshot)编码好,这里只搬运;
|
||||
// 与 Linux 回 BGRX 不同,服务端对 JPEG 直接透传,不再二次编码。
|
||||
static std::atomic<int> g_previewInFlight(0);
|
||||
|
||||
// 抓一帧 JPEG:JNI 调 ScreenShot.capture(maxWidth, quality),把 Shot 的数据/宽高取回来。
|
||||
// 失败(无障碍没开、系统限流、编码失败)返回 false,调用方回 CAPTURE_FAILED。
|
||||
static bool CaptureScreenJpeg(uint16_t maxWidth, uint8_t quality,
|
||||
std::vector<uint8_t>& jpeg, int& w, int& h)
|
||||
{
|
||||
jpeg.clear();
|
||||
w = h = 0;
|
||||
if (!g_jvm || !g_shotClass || !g_shotCaptureMid || !g_shotDataMid ||
|
||||
!g_shotWidthMid || !g_shotHeightMid)
|
||||
return false;
|
||||
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ok = false;
|
||||
jobject shot = jenv->CallStaticObjectMethod(g_shotClass, g_shotCaptureMid,
|
||||
(jint)maxWidth, (jint)quality);
|
||||
if (shot && !jenv->ExceptionCheck()) {
|
||||
jbyteArray arr = (jbyteArray)jenv->CallObjectMethod(shot, g_shotDataMid);
|
||||
if (!jenv->ExceptionCheck() && arr) {
|
||||
jint bw = jenv->CallIntMethod(shot, g_shotWidthMid);
|
||||
jint bh = jenv->CallIntMethod(shot, g_shotHeightMid);
|
||||
if (!jenv->ExceptionCheck() && bw > 0 && bh > 0) {
|
||||
jsize len = jenv->GetArrayLength(arr);
|
||||
if (len > 0) {
|
||||
jpeg.resize((size_t)len);
|
||||
jenv->GetByteArrayRegion(arr, 0, len, (jbyte*)jpeg.data());
|
||||
if (!jenv->ExceptionCheck()) {
|
||||
w = bw;
|
||||
h = bh;
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
jenv->DeleteLocalRef(arr);
|
||||
}
|
||||
jenv->DeleteLocalRef(shot);
|
||||
}
|
||||
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
if (!ok) jpeg.clear();
|
||||
return ok;
|
||||
}
|
||||
|
||||
// 登录前探测:无障碍服务在跑(API 30+)才声明 CLIENT_CAP_SCREEN_PREVIEW,
|
||||
// 服务端据此放行 get_screenshot。能力位随每次登录刷新,无障碍被关掉重连后会撤回。
|
||||
static bool ScreenPreviewAvailable()
|
||||
{
|
||||
if (!g_jvm || !g_shotClass || !g_shotCanMid) return false;
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return false;
|
||||
}
|
||||
jboolean can = jenv->CallStaticBooleanMethod(g_shotClass, g_shotCanMid);
|
||||
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); can = JNI_FALSE; }
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
return can == JNI_TRUE;
|
||||
}
|
||||
|
||||
// 抓屏可能耗几十~几百毫秒(系统回调 + 编码),必须离开接收线程;超额请求直接丢弃
|
||||
// (服务端会超时按「预览不可用」降级,与 Linux/Windows 的限流一致)。
|
||||
static void ScreenPreviewThread(ScreenPreviewReq req)
|
||||
{
|
||||
struct InFlightGuard {
|
||||
~InFlightGuard() { g_previewInFlight.fetch_sub(1); }
|
||||
} guard;
|
||||
|
||||
std::vector<uint8_t> jpeg;
|
||||
int w = 0, h = 0;
|
||||
uint8_t status = CaptureScreenJpeg(req.maxWidth, req.jpegQuality, jpeg, w, h)
|
||||
? (uint8_t)SCREEN_PREVIEW_OK
|
||||
: (uint8_t)SCREEN_PREVIEW_CAPTURE_FAILED;
|
||||
|
||||
uint32_t bytes = (status == SCREEN_PREVIEW_OK) ? (uint32_t)jpeg.size() : 0;
|
||||
std::vector<uint8_t> pkt(sizeof(ScreenPreviewRspHeader) + bytes, 0);
|
||||
ScreenPreviewRspHeader hdr;
|
||||
memset(&hdr, 0, sizeof(hdr));
|
||||
hdr.token = TOKEN_SCREEN_PREVIEW_RSP;
|
||||
hdr.reqId = req.reqId;
|
||||
hdr.status = status;
|
||||
hdr.format = SCREEN_PREVIEW_FMT_JPEG; // 失败时 format 无意义,填 JPEG 保持包头合法
|
||||
hdr.width = (uint16_t)w;
|
||||
hdr.height = (uint16_t)h;
|
||||
hdr.bytes = bytes;
|
||||
memcpy(pkt.data(), &hdr, sizeof(hdr));
|
||||
if (bytes) memcpy(pkt.data() + sizeof(hdr), jpeg.data(), bytes);
|
||||
|
||||
// 与 ActivityReportThread 同一套路:全局指针只取一次快照
|
||||
IOCPClient* mc = g_pMainClient;
|
||||
if (mc && mc->IsConnected() && g_bExit == S_CLIENT_NORMAL)
|
||||
mc->Send2Server((char*)pkt.data(), (ULONG)pkt.size());
|
||||
LOGI("[Preview] reqId=%u %dx%d %u bytes status=%u", req.reqId, w, h, bytes, status);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 在线升级
|
||||
// 载荷布局:[COMMAND_UPDATE:1][APK 大小:8 LE][APK:size][xxh64:8 LE]
|
||||
// 校验通过后写到 filesDir/update.apk,交给 Kotlin 的 PackageInstaller 会话去装
|
||||
@@ -851,6 +970,27 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
|
||||
}
|
||||
break;
|
||||
|
||||
case COMMAND_SCREEN_PREVIEW_REQ: {
|
||||
// MCP get_screenshot / 在线列表缩略图:主连接回一次 JPEG
|
||||
if (ulLength < sizeof(ScreenPreviewReq)) break;
|
||||
ScreenPreviewReq req;
|
||||
memcpy(&req, szBuffer, sizeof(req));
|
||||
if (g_previewInFlight.fetch_add(1) >= 1) {
|
||||
g_previewInFlight.fetch_sub(1); // 已有抓屏在跑,重复请求丢弃
|
||||
break;
|
||||
}
|
||||
LOGI("cmd: COMMAND_SCREEN_PREVIEW_REQ reqId=%u maxWidth=%u quality=%u",
|
||||
req.reqId, req.maxWidth, req.jpegQuality);
|
||||
try {
|
||||
std::thread(ScreenPreviewThread, req).detach();
|
||||
} catch (const std::exception& e) {
|
||||
// 线程起不来要把在途计数还回去,否则之后所有预览请求都被当成「忙」丢掉
|
||||
g_previewInFlight.fetch_sub(1);
|
||||
LOGE("Screen preview thread failed: %s", e.what());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_SHELL:
|
||||
LOGI("COMMAND_SHELL (not implemented)");
|
||||
break;
|
||||
@@ -999,6 +1139,10 @@ static void ConnectionThread(uint64_t gen)
|
||||
#ifdef YAMA_TLS_CHANNEL
|
||||
RefreshLoginTls(logInfo); // 传输方式可能已热切换,登录前刷新 RES_TLS
|
||||
#endif
|
||||
// 屏幕预览能力位按运行时状态刷新:无障碍服务在跑才声明(API 30+ 的 takeScreenshot),
|
||||
// 服务端据此放行 get_screenshot;无障碍被关掉后重连会撤回,避免服务端白等抓屏
|
||||
logInfo.SetCaps(LOGIN_INFOR::DefaultCaps() |
|
||||
(ScreenPreviewAvailable() ? CLIENT_CAP_SCREEN_PREVIEW : 0));
|
||||
client->SendLoginInfo(logInfo.Speed(clock() - c));
|
||||
g_lastHeartbeatAckMs.store(GetUnixMs(), std::memory_order_relaxed);
|
||||
LOGI("Connected & login sent");
|
||||
@@ -1191,6 +1335,33 @@ Java_com_yama_client_YamaBridge_nativeInit(
|
||||
} else {
|
||||
LOGE("CaptureService class not found");
|
||||
}
|
||||
// 屏幕预览:返回对象是 ScreenShot$Shot,JNI 签名用 '$' 写嵌套类
|
||||
jclass shotCls = env->FindClass("com/yama/client/ScreenShot");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); shotCls = nullptr; }
|
||||
if (shotCls) {
|
||||
g_shotClass = (jclass)env->NewGlobalRef(shotCls);
|
||||
g_shotCanMid = env->GetStaticMethodID(g_shotClass, "canCapture", "()Z");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCanMid = nullptr; }
|
||||
if (!g_shotCanMid) LOGE("ScreenShot.canCapture not found");
|
||||
g_shotCaptureMid = env->GetStaticMethodID(g_shotClass, "capture",
|
||||
"(II)Lcom/yama/client/ScreenShot$Shot;");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_shotCaptureMid = nullptr; }
|
||||
if (!g_shotCaptureMid) LOGE("ScreenShot.capture not found");
|
||||
jclass shotResultCls = env->FindClass("com/yama/client/ScreenShot$Shot");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); shotResultCls = nullptr; }
|
||||
if (shotResultCls) {
|
||||
g_shotDataMid = env->GetMethodID(shotResultCls, "getData", "()[B");
|
||||
g_shotWidthMid = env->GetMethodID(shotResultCls, "getWidth", "()I");
|
||||
g_shotHeightMid = env->GetMethodID(shotResultCls, "getHeight", "()I");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); }
|
||||
if (!g_shotDataMid || !g_shotWidthMid || !g_shotHeightMid)
|
||||
LOGE("ScreenShot$Shot getters not found");
|
||||
} else {
|
||||
LOGE("ScreenShot$Shot class not found");
|
||||
}
|
||||
} else {
|
||||
LOGE("ScreenShot class not found");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user