Feature: Support remote clipboard and input on Android hosts

The MCP remote-control family (remote_open / remote_close / remote_mouse /
remote_keyboard / remote_clipboard) previously stopped at Windows and Linux,
even though the Android client already translated injected events into touch
gestures and global actions. This wires up the missing piece and opens the five
tools to APK hosts.

Client side, the screen sub-connection now handles COMMAND_SCREEN_SET_CLIPBOARD:
the native handler forwards the raw GBK bytes to CaptureService.onClipboardText,
which decodes them, strips the trailing NUL that the human clipboard-sync
senders append, and writes the system clipboard from a background app (allowed
on Android 10+, only reads are restricted). Pasting cannot be a Ctrl+V combo,
so VK_V is mapped to ACTION_PASTE on the focused node, falling back to the
accessibility focus and then to the first editable node, since WebView and
self-drawn inputs often do not flag FOCUS_INPUT or isEditable.

A new capability bit CLIENT_CAP_SCREEN_CLIPBOARD (0x0800) tells the server the
APK understands both paths, so builds without the handler get an explicit
-32005 instead of a package that is silently dropped. action=type is rejected
outright for APK hosts - physical key events would only turn 'v' into a paste -
and the error points at remote_clipboard + key=V.

The platform table, server instructions and the five tool descriptions now
describe Android support, which keys actually act as global actions (ESC, HOME,
APPS, PRINTSCREEN), and that injection needs the accessibility service while
capture needs an authorized MediaProjection session, both without receipts.

Verified on the API 35 emulator against a Debug server: ASCII and GBK Chinese
text round-tripped through the clipboard into a focused Settings search field,
type / expect_hwnd / window_hwnd returned -32005, and PRINTSCREEN produced a
saved screenshot. check_platform_table.py and verify_tools.py --offline pass.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-11 20:59:10 +01:00
1 parent ac43ad92ef
commit 0a0b709e8a
9 files changed
+351 -179

No files matched your search

+50
View File
@@ -242,6 +242,7 @@ static jmethodID g_forceFirstFrameMid = nullptr;
static jmethodID g_statusMid = nullptr; // CaptureService.onNativeStatus(String)
static jmethodID g_nativeExitMid = nullptr; // CaptureService.onNativeExit() — 服务端主动断开时停止服务
static jmethodID g_updateReceivedMid = nullptr; // CaptureService.onUpdateReceived(String) — 升级 APK 已落盘
static jmethodID g_clipboardMid = nullptr; // CaptureService.onClipboardText([B)V — 远程剪贴板
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
static jclass g_shotClass = nullptr;
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
@@ -282,6 +283,52 @@ void DispatchControlEvent(uint32_t msgVal, uint64_t wParam, int32_t ptX, int32_t
if (attached) g_jvm->DetachCurrentThread();
}
// 远程剪贴板:服务端下发 [COMMAND_SCREEN_SET_CLIPBOARD][GBK text],字节原样交给
// CaptureService.onClipboardText(Kotlin 侧按 GBK 解码后写系统剪贴板)。
// 与 DispatchControlEvent 同一套 attach/detach 套路,供屏幕子连接 OnReceive 调用。
void DispatchClipboardText(const uint8_t* data, size_t size)
{
if (!g_jvm || !g_captureClass || !g_clipboardMid) {
LOGE("CLIPBOARD: JNI not ready (cls=%p method=%p)", g_captureClass, g_clipboardMid);
return;
}
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return;
attached = true;
} else if (st != JNI_OK || !jenv) {
return;
}
// 服务端已把剪贴板文本限制在 256 KB,这里再兜一层 1 MiB:超出直接丢,
// 不截断(半截文本写进剪贴板比不写更危险)
const size_t kMaxClipboardBytes = 1u << 20;
if (size > kMaxClipboardBytes) {
LOGE("CLIPBOARD: payload %zu bytes exceeds %zu, dropped", size, kMaxClipboardBytes);
if (attached) g_jvm->DetachCurrentThread();
return;
}
jbyteArray arr = jenv->NewByteArray((jsize)size);
if (!arr) {
if (jenv->ExceptionOccurred()) jenv->ExceptionClear();
if (attached) g_jvm->DetachCurrentThread();
LOGE("CLIPBOARD: NewByteArray(%zu) failed", size);
return;
}
jenv->SetByteArrayRegion(arr, 0, (jsize)size, (const jbyte*)data);
if (!jenv->ExceptionCheck()) {
jenv->CallStaticVoidMethod(g_captureClass, g_clipboardMid, arr);
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
LOGI("CLIPBOARD: dispatched %zu bytes", size);
} else {
jenv->ExceptionClear();
LOGE("CLIPBOARD: SetByteArrayRegion failed");
}
jenv->DeleteLocalRef(arr);
if (attached) g_jvm->DetachCurrentThread();
}
static std::string GetActiveWindowFromJava()
{
if (!g_jvm || !g_ctrlClass || !g_getActiveWindowMid) return "Android";
@@ -1369,6 +1416,9 @@ Java_com_yama_client_YamaBridge_nativeInit(
g_updateReceivedMid = env->GetStaticMethodID(g_captureClass, "onUpdateReceived", "(Ljava/lang/String;)V");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_updateReceivedMid = nullptr; }
if (!g_updateReceivedMid) LOGE("CaptureService.onUpdateReceived not found");
g_clipboardMid = env->GetStaticMethodID(g_captureClass, "onClipboardText", "([B)V");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_clipboardMid = nullptr; }
if (!g_clipboardMid) LOGE("CaptureService.onClipboardText not found");
UseAndroidLog(PostStatus);
} else {
LOGE("CaptureService class not found");