Feature: Support voice monitoring on Android hosts

The master's voice monitoring menu sends COMMAND_AUDIO; the client opens a
ConnAuth sub-connection with TOKEN_AUDIO_START, the server opens CAudioDlg
and answers COMMAND_NEXT, then audio flows both ways. The protocol matches
the Windows client, so the server needs no changes.

The server's ACM format was read from Audio.cpp and AudioDlg.cpp first:
WAVE_FORMAT_GSM610, 8000 Hz mono, 65-byte blocks of 320 samples (40 ms),
about 1625 B/s. libgsm 1.0.13 (AOSP external/libgsm mirror) is vendored
under cpp/gsm and built with -DSASR -DWAV49 so its MS-GSM packing is
byte-identical to ACM GSM610. Encode and decode use separate gsm states
because frame_index/frame_chain live in the state.

Uplink packets are always exactly 1000 bytes: CAudio::PlayBuffer copies
1000 bytes from the buffer start regardless of packet length, so a short
packet would over-read. Since 1000 is not a multiple of 65, blocks
straddle packets; the stream is continuous and codec state carries over.

AudioMonitor.kt owns AudioRecord (8 kHz/16-bit/mono, 320 samples a read)
and AudioTrack playback of decoded downlink PCM. RECORD_AUDIO is requested
with POST_NOTIFICATIONS before the projection dialog, and CaptureService
adds the microphone FGS type only when held, falling back on failure. An
8-second dialog wait or missing microphone reports a tip to the master
instead of failing silently.

Verified on an Android 15 emulator against the Debug server: dialog
opened, uplink started, server receive counter grew at ~1.6 KB/s.
Downlink playback was not exercised (the test machine has no recording
device); that path is code-reviewed only.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-11 21:58:54 +01:00
1 parent 3ab5154db1
commit 4aedca8ee0
31 files changed
+4979 -22

No files matched your search

+201
View File
@@ -26,6 +26,7 @@
#include "client/IOCPClient.h"
#include "ScreenHandler.h"
#include "ChatHandler.h"
#include "AudioHandler.h"
#include "common/PTYHandler.h" // 远程终端(COMMAND_SHELL):与 Linux/macOS 共用 PTY 实现
#define XXH_INLINE_ALL
#include "common/xxhash.h"
@@ -260,6 +261,11 @@ static jmethodID g_onQualityProfilesMid = nullptr; // ScreenTuning.onQualityPr
// ChatSession(远程交谈界面:帧入站文本、发送按钮)
static jclass g_chatClass = nullptr;
static jmethodID g_chatEventMid = nullptr; // ChatSession.onNativeEvent(I[B)V
// AudioMonitor(语音监听:AudioRecord 采集 / AudioTrack 播放)
static jclass g_audioClass = nullptr;
static jmethodID g_audioStartMid = nullptr; // AudioMonitor.start()Z
static jmethodID g_audioStopMid = nullptr; // AudioMonitor.stop()V
static jmethodID g_audioPcmMid = nullptr; // AudioMonitor.onPcm([S)V
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
static jclass g_shotClass = nullptr;
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
@@ -390,6 +396,76 @@ void DispatchChatEvent(int kind, const uint8_t* data, size_t size)
if (attached) g_jvm->DetachCurrentThread();
}
// 语音监听:native 子连接解码出 PCM → AudioMonitor.onPcm(入 AudioTrack 播放)。
// 与 DispatchChatEvent 同一套 attach/detach 套路,供 AndroidAudioHandler::OnReceive 调用。
void DispatchAudioPcm(const int16_t* samples, size_t count)
{
if (!g_jvm || !g_audioClass || !g_audioPcmMid || !samples || !count) return;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return;
attached = true;
} else if (st != JNI_OK || !jenv) {
return;
}
jshortArray arr = jenv->NewShortArray((jsize)count);
if (arr) {
jenv->SetShortArrayRegion(arr, 0, (jsize)count, (const jshort*)samples);
if (!jenv->ExceptionCheck()) {
jenv->CallStaticVoidMethod(g_audioClass, g_audioPcmMid, arr);
}
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
jenv->DeleteLocalRef(arr);
} else {
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
}
if (attached) g_jvm->DetachCurrentThread();
}
// 开始采集:AudioMonitor.start() 内部检查 RECORD_AUDIO、建 AudioRecord/AudioTrack。
// 返回 false(没权限 / 设备没有麦克风)时调用方回报服务端并放弃本次会话。
static bool StartJavaAudio()
{
if (!g_jvm || !g_audioClass || !g_audioStartMid) {
LOGE("AUDIO: JNI not ready (cls=%p start=%p)", g_audioClass, g_audioStartMid);
return false;
}
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
attached = true;
} else if (st != JNI_OK || !jenv) {
return false;
}
jboolean ok = jenv->CallStaticBooleanMethod(g_audioClass, g_audioStartMid);
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); ok = JNI_FALSE; }
if (attached) g_jvm->DetachCurrentThread();
return ok == JNI_TRUE;
}
// 停止采集/播放并回收 AudioRecord/AudioTrack(Kotlin 侧会 join 采集线程,
// 返回后不会再回调 nativeAudioSendPcm)
static void StopJavaAudio()
{
if (!g_jvm || !g_audioClass || !g_audioStopMid) return;
JNIEnv* jenv = nullptr;
bool attached = false;
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
if (st == JNI_EDETACHED) {
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return;
attached = true;
} else if (st != JNI_OK || !jenv) {
return;
}
jenv->CallStaticVoidMethod(g_audioClass, g_audioStopMid);
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
if (attached) g_jvm->DetachCurrentThread();
}
// 锁屏:服务端下发 COMMAND_SCREEN_CTRL_ALT_DEL(Android 无此组合键,映射为
// GLOBAL_ACTION_LOCK_SCREEN)。与 DispatchClipboardText 同一套 attach/detach 套路。
void DispatchLockScreen()
@@ -920,6 +996,80 @@ static void ChatThread()
LOGI("ChatThread exit");
}
// ------------------------------------------------------------------ 语音监听
// 服务端 COMMAND_AUDIO(主控「语音监听」)→ 子连接 + 首包 TOKEN_AUDIO_START →
// CAudioDlg 弹窗后回 COMMAND_NEXT → Kotlin 侧开 AudioRecord 采集(GSM 6.10/WAV49
// 上行);主控勾选「发送语音」后下行是裸 GSM 流。协议细节见 AudioHandler.h。
// 同一时刻只允许一个会话:服务端菜单重复点击/多主控会再发 COMMAND_AUDIO,
// 第二个子连接会被服务端当重复子连接吸收,这里直接忽略。
static std::atomic<bool> g_audioInFlight{false};
static std::mutex g_audioMx;
static AndroidAudioHandler* g_audioHandler = nullptr;
static void sendClientMsg(const char* title, const char* text); // 定义在下方的在线升级段
static void AudioThread()
{
// detached 线程:异常逃出去会 std::terminate;在途标志必须无条件还回去
struct InFlightGuard {
~InFlightGuard() { g_audioInFlight.store(false); }
} guard;
LOGI("AudioThread start → %s:%d", g_serverIp.c_str(), g_serverPort);
std::unique_ptr<IOCPClient> client;
std::unique_ptr<AndroidAudioHandler> handler;
bool started = false;
try {
client = std::make_unique<IOCPClient>(g_bExit, true);
client->EnableSubConnAuth(true, g_myClientID);
// 与屏幕/日志/交谈子连接一致:必须继承配置的传输方式(WS/WSS 部署)
client->SetDefaultProto((int)(unsigned char)g_SETTINGS.protoType);
if (!client->ConnectServer(g_serverIp.c_str(), g_serverPort)) {
LOGE("AudioThread: connect failed");
return;
}
// 构造即发 TOKEN_AUDIO_START;服务端 CAudioDlg 打开后回单字节 COMMAND_NEXT
handler = std::make_unique<AndroidAudioHandler>(client.get());
client->setManagerCallBack(handler.get(),
IOCPManager::DataProcess,
IOCPManager::ReconnectProcess);
{
std::lock_guard<std::mutex> lk(g_audioMx);
g_audioHandler = handler.get();
}
if (!handler->WaitDialogOpen(8000)) {
// Windows 客户端同样等 8 秒;超时说明服务端没接管(老服务端/对话框被关)
LOGE("AudioThread: server dialog did not open within 8s");
sendClientMsg("Audio monitor", "The audio dialog did not open on the server");
} else if (!StartJavaAudio()) {
// 常见原因:用户没给 RECORD_AUDIO,或设备没有可用麦克风
LOGE("AudioThread: AudioRecord unavailable (RECORD_AUDIO not granted?)");
sendClientMsg("Audio monitor",
"Microphone unavailable: RECORD_AUDIO permission is not granted");
} else {
started = true;
while (client->IsRunning() && client->IsConnected() && S_CLIENT_NORMAL == g_bExit)
Sleep(200);
LOGI("AudioThread: sub-conn closed");
}
} catch (const std::exception& e) {
LOGE("AudioThread exception: %s", e.what());
}
// 先停采集(join 采集线程,保证不再回调 nativeAudioSendPcm),再摘掉全局指针,
// 最后断开回调,让 handler 析构时没有任何线程还能碰它
if (started) StopJavaAudio();
{
std::lock_guard<std::mutex> lk(g_audioMx);
g_audioHandler = nullptr;
}
if (client) client->setManagerCallBack(nullptr, nullptr, nullptr);
// 局部变量按声明逆序析构(handler 先于 client),而 IOCPClient 的析构才会等待
// 接收线程退出。显式先销毁连接,避免接收线程正卡在 OnReceive 里时 handler 已被释放。
client.reset();
handler.reset();
LOGI("AudioThread exit");
}
// ------------------------------------------------------------------ 远程终端
// 服务端 COMMAND_SHELL → 子连接 → 首包 TOKEN_TERMINAL_START → 服务端接管后发
// CMD_TERMINAL_RESIZE + COMMAND_NEXT → PTY 读线程开始回传 shell 输出。
@@ -1389,6 +1539,23 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
break;
}
case COMMAND_AUDIO: {
// 语音监听(主控菜单「语音监听」):子连接 + 上行 GSM 6.10。需要 RECORD_AUDIO;
// 没有权限时 StartJavaAudio 失败,回报服务端而不是静默无声。
if (g_audioInFlight.exchange(true)) {
LOGI("cmd: COMMAND_AUDIO ignored (audio session already in flight)");
break;
}
LOGI("cmd: COMMAND_AUDIO → audio sub-conn");
try {
std::thread(AudioThread).detach();
} catch (const std::exception& e) {
g_audioInFlight.store(false);
LOGE("Audio thread failed: %s", e.what());
}
break;
}
case COMMAND_SCREEN_CONTROL: {
if (ulLength < 1 + 48u) { LOGI("SCREEN_CONTROL(main): too short %u", ulLength); break; }
const uint8_t* p = szBuffer + 1;
@@ -1888,6 +2055,23 @@ Java_com_yama_client_YamaBridge_nativeInit(
} else {
LOGE("ChatSession class not found");
}
// 语音监听:AudioMonitor 的启停与下行 PCM 回调
jclass audioCls = env->FindClass("com/yama/client/AudioMonitor");
if (env->ExceptionCheck()) { env->ExceptionClear(); audioCls = nullptr; }
if (audioCls) {
g_audioClass = (jclass)env->NewGlobalRef(audioCls);
g_audioStartMid = env->GetStaticMethodID(g_audioClass, "start", "()Z");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioStartMid = nullptr; }
if (!g_audioStartMid) LOGE("AudioMonitor.start not found");
g_audioStopMid = env->GetStaticMethodID(g_audioClass, "stop", "()V");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioStopMid = nullptr; }
if (!g_audioStopMid) LOGE("AudioMonitor.stop not found");
g_audioPcmMid = env->GetStaticMethodID(g_audioClass, "onPcm", "([S)V");
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioPcmMid = nullptr; }
if (!g_audioPcmMid) LOGE("AudioMonitor.onPcm not found");
} else {
LOGE("AudioMonitor class not found");
}
// 远程桌面画质/帧率协商:ScreenTuning 的读写入口(见本文件上方的说明)
jclass tuningCls = env->FindClass("com/yama/client/ScreenTuning");
if (env->ExceptionCheck()) { env->ExceptionClear(); tuningCls = nullptr; }
@@ -1971,6 +2155,23 @@ Java_com_yama_client_YamaBridge_nativeChatSend(JNIEnv* env, jobject, jbyteArray
return ok ? JNI_TRUE : JNI_FALSE;
}
// AudioMonitor 采集线程每读一次 PCM 调一次(8kHz/16bit/单声道,长度任意)。
// 这里只做搬运动作:攒够 160 采样交给 libgsm 编码,满 1000 字节发一个 TOKEN_AUDIO_DATA。
JNIEXPORT void JNICALL
Java_com_yama_client_YamaBridge_nativeAudioSendPcm(JNIEnv* env, jobject, jshortArray samples)
{
if (!samples) return;
const jsize len = env->GetArrayLength(samples);
// 采集端一次读 320 采样(40ms);给异常调用留 1 秒上限,超了直接丢
if (len <= 0 || len > 8000) return;
std::lock_guard<std::mutex> lk(g_audioMx);
if (!g_audioHandler) return;
jshort* buf = env->GetShortArrayElements(samples, nullptr);
if (!buf) return;
g_audioHandler->SendPcm((const int16_t*)buf, (size_t)len);
env->ReleaseShortArrayElements(samples, buf, JNI_ABORT);
}
// 由 CaptureService 在 MediaCodec 配置完成后调用,告知 C++ 侧实际捕获尺寸
JNIEXPORT void JNICALL
Java_com_yama_client_YamaBridge_nativeSetScreenSize(JNIEnv*, jobject, jint width, jint height)