Feature: Support voice monitoring on Android hosts

The master's voice monitoring menu sends COMMAND_AUDIO; the client opens a
ConnAuth sub-connection with TOKEN_AUDIO_START, the server opens CAudioDlg
and answers COMMAND_NEXT, then audio flows both ways. The protocol matches
the Windows client, so the server needs no changes.

The server's ACM format was read from Audio.cpp and AudioDlg.cpp first:
WAVE_FORMAT_GSM610, 8000 Hz mono, 65-byte blocks of 320 samples (40 ms),
about 1625 B/s. libgsm 1.0.13 (AOSP external/libgsm mirror) is vendored
under cpp/gsm and built with -DSASR -DWAV49 so its MS-GSM packing is
byte-identical to ACM GSM610. Encode and decode use separate gsm states
because frame_index/frame_chain live in the state.

Uplink packets are always exactly 1000 bytes: CAudio::PlayBuffer copies
1000 bytes from the buffer start regardless of packet length, so a short
packet would over-read. Since 1000 is not a multiple of 65, blocks
straddle packets; the stream is continuous and codec state carries over.

AudioMonitor.kt owns AudioRecord (8 kHz/16-bit/mono, 320 samples a read)
and AudioTrack playback of decoded downlink PCM. RECORD_AUDIO is requested
with POST_NOTIFICATIONS before the projection dialog, and CaptureService
adds the microphone FGS type only when held, falling back on failure. An
8-second dialog wait or missing microphone reports a tip to the master
instead of failing silently.

Verified on an Android 15 emulator against the Debug server: dialog
opened, uplink started, server receive counter grew at ~1.6 KB/s.
Downlink playback was not exercised (the test machine has no recording
device); that path is code-reviewed only.

Co-Authored-By: deepseek-flash
This commit is contained in:
yuanyuanxianganddeepseek-flash committed 2026-11-11 21:58:54 +01:00
1 parent 3ab5154db1
commit 4aedca8ee0
31 files changed
+4979 -22

No files matched your search

@@ -0,0 +1,194 @@
package com.yama.client
import android.Manifest
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioManager
import android.media.AudioRecord
import android.media.AudioTrack
import android.media.MediaRecorder
import android.util.Log
import androidx.core.content.ContextCompat
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.TimeUnit
/**
* 语音监听(COMMAND_AUDIO)的安卓音频端。
*
* 服务端 CAudioDlg 要的是 Windows ACM 的 WAVE_FORMAT_GSM610:8kHz 单声道、
* 65 字节/320 采样。AudioRecord 只出 PCM,编码在 native 侧用随包编译的 libgsm
* (WAV49 模式)完成,见 cpp/AudioHandler.h。所以这里只负责:
* - AudioRecord 采集 PCM(40ms 一次,320 采样)→ YamaBridge.nativeAudioSendPcm
* - 下行解码后的 PCM(native 回调 onPcm)→ AudioTrack 播放(主控勾选「发送语音」时)
*
* 权限/后台限制:Android 11+ 后台采集麦克风需要带 microphone 类型的
* 前台服务(CaptureService 已声明,见 AndroidManifest),且 RECORD_AUDIO 必须是
* 运行时已授权。start() 失败时 native 侧会回报服务端而不是静默无声。
*/
object AudioMonitor {
private const val TAG = "YAMA_AUD"
private const val SAMPLE_RATE = 8000
private const val READ_SAMPLES = 320 // 40ms @ 8kHz
private const val QUEUE_CAPACITY = 32 // 下行最多缓存 ~1.3 秒,超了丢最旧的
@Volatile private var running = false
private var record: AudioRecord? = null
private var track: AudioTrack? = null
private var captureThread: Thread? = null
private var playThread: Thread? = null
private val pcmQueue = LinkedBlockingQueue<ShortArray>(QUEUE_CAPACITY)
/** native 侧在服务端对话框打开后调用;false = 没权限 / 没有可用麦克风 */
@JvmStatic
@Synchronized
fun start(): Boolean {
if (running) return true
val ctx = CaptureService.appContext
if (ctx == null) {
Log.e(TAG, "start: app context is not ready")
return false
}
if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.RECORD_AUDIO)
!= PackageManager.PERMISSION_GRANTED) {
Log.e(TAG, "start: RECORD_AUDIO not granted")
return false
}
val minRec = AudioRecord.getMinBufferSize(
SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
if (minRec <= 0) {
Log.e(TAG, "start: getMinBufferSize(record) = $minRec")
return false
}
val rec = try {
AudioRecord(
MediaRecorder.AudioSource.MIC, SAMPLE_RATE,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT,
maxOf(minRec, READ_SAMPLES * 2 * 8))
} catch (e: Exception) {
Log.e(TAG, "start: AudioRecord create failed: $e")
return false
}
if (rec.state != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "start: AudioRecord not initialized")
rec.release()
return false
}
val minTrk = AudioTrack.getMinBufferSize(
SAMPLE_RATE, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT)
val trk = try {
AudioTrack(
AudioManager.STREAM_MUSIC, SAMPLE_RATE,
AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT,
maxOf(minTrk, READ_SAMPLES * 2 * 8), AudioTrack.MODE_STREAM)
} catch (e: Exception) {
Log.e(TAG, "start: AudioTrack create failed: $e")
rec.release()
return false
}
// Android 11+ 在后台且没有 microphone 类型前台服务时,startRecording() 会抛
// SecurityException(或被系统直接静音)。失败就整体回滚,让 native 回报服务端。
try {
rec.startRecording()
} catch (e: Exception) {
Log.e(TAG, "start: startRecording failed: $e")
runCatching { rec.release() }
runCatching { trk.release() }
return false
}
if (rec.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
Log.e(TAG, "start: recorder is not recording (background mic restriction?)")
runCatching { rec.release() }
runCatching { trk.release() }
return false
}
trk.play()
running = true
record = rec
track = trk
pcmQueue.clear()
captureThread = Thread({ captureLoop(rec) }, "AudioCapture").also { it.start() }
playThread = Thread({ playLoop(trk) }, "AudioPlayback").also { it.start() }
Log.i(TAG, "audio monitor started (${SAMPLE_RATE}Hz mono PCM -> GSM)")
return true
}
/** native 侧在会话结束/子连接断开时调用;会等采集线程退出后才返回 */
@JvmStatic
@Synchronized
fun stop() {
if (!running && record == null && track == null) return
running = false
val cap = captureThread
val play = playThread
captureThread = null
playThread = null
// 先停录制,阻塞中的 read() 才会尽快返回
record?.let { r -> runCatching { r.stop() } }
cap?.join(1500)
record?.let { r -> runCatching { r.release() } }
record = null
play?.join(1500) // playLoop 每 200ms 检查一次 running;播放头可能阻塞在 write()
track?.let { t ->
runCatching { t.pause() }
runCatching { t.flush() }
runCatching { t.stop() }
runCatching { t.release() }
}
track = null
pcmQueue.clear()
Log.i(TAG, "audio monitor stopped")
}
/** native 子连接解出一块 PCM(320 采样)后回调;主控没勾「发送语音」时不会有 */
@JvmStatic
fun onPcm(samples: ShortArray) {
if (!running || samples.isEmpty()) return
if (!pcmQueue.offer(samples)) {
pcmQueue.poll() // 丢最旧的,保持低延迟
pcmQueue.offer(samples)
}
}
private fun captureLoop(rec: AudioRecord) {
val buf = ShortArray(READ_SAMPLES)
while (running) {
val n = try {
rec.read(buf, 0, buf.size)
} catch (e: Exception) {
Log.e(TAG, "read failed: $e")
-1
}
if (n <= 0) {
// 读失败/暂时没数据:让出 CPU,别空转
try {
Thread.sleep(5)
} catch (_: InterruptedException) {
break
}
continue
}
val out = if (n == buf.size) buf else buf.copyOf(n)
YamaBridge.nativeAudioSendPcm(out)
}
}
private fun playLoop(trk: AudioTrack) {
while (running) {
val pcm = try {
pcmQueue.poll(200, TimeUnit.MILLISECONDS)
} catch (e: InterruptedException) {
break
} ?: continue
runCatching { trk.write(pcm, 0, pcm.size) }
.onFailure { Log.e(TAG, "playback write failed: $it") }
}
}
}
@@ -1,5 +1,6 @@
package com.yama.client
import android.Manifest
import android.app.Notification
import android.app.NotificationChannel
import android.app.NotificationManager
@@ -8,6 +9,8 @@ import android.content.ClipData
import android.content.ClipboardManager
import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.content.pm.ServiceInfo
import android.hardware.display.DisplayManager
import android.hardware.display.VirtualDisplay
import android.media.MediaCodec
@@ -201,12 +204,7 @@ class CaptureService : Service() {
// kill the service after a ~5-second grace period — the root cause of the
// 3-9 s disconnect seen on Google TV when no screen-capture permission
// was granted.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIF_ID, buildNotification(),
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
} else {
startForeground(NOTIF_ID, buildNotification())
}
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
@@ -254,10 +252,7 @@ class CaptureService : Service() {
if (projResult == android.app.Activity.RESULT_OK && projData != null) {
// Android 14+: upgrade foreground type to MEDIA_PROJECTION in the same
// start command that calls getMediaProjection(), as required by API 34+.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIF_ID, buildNotification(),
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
}
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
val pm = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
val mp = pm.getMediaProjection(projResult, projData)
mediaProjection = mp
@@ -354,10 +349,7 @@ class CaptureService : Service() {
stopCapture()
// Android 14+: system auto-kills a foreground service whose mediaProjection
// is revoked. Re-declare as dataSync-only to survive without screen capture.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIF_ID, buildNotification(),
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
}
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
}
}, null)
@@ -481,6 +473,34 @@ class CaptureService : Service() {
}
}
/**
* 带类型的前台服务声明。语音监听(COMMAND_AUDIO)要在后台用麦克风,按 Android 11+
* 的要求前台服务必须带 microphone 类型,所以只要 RECORD_AUDIO 已授权就把这一位加上。
* 没授权时绝不能带:Android 14+ 会在 startForeground 时校验 while-in-use 权限,
* 直接抛 SecurityException 把服务打死。加了之后仍可能被系统拒绝(例如进程已在
* 后台且权限不是「使用中」),因此失败时退化回原始类型,保证远程控制本身不受影响。
*/
private fun startForegroundTyped(base: Int) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
startForeground(NOTIF_ID, buildNotification())
return
}
var types = base
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R &&
checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
}
try {
startForeground(NOTIF_ID, buildNotification(), types)
} catch (e: Exception) {
Log.e(TAG, "startForeground(types=0x${types.toString(16)}) failed: $e")
if (types != base) {
runCatching { startForeground(NOTIF_ID, buildNotification(), base) }
.onFailure { Log.e(TAG, "startForeground(base=0x${base.toString(16)}) also failed: $it") }
}
}
}
private fun buildNotification(): Notification {
val b = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)
Notification.Builder(this, CHANNEL_ID)
@@ -19,16 +19,13 @@ class MainActivity : AppCompatActivity() {
companion object {
const val REQUEST_MEDIA_PROJECTION = 1001
const val REQUEST_NOTIFICATIONS = 1002
// POST_NOTIFICATIONS(API 33+)与 RECORD_AUDIO(语音监听)合并成一次请求;
// 结果回来后再弹录屏授权,两个系统对话框不会叠在一起。
const val REQUEST_PERMISSIONS = 1002
}
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
// 升级完成/待确认的提示靠通知兜底,拒绝了也不影响主流程
requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), REQUEST_NOTIFICATIONS)
}
// 服务端下发升级包后系统安装框没拉起来时,靠通知把用户带到这一步:
// 先把待确认的安装框弹出来,装完再重新授权录屏(录制授权随旧进程一起失效了)。
val pendingInstall = UpdateInstaller.consumePendingConfirm()
@@ -49,6 +46,33 @@ class MainActivity : AppCompatActivity() {
// MA-01 fix: 系统重建 Activity 时 savedInstanceState != null,
// 此时 CaptureService 可能已在运行,不重复弹权限对话框
if (savedInstanceState != null) { finish(); return }
val missing = ArrayList<String>()
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
// 升级完成/待确认的提示靠通知兜底,拒绝了也不影响主流程
missing.add(Manifest.permission.POST_NOTIFICATIONS)
}
if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
// 语音监听(COMMAND_AUDIO)要麦克风;拒绝了不影响其它功能,
// 主控打开语音监听时客户端会回一条「麦克风不可用」的提示
missing.add(Manifest.permission.RECORD_AUDIO)
}
if (missing.isNotEmpty()) {
requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS)
return
}
startProjection()
}
override fun onRequestPermissionsResult(
requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults)
// 权限给不给都要继续:录屏授权本身是可选的(拒绝了 CaptureService 照常连服务端)
if (requestCode == REQUEST_PERMISSIONS) startProjection()
}
private fun startProjection() {
projectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
startActivityForResult(projectionManager.createScreenCaptureIntent(), REQUEST_MEDIA_PROJECTION)
}
@@ -35,4 +35,10 @@ object YamaBridge {
* 没有活跃的交谈子连接时返回 false。
*/
external fun nativeChatSend(data: ByteArray): Boolean
/**
* 语音监听:把 AudioRecord 采集的 PCM(8kHz/16bit/单声道)交给 native 编码成
* GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。
*/
external fun nativeAudioSendPcm(samples: ShortArray)
}