Feature: Support voice monitoring on Android hosts
The master's voice monitoring menu sends COMMAND_AUDIO; the client opens a ConnAuth sub-connection with TOKEN_AUDIO_START, the server opens CAudioDlg and answers COMMAND_NEXT, then audio flows both ways. The protocol matches the Windows client, so the server needs no changes. The server's ACM format was read from Audio.cpp and AudioDlg.cpp first: WAVE_FORMAT_GSM610, 8000 Hz mono, 65-byte blocks of 320 samples (40 ms), about 1625 B/s. libgsm 1.0.13 (AOSP external/libgsm mirror) is vendored under cpp/gsm and built with -DSASR -DWAV49 so its MS-GSM packing is byte-identical to ACM GSM610. Encode and decode use separate gsm states because frame_index/frame_chain live in the state. Uplink packets are always exactly 1000 bytes: CAudio::PlayBuffer copies 1000 bytes from the buffer start regardless of packet length, so a short packet would over-read. Since 1000 is not a multiple of 65, blocks straddle packets; the stream is continuous and codec state carries over. AudioMonitor.kt owns AudioRecord (8 kHz/16-bit/mono, 320 samples a read) and AudioTrack playback of decoded downlink PCM. RECORD_AUDIO is requested with POST_NOTIFICATIONS before the projection dialog, and CaptureService adds the microphone FGS type only when held, falling back on failure. An 8-second dialog wait or missing microphone reports a tip to the master instead of failing silently. Verified on an Android 15 emulator against the Debug server: dialog opened, uplink started, server receive counter grew at ~1.6 KB/s. Downlink playback was not exercised (the test machine has no recording device); that path is code-reviewed only. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
3ab5154db1
commit
4aedca8ee0
31 files changed
+4979
-22
No files matched your search
@@ -0,0 +1,194 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.Manifest
|
||||
import android.content.pm.PackageManager
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioRecord
|
||||
import android.media.AudioTrack
|
||||
import android.media.MediaRecorder
|
||||
import android.util.Log
|
||||
import androidx.core.content.ContextCompat
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 语音监听(COMMAND_AUDIO)的安卓音频端。
|
||||
*
|
||||
* 服务端 CAudioDlg 要的是 Windows ACM 的 WAVE_FORMAT_GSM610:8kHz 单声道、
|
||||
* 65 字节/320 采样。AudioRecord 只出 PCM,编码在 native 侧用随包编译的 libgsm
|
||||
* (WAV49 模式)完成,见 cpp/AudioHandler.h。所以这里只负责:
|
||||
* - AudioRecord 采集 PCM(40ms 一次,320 采样)→ YamaBridge.nativeAudioSendPcm
|
||||
* - 下行解码后的 PCM(native 回调 onPcm)→ AudioTrack 播放(主控勾选「发送语音」时)
|
||||
*
|
||||
* 权限/后台限制:Android 11+ 后台采集麦克风需要带 microphone 类型的
|
||||
* 前台服务(CaptureService 已声明,见 AndroidManifest),且 RECORD_AUDIO 必须是
|
||||
* 运行时已授权。start() 失败时 native 侧会回报服务端而不是静默无声。
|
||||
*/
|
||||
object AudioMonitor {
|
||||
private const val TAG = "YAMA_AUD"
|
||||
|
||||
private const val SAMPLE_RATE = 8000
|
||||
private const val READ_SAMPLES = 320 // 40ms @ 8kHz
|
||||
private const val QUEUE_CAPACITY = 32 // 下行最多缓存 ~1.3 秒,超了丢最旧的
|
||||
|
||||
@Volatile private var running = false
|
||||
private var record: AudioRecord? = null
|
||||
private var track: AudioTrack? = null
|
||||
private var captureThread: Thread? = null
|
||||
private var playThread: Thread? = null
|
||||
|
||||
private val pcmQueue = LinkedBlockingQueue<ShortArray>(QUEUE_CAPACITY)
|
||||
|
||||
/** native 侧在服务端对话框打开后调用;false = 没权限 / 没有可用麦克风 */
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun start(): Boolean {
|
||||
if (running) return true
|
||||
val ctx = CaptureService.appContext
|
||||
if (ctx == null) {
|
||||
Log.e(TAG, "start: app context is not ready")
|
||||
return false
|
||||
}
|
||||
if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.RECORD_AUDIO)
|
||||
!= PackageManager.PERMISSION_GRANTED) {
|
||||
Log.e(TAG, "start: RECORD_AUDIO not granted")
|
||||
return false
|
||||
}
|
||||
|
||||
val minRec = AudioRecord.getMinBufferSize(
|
||||
SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
||||
if (minRec <= 0) {
|
||||
Log.e(TAG, "start: getMinBufferSize(record) = $minRec")
|
||||
return false
|
||||
}
|
||||
val rec = try {
|
||||
AudioRecord(
|
||||
MediaRecorder.AudioSource.MIC, SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT,
|
||||
maxOf(minRec, READ_SAMPLES * 2 * 8))
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "start: AudioRecord create failed: $e")
|
||||
return false
|
||||
}
|
||||
if (rec.state != AudioRecord.STATE_INITIALIZED) {
|
||||
Log.e(TAG, "start: AudioRecord not initialized")
|
||||
rec.release()
|
||||
return false
|
||||
}
|
||||
|
||||
val minTrk = AudioTrack.getMinBufferSize(
|
||||
SAMPLE_RATE, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
||||
val trk = try {
|
||||
AudioTrack(
|
||||
AudioManager.STREAM_MUSIC, SAMPLE_RATE,
|
||||
AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT,
|
||||
maxOf(minTrk, READ_SAMPLES * 2 * 8), AudioTrack.MODE_STREAM)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "start: AudioTrack create failed: $e")
|
||||
rec.release()
|
||||
return false
|
||||
}
|
||||
|
||||
// Android 11+ 在后台且没有 microphone 类型前台服务时,startRecording() 会抛
|
||||
// SecurityException(或被系统直接静音)。失败就整体回滚,让 native 回报服务端。
|
||||
try {
|
||||
rec.startRecording()
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "start: startRecording failed: $e")
|
||||
runCatching { rec.release() }
|
||||
runCatching { trk.release() }
|
||||
return false
|
||||
}
|
||||
if (rec.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
|
||||
Log.e(TAG, "start: recorder is not recording (background mic restriction?)")
|
||||
runCatching { rec.release() }
|
||||
runCatching { trk.release() }
|
||||
return false
|
||||
}
|
||||
trk.play()
|
||||
|
||||
running = true
|
||||
record = rec
|
||||
track = trk
|
||||
pcmQueue.clear()
|
||||
captureThread = Thread({ captureLoop(rec) }, "AudioCapture").also { it.start() }
|
||||
playThread = Thread({ playLoop(trk) }, "AudioPlayback").also { it.start() }
|
||||
Log.i(TAG, "audio monitor started (${SAMPLE_RATE}Hz mono PCM -> GSM)")
|
||||
return true
|
||||
}
|
||||
|
||||
/** native 侧在会话结束/子连接断开时调用;会等采集线程退出后才返回 */
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun stop() {
|
||||
if (!running && record == null && track == null) return
|
||||
running = false
|
||||
val cap = captureThread
|
||||
val play = playThread
|
||||
captureThread = null
|
||||
playThread = null
|
||||
|
||||
// 先停录制,阻塞中的 read() 才会尽快返回
|
||||
record?.let { r -> runCatching { r.stop() } }
|
||||
cap?.join(1500)
|
||||
record?.let { r -> runCatching { r.release() } }
|
||||
record = null
|
||||
|
||||
play?.join(1500) // playLoop 每 200ms 检查一次 running;播放头可能阻塞在 write()
|
||||
track?.let { t ->
|
||||
runCatching { t.pause() }
|
||||
runCatching { t.flush() }
|
||||
runCatching { t.stop() }
|
||||
runCatching { t.release() }
|
||||
}
|
||||
track = null
|
||||
pcmQueue.clear()
|
||||
Log.i(TAG, "audio monitor stopped")
|
||||
}
|
||||
|
||||
/** native 子连接解出一块 PCM(320 采样)后回调;主控没勾「发送语音」时不会有 */
|
||||
@JvmStatic
|
||||
fun onPcm(samples: ShortArray) {
|
||||
if (!running || samples.isEmpty()) return
|
||||
if (!pcmQueue.offer(samples)) {
|
||||
pcmQueue.poll() // 丢最旧的,保持低延迟
|
||||
pcmQueue.offer(samples)
|
||||
}
|
||||
}
|
||||
|
||||
private fun captureLoop(rec: AudioRecord) {
|
||||
val buf = ShortArray(READ_SAMPLES)
|
||||
while (running) {
|
||||
val n = try {
|
||||
rec.read(buf, 0, buf.size)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "read failed: $e")
|
||||
-1
|
||||
}
|
||||
if (n <= 0) {
|
||||
// 读失败/暂时没数据:让出 CPU,别空转
|
||||
try {
|
||||
Thread.sleep(5)
|
||||
} catch (_: InterruptedException) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
val out = if (n == buf.size) buf else buf.copyOf(n)
|
||||
YamaBridge.nativeAudioSendPcm(out)
|
||||
}
|
||||
}
|
||||
|
||||
private fun playLoop(trk: AudioTrack) {
|
||||
while (running) {
|
||||
val pcm = try {
|
||||
pcmQueue.poll(200, TimeUnit.MILLISECONDS)
|
||||
} catch (e: InterruptedException) {
|
||||
break
|
||||
} ?: continue
|
||||
runCatching { trk.write(pcm, 0, pcm.size) }
|
||||
.onFailure { Log.e(TAG, "playback write failed: $it") }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.Manifest
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
@@ -8,6 +9,8 @@ import android.content.ClipData
|
||||
import android.content.ClipboardManager
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.content.pm.ServiceInfo
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.hardware.display.VirtualDisplay
|
||||
import android.media.MediaCodec
|
||||
@@ -201,12 +204,7 @@ class CaptureService : Service() {
|
||||
// kill the service after a ~5-second grace period — the root cause of the
|
||||
// 3-9 s disconnect seen on Google TV when no screen-capture permission
|
||||
// was granted.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification(),
|
||||
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
|
||||
} else {
|
||||
startForeground(NOTIF_ID, buildNotification())
|
||||
}
|
||||
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
@@ -254,10 +252,7 @@ class CaptureService : Service() {
|
||||
if (projResult == android.app.Activity.RESULT_OK && projData != null) {
|
||||
// Android 14+: upgrade foreground type to MEDIA_PROJECTION in the same
|
||||
// start command that calls getMediaProjection(), as required by API 34+.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification(),
|
||||
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
|
||||
}
|
||||
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
|
||||
val pm = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val mp = pm.getMediaProjection(projResult, projData)
|
||||
mediaProjection = mp
|
||||
@@ -354,10 +349,7 @@ class CaptureService : Service() {
|
||||
stopCapture()
|
||||
// Android 14+: system auto-kills a foreground service whose mediaProjection
|
||||
// is revoked. Re-declare as dataSync-only to survive without screen capture.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification(),
|
||||
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
|
||||
}
|
||||
startForegroundTyped(ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC)
|
||||
}
|
||||
}, null)
|
||||
|
||||
@@ -481,6 +473,34 @@ class CaptureService : Service() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 带类型的前台服务声明。语音监听(COMMAND_AUDIO)要在后台用麦克风,按 Android 11+
|
||||
* 的要求前台服务必须带 microphone 类型,所以只要 RECORD_AUDIO 已授权就把这一位加上。
|
||||
* 没授权时绝不能带:Android 14+ 会在 startForeground 时校验 while-in-use 权限,
|
||||
* 直接抛 SecurityException 把服务打死。加了之后仍可能被系统拒绝(例如进程已在
|
||||
* 后台且权限不是「使用中」),因此失败时退化回原始类型,保证远程控制本身不受影响。
|
||||
*/
|
||||
private fun startForegroundTyped(base: Int) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification())
|
||||
return
|
||||
}
|
||||
var types = base
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R &&
|
||||
checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
|
||||
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
}
|
||||
try {
|
||||
startForeground(NOTIF_ID, buildNotification(), types)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "startForeground(types=0x${types.toString(16)}) failed: $e")
|
||||
if (types != base) {
|
||||
runCatching { startForeground(NOTIF_ID, buildNotification(), base) }
|
||||
.onFailure { Log.e(TAG, "startForeground(base=0x${base.toString(16)}) also failed: $it") }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildNotification(): Notification {
|
||||
val b = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)
|
||||
Notification.Builder(this, CHANNEL_ID)
|
||||
|
||||
@@ -19,16 +19,13 @@ class MainActivity : AppCompatActivity() {
|
||||
|
||||
companion object {
|
||||
const val REQUEST_MEDIA_PROJECTION = 1001
|
||||
const val REQUEST_NOTIFICATIONS = 1002
|
||||
// POST_NOTIFICATIONS(API 33+)与 RECORD_AUDIO(语音监听)合并成一次请求;
|
||||
// 结果回来后再弹录屏授权,两个系统对话框不会叠在一起。
|
||||
const val REQUEST_PERMISSIONS = 1002
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
|
||||
checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
|
||||
// 升级完成/待确认的提示靠通知兜底,拒绝了也不影响主流程
|
||||
requestPermissions(arrayOf(Manifest.permission.POST_NOTIFICATIONS), REQUEST_NOTIFICATIONS)
|
||||
}
|
||||
// 服务端下发升级包后系统安装框没拉起来时,靠通知把用户带到这一步:
|
||||
// 先把待确认的安装框弹出来,装完再重新授权录屏(录制授权随旧进程一起失效了)。
|
||||
val pendingInstall = UpdateInstaller.consumePendingConfirm()
|
||||
@@ -49,6 +46,33 @@ class MainActivity : AppCompatActivity() {
|
||||
// MA-01 fix: 系统重建 Activity 时 savedInstanceState != null,
|
||||
// 此时 CaptureService 可能已在运行,不重复弹权限对话框
|
||||
if (savedInstanceState != null) { finish(); return }
|
||||
|
||||
val missing = ArrayList<String>()
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU &&
|
||||
checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) {
|
||||
// 升级完成/待确认的提示靠通知兜底,拒绝了也不影响主流程
|
||||
missing.add(Manifest.permission.POST_NOTIFICATIONS)
|
||||
}
|
||||
if (checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
|
||||
// 语音监听(COMMAND_AUDIO)要麦克风;拒绝了不影响其它功能,
|
||||
// 主控打开语音监听时客户端会回一条「麦克风不可用」的提示
|
||||
missing.add(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
if (missing.isNotEmpty()) {
|
||||
requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS)
|
||||
return
|
||||
}
|
||||
startProjection()
|
||||
}
|
||||
|
||||
override fun onRequestPermissionsResult(
|
||||
requestCode: Int, permissions: Array<out String>, grantResults: IntArray) {
|
||||
super.onRequestPermissionsResult(requestCode, permissions, grantResults)
|
||||
// 权限给不给都要继续:录屏授权本身是可选的(拒绝了 CaptureService 照常连服务端)
|
||||
if (requestCode == REQUEST_PERMISSIONS) startProjection()
|
||||
}
|
||||
|
||||
private fun startProjection() {
|
||||
projectionManager = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
startActivityForResult(projectionManager.createScreenCaptureIntent(), REQUEST_MEDIA_PROJECTION)
|
||||
}
|
||||
|
||||
@@ -35,4 +35,10 @@ object YamaBridge {
|
||||
* 没有活跃的交谈子连接时返回 false。
|
||||
*/
|
||||
external fun nativeChatSend(data: ByteArray): Boolean
|
||||
|
||||
/**
|
||||
* 语音监听:把 AudioRecord 采集的 PCM(8kHz/16bit/单声道)交给 native 编码成
|
||||
* GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。
|
||||
*/
|
||||
external fun nativeAudioSendPcm(samples: ShortArray)
|
||||
}
|
||||
Reference in new issue
Block a user