Feat: Android client Phase 0-3 full implementation
This commit was merged in pull request #3.
This commit is contained in:
yuanyuanxiang
360
android/app/src/main/java/com/yama/client/CaptureService.kt
Normal file
360
android/app/src/main/java/com/yama/client/CaptureService.kt
Normal file
@@ -0,0 +1,360 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.app.Notification
|
||||
import android.app.NotificationChannel
|
||||
import android.app.NotificationManager
|
||||
import android.app.Service
|
||||
import android.content.Intent
|
||||
import android.hardware.display.DisplayManager
|
||||
import android.hardware.display.VirtualDisplay
|
||||
import android.media.MediaCodec
|
||||
import android.media.MediaCodecInfo
|
||||
import android.media.MediaFormat
|
||||
import android.media.projection.MediaProjection
|
||||
import android.media.projection.MediaProjectionManager
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.IBinder
|
||||
import android.os.Looper
|
||||
import android.provider.Settings
|
||||
import android.util.DisplayMetrics
|
||||
import android.util.Log
|
||||
import android.view.WindowManager
|
||||
|
||||
class CaptureService : Service() {
|
||||
|
||||
companion object {
|
||||
const val TAG = "YAMA"
|
||||
const val CHANNEL_ID = "yama_service"
|
||||
const val NOTIF_ID = 1
|
||||
|
||||
const val EXTRA_SERVER_IP = "server_ip"
|
||||
const val EXTRA_SERVER_PORT = "server_port"
|
||||
const val EXTRA_PROJECTION_RESULT = "projection_result"
|
||||
const val EXTRA_PROJECTION_DATA = "projection_data"
|
||||
|
||||
private const val MAX_LONG_SIDE = 1080
|
||||
|
||||
// CS-01 fix: 通过 idrHandler.post 序列化到主线程，避免 TOCTOU 竞争
|
||||
@Volatile var instance: CaptureService? = null
|
||||
|
||||
@JvmStatic
|
||||
fun requestIdr() {
|
||||
val svc = instance ?: return
|
||||
svc.idrHandler.post {
|
||||
runCatching {
|
||||
val p = Bundle()
|
||||
p.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0)
|
||||
svc.mediaCodec?.setParameters(p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 静止屏幕黑屏修复：SurfaceFlinger 在无脏区时不向 VirtualDisplay 推帧，
|
||||
* 导致编码器无输入，REQUEST_SYNC_FRAME 悬空排队，浏览器等 10-60 秒才见画面。
|
||||
* 解决方法：将 VirtualDisplay 宽度临时改变 2px，触发一次强制合成，
|
||||
* 编码器随即得到输入帧并消费已排队的 IDR 请求。150ms 后恢复原始尺寸。
|
||||
*/
|
||||
@JvmStatic
|
||||
fun forceFirstFrame() {
|
||||
val svc = instance ?: return
|
||||
svc.idrHandler.post {
|
||||
runCatching {
|
||||
val vd = svc.virtualDisplay ?: return@runCatching
|
||||
val w = ControlService.encW
|
||||
val h = ControlService.encH
|
||||
val dpi = svc.resources.displayMetrics.densityDpi
|
||||
Log.d(TAG, "forceFirstFrame: resize ${w}x${h} → ${w+2}x${h} to trigger composition")
|
||||
vd.resize(w + 2, h, dpi)
|
||||
svc.idrHandler.postDelayed({
|
||||
runCatching { vd.resize(w, h, dpi) }
|
||||
Log.d(TAG, "forceFirstFrame: restored ${w}x${h}")
|
||||
}, 150)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private var mediaProjection: MediaProjection? = null
|
||||
private var virtualDisplay: VirtualDisplay? = null
|
||||
private var mediaCodec: MediaCodec? = null
|
||||
|
||||
private val idrHandler = Handler(Looper.getMainLooper())
|
||||
// CS-04 fix: runCatching 包住 setParameters，防止 codec 已停止时抛异常
|
||||
private val idrRunnable = object : Runnable {
|
||||
override fun run() {
|
||||
runCatching {
|
||||
val p = Bundle()
|
||||
p.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0)
|
||||
mediaCodec?.setParameters(p)
|
||||
}
|
||||
idrHandler.postDelayed(this, 1000)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
instance = this
|
||||
createNotificationChannel()
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
startForeground(NOTIF_ID, buildNotification(),
|
||||
android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION)
|
||||
} else {
|
||||
startForeground(NOTIF_ID, buildNotification())
|
||||
}
|
||||
}
|
||||
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
// CS-03 fix: START_STICKY 重启时 intent 为 null，必须在最前面检查
|
||||
if (intent == null) return START_NOT_STICKY
|
||||
val ip = intent.getStringExtra(EXTRA_SERVER_IP) ?: return START_NOT_STICKY
|
||||
val port = intent.getIntExtra(EXTRA_SERVER_PORT, 443)
|
||||
|
||||
val androidId = Settings.Secure.getString(contentResolver, Settings.Secure.ANDROID_ID) ?: ""
|
||||
val model = "${Build.MANUFACTURER} ${Build.MODEL}"
|
||||
val osVer = "Android ${Build.VERSION.RELEASE} (API ${Build.VERSION.SDK_INT})"
|
||||
val (sw, sh) = getPhysicalResolution()
|
||||
val res = "1:${sw}*${sh}"
|
||||
val user = Build.USER
|
||||
|
||||
Log.i(TAG, "CaptureService: server=$ip:$port device=$model res=$res")
|
||||
|
||||
// YB-01 fix: so 库加载失败是 Error，不被 catch(Exception) 捕获，需单独处理
|
||||
try {
|
||||
YamaBridge.nativeInit(ip, port, androidId, model, osVer, res, user, packageCodePath ?: "")
|
||||
} catch (e: UnsatisfiedLinkError) {
|
||||
Log.e(TAG, "Native library load failed: $e")
|
||||
stopSelf()
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
|
||||
val projResult = intent.getIntExtra(EXTRA_PROJECTION_RESULT, -1)
|
||||
val projData = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU)
|
||||
intent.getParcelableExtra(EXTRA_PROJECTION_DATA, Intent::class.java)
|
||||
else
|
||||
@Suppress("DEPRECATION") intent.getParcelableExtra(EXTRA_PROJECTION_DATA)
|
||||
|
||||
if (projResult == android.app.Activity.RESULT_OK && projData != null) {
|
||||
val pm = getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager
|
||||
val mp = pm.getMediaProjection(projResult, projData)
|
||||
mediaProjection = mp
|
||||
// CS-05 fix: 分辨率为 0 时不启动采集，避免 MediaCodec 配置崩溃
|
||||
if (sw > 0 && sh > 0) {
|
||||
startCapture(mp, sw, sh)
|
||||
} else {
|
||||
Log.e(TAG, "Invalid screen resolution ${sw}x${sh}, skipping capture")
|
||||
}
|
||||
} else {
|
||||
Log.w(TAG, "No MediaProjection token, skipping screen capture")
|
||||
}
|
||||
|
||||
return START_STICKY
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
super.onDestroy()
|
||||
instance = null
|
||||
stopCapture()
|
||||
YamaBridge.nativeStop()
|
||||
Log.i(TAG, "CaptureService destroyed")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- MediaProjection + MediaCodec
|
||||
|
||||
private fun startCapture(mp: MediaProjection, physW: Int, physH: Int) {
|
||||
val (encW, encH) = scaleDown(physW, physH, MAX_LONG_SIDE)
|
||||
val dpi = resources.displayMetrics.densityDpi
|
||||
|
||||
Log.i(TAG, "Capture: phys=${physW}x${physH} enc=${encW}x${encH} dpi=$dpi")
|
||||
|
||||
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC)
|
||||
val surface = configureEncoder(codec, encW, encH)
|
||||
|
||||
var configData: ByteArray? = null
|
||||
|
||||
codec.setCallback(object : MediaCodec.Callback() {
|
||||
override fun onInputBufferAvailable(mc: MediaCodec, index: Int) { /* Surface 模式不用 */ }
|
||||
|
||||
override fun onOutputBufferAvailable(mc: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {
|
||||
val buf = mc.getOutputBuffer(index)
|
||||
if (buf == null || info.size == 0) {
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
return
|
||||
}
|
||||
buf.position(info.offset)
|
||||
buf.limit(info.offset + info.size)
|
||||
|
||||
if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
|
||||
configData = ByteArray(info.size).also { buf.get(it) }
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
return
|
||||
}
|
||||
|
||||
val isKey = info.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME != 0
|
||||
val frame = ByteArray(info.size).also { buf.get(it) }
|
||||
mc.releaseOutputBuffer(index, false)
|
||||
|
||||
// 部分安卓硬件编码器（如某些高通实现）对 REQUEST_SYNC_FRAME 强制产生的
|
||||
// IDR 帧不设置 BUFFER_FLAG_KEY_FRAME，导致 SendLoop 将所有帧视为 P 帧丢弃。
|
||||
// 补充扫描 NALU 字节来识别真正的关键帧，确保首帧能顺利发出。
|
||||
val actualIsKey = isKey || isNaluKeyframe(frame)
|
||||
val payload = if (actualIsKey && configData != null) configData!! + frame else frame
|
||||
YamaBridge.nativeOnH264Frame(payload, 0, payload.size, actualIsKey)
|
||||
}
|
||||
|
||||
override fun onError(mc: MediaCodec, e: MediaCodec.CodecException) {
|
||||
Log.e(TAG, "MediaCodec error: $e")
|
||||
}
|
||||
|
||||
override fun onOutputFormatChanged(mc: MediaCodec, format: MediaFormat) {
|
||||
Log.i(TAG, "Output format: $format")
|
||||
}
|
||||
})
|
||||
|
||||
codec.start()
|
||||
mediaCodec = codec
|
||||
|
||||
idrHandler.postDelayed(idrRunnable, 500)
|
||||
|
||||
// Android 14+ requires callback registered before createVirtualDisplay()
|
||||
mp.registerCallback(object : MediaProjection.Callback() {
|
||||
override fun onStop() {
|
||||
Log.i(TAG, "MediaProjection stopped")
|
||||
stopCapture()
|
||||
stopSelf()
|
||||
}
|
||||
}, null)
|
||||
|
||||
virtualDisplay = mp.createVirtualDisplay(
|
||||
"YAMA",
|
||||
encW, encH, dpi,
|
||||
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
||||
surface, null, null
|
||||
)
|
||||
|
||||
YamaBridge.nativeSetScreenSize(encW, encH)
|
||||
|
||||
ControlService.physW = physW
|
||||
ControlService.physH = physH
|
||||
ControlService.encW = encW
|
||||
ControlService.encH = encH
|
||||
|
||||
Log.i(TAG, "MediaCodec + VirtualDisplay started ${encW}x${encH}")
|
||||
}
|
||||
|
||||
private fun buildFmt(w: Int, h: Int, lowLatency: Boolean, baseline: Boolean) =
|
||||
MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, w, h).apply {
|
||||
setInteger(MediaFormat.KEY_COLOR_FORMAT,
|
||||
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
|
||||
setInteger(MediaFormat.KEY_BIT_RATE, 4_000_000)
|
||||
setInteger(MediaFormat.KEY_FRAME_RATE, 30)
|
||||
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2)
|
||||
setInteger(MediaFormat.KEY_PRIORITY, 0)
|
||||
setInteger(MediaFormat.KEY_OPERATING_RATE, 30)
|
||||
if (lowLatency) {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N)
|
||||
setInteger(MediaFormat.KEY_LATENCY, 0)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q)
|
||||
setInteger(MediaFormat.KEY_LOW_LATENCY, 1)
|
||||
}
|
||||
if (baseline)
|
||||
setInteger(MediaFormat.KEY_PROFILE,
|
||||
MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline)
|
||||
}
|
||||
|
||||
// 逐级降级：低延迟+Baseline → 低延迟 → 默认，确保真机兼容
|
||||
private fun configureEncoder(codec: MediaCodec, w: Int, h: Int): android.view.Surface {
|
||||
val configs = listOf(
|
||||
true to true,
|
||||
true to false,
|
||||
false to false,
|
||||
)
|
||||
for ((ll, bl) in configs) {
|
||||
try {
|
||||
val fmt = buildFmt(w, h, ll, bl)
|
||||
codec.configure(fmt, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
||||
Log.i(TAG, "Encoder configured: lowLatency=$ll baseline=$bl")
|
||||
return codec.createInputSurface()
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "Encoder config failed (lowLatency=$ll baseline=$bl): $e, retrying…")
|
||||
try { codec.reset() } catch (_: Exception) {}
|
||||
}
|
||||
}
|
||||
throw RuntimeException("MediaCodec: all encoder configs failed")
|
||||
}
|
||||
|
||||
private fun stopCapture() {
|
||||
idrHandler.removeCallbacks(idrRunnable)
|
||||
runCatching { virtualDisplay?.release() }
|
||||
// CS-07 fix: stop 和 release 分开，stop 失败不会跳过 release 导致资源泄漏
|
||||
runCatching { mediaCodec?.stop() }
|
||||
runCatching { mediaCodec?.release() }
|
||||
runCatching { mediaProjection?.stop() }
|
||||
virtualDisplay = null
|
||||
mediaCodec = null
|
||||
mediaProjection = null
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- 辅助函数
|
||||
|
||||
private fun getPhysicalResolution(): Pair<Int, Int> {
|
||||
val wm = getSystemService(WINDOW_SERVICE) as WindowManager
|
||||
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
|
||||
val b = wm.currentWindowMetrics.bounds
|
||||
Pair(b.width(), b.height())
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
val dm = DisplayMetrics()
|
||||
@Suppress("DEPRECATION")
|
||||
wm.defaultDisplay.getRealMetrics(dm)
|
||||
Pair(dm.widthPixels, dm.heightPixels)
|
||||
}
|
||||
}
|
||||
|
||||
/** 扫描 Annex-B NALU 流，遇到 NAL type 5(IDR)、7(SPS)、8(PPS) 即判定为关键帧 */
|
||||
private fun isNaluKeyframe(data: ByteArray): Boolean {
|
||||
var i = 0
|
||||
while (i + 4 < data.size) {
|
||||
if (data[i] == 0.toByte() && data[i+1] == 0.toByte() &&
|
||||
data[i+2] == 0.toByte() && data[i+3] == 1.toByte()) {
|
||||
val nalType = data[i + 4].toInt() and 0x1F
|
||||
if (nalType == 5 || nalType == 7 || nalType == 8) return true
|
||||
i += 5
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 按长边上限等比缩小，宽高各自 2 对齐（H.264 要求） */
|
||||
private fun scaleDown(w: Int, h: Int, maxLong: Int): Pair<Int, Int> {
|
||||
val longSide = maxOf(w, h)
|
||||
if (longSide <= maxLong) return Pair(w and -2, h and -2)
|
||||
val scale = maxLong.toDouble() / longSide
|
||||
return Pair((w * scale).toInt() and -2, (h * scale).toInt() and -2)
|
||||
}
|
||||
|
||||
private fun createNotificationChannel() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
||||
val ch = NotificationChannel(CHANNEL_ID, "YAMA Service",
|
||||
NotificationManager.IMPORTANCE_LOW)
|
||||
getSystemService(NotificationManager::class.java)?.createNotificationChannel(ch)
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildNotification(): Notification {
|
||||
val b = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O)
|
||||
Notification.Builder(this, CHANNEL_ID)
|
||||
else
|
||||
@Suppress("DEPRECATION") Notification.Builder(this)
|
||||
return b.setContentTitle("YAMA")
|
||||
.setContentText("Screen monitoring active")
|
||||
.setSmallIcon(android.R.drawable.ic_menu_camera)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user