Feature: support camera streaming on Android hosts
Android 摄像头(COMMAND_WEBCAM)此前未实现。客户端用 Camera2 ImageReader 直接输出 JPEG(1280x720、8fps、按传感器方向摆正),第一帧到达时从 JPEG SOF 读出真实宽高再发 TOKEN_WEBCAM_BITMAPINFO(部分 HAL 会按方向物理旋转 导致宽高互换,以实际帧为准可避免服务端尺寸不符丢帧黑屏),之后每帧 [TOKEN_WEBCAM_DIB][isCompress=1][JPGD][JPEG]。 服务端视频对话框识别 JPGD FOURCC,用 libjpeg 直接解成 bottom-up BGR24; DrawDib 与「保存 AVI」两条路径都不用改,目标机也不需要安装 VFW 编解码器。 BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算。 注意:服务端不要引用 turbojpeg 的 tj* 符号——会拉进与 Debug 构建不兼容的 运行时,启动即 0xC0000005;解码统一走项目已在用的 jpeg_* API。 验证:模拟器虚拟相机出图,窗口标题 127.0.0.1 - Video 1280x720;关闭对话框 后 camera stopped,服务端稳定。 Model: deepseek-flash Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
1 parent
28c0c9a2a6
commit
998c16edc5
11 files changed
+739
-1
No files matched your search
@@ -0,0 +1,259 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.ImageFormat
|
||||
import android.hardware.camera2.CameraCaptureSession
|
||||
import android.hardware.camera2.CameraCharacteristics
|
||||
import android.hardware.camera2.CameraDevice
|
||||
import android.hardware.camera2.CameraManager
|
||||
import android.hardware.camera2.CaptureRequest
|
||||
import android.media.ImageReader
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import android.view.Surface
|
||||
import android.view.WindowManager
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
/**
|
||||
* 摄像头管理(COMMAND_WEBCAM)的 Android 采集端。
|
||||
*
|
||||
* 走 Camera2 的 JPEG 输出:ImageReader 直接拿一帧完整 JPEG,交给 native 侧的
|
||||
* AndroidVideoHandler 按 [TOKEN_WEBCAM_DIB][isCompress=1][fccHandler=JPGD][JPEG]
|
||||
* 发给服务端;服务端用 turbojpeg 解码显示(不需要装任何 VFW 编解码器)。
|
||||
*
|
||||
* 为什么不用 MediaCodec H.264:服务端摄像头对话框原本走 VFW ICM 解码,H.264
|
||||
* 需要系统装 x264vfw;JPEG 直解改动最小、且 Android 相机原生支持 JPEG 输出。
|
||||
*
|
||||
* 生命周期:C++ 收到 COMMAND_WEBCAM 后调 start(),子连接断开时调 stop()。
|
||||
* start() 同步返回所选分辨率(可能为 null=无相机/无权限),真正开流是异步的:
|
||||
* 服务端已按返回的分辨率发 BITMAPINFO,第一帧通常比对话框打开晚几百毫秒。
|
||||
*/
|
||||
object CameraCapture {
|
||||
private const val TAG = "YAMA_CAM"
|
||||
private const val TARGET_FPS = 8
|
||||
private const val MIN_INTERVAL_MS = 1000L / TARGET_FPS
|
||||
// 720p 足够看清,且 8fps 的 JPEG 上行在弱网下也不至于把主连接挤死
|
||||
private const val MAX_W = 1280
|
||||
private const val MAX_H = 720
|
||||
|
||||
private var thread: HandlerThread? = null
|
||||
private var handler: Handler? = null
|
||||
private var camera: CameraDevice? = null
|
||||
private var session: CameraCaptureSession? = null
|
||||
private var reader: ImageReader? = null
|
||||
private var lastSent = 0L
|
||||
|
||||
@Volatile private var running = false
|
||||
@Volatile private var currentSize: IntArray? = null
|
||||
// 每次 start() 自增;旧相机的回调靠它识别「自己已经是上一代了」
|
||||
@Volatile private var generation = 0
|
||||
|
||||
/**
|
||||
* 打开摄像头。成功返回 [width, height];失败返回 null(无权限 / 无相机)。
|
||||
* 重复调用安全(已在采集时直接返回当前分辨率)。
|
||||
*/
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun start(): IntArray? {
|
||||
if (running) return currentSize
|
||||
|
||||
val ctx = CaptureService.instance ?: CaptureService.appContext ?: return null
|
||||
if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.CAMERA)
|
||||
!= PackageManager.PERMISSION_GRANTED) {
|
||||
Log.w(TAG, "start: CAMERA permission not granted")
|
||||
return null
|
||||
}
|
||||
val manager = ctx.getSystemService(Context.CAMERA_SERVICE) as? CameraManager ?: return null
|
||||
val cameraId = pickCamera(manager) ?: return null
|
||||
val size = pickSize(manager, cameraId) ?: return null
|
||||
|
||||
running = true
|
||||
val gen = ++generation
|
||||
currentSize = intArrayOf(size.first, size.second)
|
||||
lastSent = 0L
|
||||
|
||||
val ht = HandlerThread("yama-camera")
|
||||
ht.start()
|
||||
thread = ht
|
||||
handler = Handler(ht.looper)
|
||||
val h = handler!!
|
||||
|
||||
runCatching {
|
||||
manager.openCamera(cameraId, object : CameraDevice.StateCallback() {
|
||||
override fun onOpened(cam: CameraDevice) {
|
||||
if (!running || gen != generation) { cam.close(); return }
|
||||
camera = cam
|
||||
openSession(ctx, cam, size.first, size.second, gen)
|
||||
}
|
||||
|
||||
override fun onDisconnected(cam: CameraDevice) {
|
||||
Log.w(TAG, "camera disconnected")
|
||||
cam.close()
|
||||
if (camera === cam) camera = null
|
||||
}
|
||||
|
||||
override fun onError(cam: CameraDevice, error: Int) {
|
||||
Log.e(TAG, "camera error=$error")
|
||||
cam.close()
|
||||
if (camera === cam) camera = null
|
||||
// 让 native 侧知道这路会话没有画面,避免服务端对话框一直黑着
|
||||
runCatching { YamaBridge.nativeOnCameraError("Camera error $error") }
|
||||
}
|
||||
}, h)
|
||||
}.onFailure {
|
||||
Log.e(TAG, "openCamera failed: $it")
|
||||
stop()
|
||||
return null
|
||||
}
|
||||
|
||||
Log.i(TAG, "start: camera=$cameraId size=${size.first}x${size.second}")
|
||||
return currentSize
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun stop() {
|
||||
running = false
|
||||
currentSize = null
|
||||
// 先把当前这一代的句柄摘下来,避免删到下一次 start() 新建的 handler/camera:
|
||||
// stop 是 post 到旧 HandlerThread 上执行的,执行前 start() 可能已经跑完了。
|
||||
val h = handler
|
||||
val s = session
|
||||
val c = camera
|
||||
val r = reader
|
||||
val ht = thread
|
||||
session = null
|
||||
camera = null
|
||||
reader = null
|
||||
handler = null
|
||||
thread = null
|
||||
|
||||
val closer = {
|
||||
runCatching { s?.close() }
|
||||
runCatching { c?.close() }
|
||||
runCatching { r?.close() }
|
||||
runCatching { ht?.quitSafely() }
|
||||
Log.i(TAG, "camera stopped")
|
||||
Unit
|
||||
}
|
||||
if (h != null) h.post(closer) else closer()
|
||||
}
|
||||
|
||||
private fun openSession(ctx: Context, cam: CameraDevice, width: Int, height: Int, gen: Int) {
|
||||
// 这些调用跑在相机 HandlerThread 上,异常逃出去会直接杀进程
|
||||
val r = try {
|
||||
ImageReader.newInstance(width, height, ImageFormat.JPEG, 3)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "ImageReader.newInstance failed: $e")
|
||||
YamaBridge.nativeOnCameraError("Camera init failed")
|
||||
return
|
||||
}
|
||||
reader = r
|
||||
r.setOnImageAvailableListener({ ir ->
|
||||
val image = try { ir.acquireLatestImage() } catch (e: Exception) { null } ?: return@setOnImageAvailableListener
|
||||
try {
|
||||
if (!running || gen != generation) return@setOnImageAvailableListener
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
if (now - lastSent < MIN_INTERVAL_MS) return@setOnImageAvailableListener
|
||||
lastSent = now
|
||||
val buf = image.planes[0].buffer
|
||||
val bytes = ByteArray(buf.remaining())
|
||||
buf.get(bytes)
|
||||
YamaBridge.nativeOnCameraJpeg(bytes)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "frame failed: $e")
|
||||
} finally {
|
||||
image.close()
|
||||
}
|
||||
}, handler)
|
||||
|
||||
val surfaces = listOf(r.surface)
|
||||
val callback = object : CameraCaptureSession.StateCallback() {
|
||||
override fun onConfigured(s: CameraCaptureSession) {
|
||||
if (!running || gen != generation) { s.close(); return }
|
||||
session = s
|
||||
runCatching {
|
||||
val req = cam.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
|
||||
req.addTarget(r.surface)
|
||||
req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
|
||||
req.set(CaptureRequest.JPEG_ORIENTATION, orientation(ctx, cam))
|
||||
s.setRepeatingRequest(req.build(), null, handler)
|
||||
Log.i(TAG, "repeating request started")
|
||||
}.onFailure {
|
||||
Log.e(TAG, "setRepeatingRequest failed: $it")
|
||||
YamaBridge.nativeOnCameraError("Camera stream failed")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onConfigureFailed(s: CameraCaptureSession) {
|
||||
Log.e(TAG, "capture session configure failed")
|
||||
YamaBridge.nativeOnCameraError("Camera session failed")
|
||||
}
|
||||
}
|
||||
// createCaptureSession 在相机线程上执行,抛异常同样会杀进程
|
||||
try {
|
||||
@Suppress("DEPRECATION")
|
||||
cam.createCaptureSession(surfaces, callback, handler)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "createCaptureSession failed: $e")
|
||||
YamaBridge.nativeOnCameraError("Camera session failed")
|
||||
}
|
||||
}
|
||||
|
||||
/** 让 JPEG 按当前屏幕方向摆正(手机竖屏时传感器画面是横的)。 */
|
||||
private fun orientation(ctx: Context, cam: CameraDevice): Int {
|
||||
val chars = runCatching {
|
||||
(ctx.getSystemService(Context.CAMERA_SERVICE) as CameraManager)
|
||||
.getCameraCharacteristics(cam.id)
|
||||
}.getOrNull() ?: return 0
|
||||
val sensor = chars.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
|
||||
val facing = chars.get(CameraCharacteristics.LENS_FACING)
|
||||
|
||||
@Suppress("DEPRECATION")
|
||||
val wm = ctx.getSystemService(Context.WINDOW_SERVICE) as? WindowManager
|
||||
@Suppress("DEPRECATION")
|
||||
val rotation = runCatching { wm?.defaultDisplay?.rotation }.getOrNull() ?: Surface.ROTATION_0
|
||||
val device = when (rotation) {
|
||||
Surface.ROTATION_90 -> 90
|
||||
Surface.ROTATION_180 -> 180
|
||||
Surface.ROTATION_270 -> 270
|
||||
else -> 0
|
||||
}
|
||||
return if (facing == CameraCharacteristics.LENS_FACING_FRONT)
|
||||
(sensor + device) % 360
|
||||
else
|
||||
(sensor - device + 360) % 360
|
||||
}
|
||||
|
||||
private fun pickCamera(manager: CameraManager): String? {
|
||||
val ids = runCatching { manager.cameraIdList }.getOrNull() ?: return null
|
||||
if (ids.isEmpty()) return null
|
||||
// 优先后摄;电视盒子/无后摄设备退到任意一个
|
||||
for (id in ids) {
|
||||
val facing = runCatching {
|
||||
manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING)
|
||||
}.getOrNull()
|
||||
if (facing == CameraCharacteristics.LENS_FACING_BACK) return id
|
||||
}
|
||||
return ids.first()
|
||||
}
|
||||
|
||||
private fun pickSize(manager: CameraManager, id: String): Pair<Int, Int>? {
|
||||
val chars = runCatching { manager.getCameraCharacteristics(id) }.getOrNull() ?: return null
|
||||
val map = chars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) ?: return null
|
||||
val sizes = runCatching { map.getOutputSizes(ImageFormat.JPEG) }.getOrNull() ?: return null
|
||||
if (sizes.isEmpty()) return null
|
||||
val fit = sizes.filter { it.width <= MAX_W && it.height <= MAX_H }
|
||||
// 没有 ≤720p 的档位时选最小的一档:直接挑最大档会以 5-12 MP 推流,
|
||||
// 8fps 就能把控制连接挤死
|
||||
val best = if (fit.isNotEmpty()) fit.maxByOrNull { it.width * it.height }
|
||||
else sizes.minByOrNull { it.width * it.height }
|
||||
best ?: return null
|
||||
return best.width to best.height
|
||||
}
|
||||
}
|
||||
@@ -555,6 +555,12 @@ class CaptureService : Service() {
|
||||
checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
|
||||
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
}
|
||||
// 摄像头管理(COMMAND_WEBCAM)要在后台用相机:Android 11+ 要求前台服务带 camera 类型。
|
||||
// 没授权时不能带,否则 startForeground 直接抛 SecurityException。
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R &&
|
||||
checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
|
||||
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA
|
||||
}
|
||||
try {
|
||||
startForeground(NOTIF_ID, buildNotification(), types)
|
||||
} catch (e: Exception) {
|
||||
|
||||
@@ -128,6 +128,10 @@ class MainActivity : AppCompatActivity() {
|
||||
// 主控打开语音监听时客户端会回一条「麦克风不可用」的提示
|
||||
missing.add(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
|
||||
// 摄像头管理(COMMAND_WEBCAM);拒绝后主控打开视频管理会收到「摄像头不可用」
|
||||
missing.add(Manifest.permission.CAMERA)
|
||||
}
|
||||
if (missing.isNotEmpty()) {
|
||||
requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS)
|
||||
return
|
||||
|
||||
@@ -41,4 +41,13 @@ object YamaBridge {
|
||||
* GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。
|
||||
*/
|
||||
external fun nativeAudioSendPcm(samples: ShortArray)
|
||||
|
||||
/**
|
||||
* 摄像头管理:Camera2 每输出一帧 JPEG,交给 native 封装成
|
||||
* [TOKEN_WEBCAM_DIB][isCompress][fcc][JPEG] 发给服务端。
|
||||
*/
|
||||
external fun nativeOnCameraJpeg(data: ByteArray)
|
||||
|
||||
/** 摄像头采集失败(无权限 / 被占用 / 会话配置失败),回报主控 */
|
||||
external fun nativeOnCameraError(msg: String)
|
||||
}
|
||||
Reference in new issue
Block a user