Feature: support camera streaming on Android hosts

Android 摄像头(COMMAND_WEBCAM)此前未实现。客户端用 Camera2 ImageReader
直接输出 JPEG(1280x720、8fps、按传感器方向摆正),第一帧到达时从 JPEG
SOF 读出真实宽高再发 TOKEN_WEBCAM_BITMAPINFO(部分 HAL 会按方向物理旋转
导致宽高互换,以实际帧为准可避免服务端尺寸不符丢帧黑屏),之后每帧
[TOKEN_WEBCAM_DIB][isCompress=1][JPGD][JPEG]。

服务端视频对话框识别 JPGD FOURCC,用 libjpeg 直接解成 bottom-up BGR24;
DrawDib 与「保存 AVI」两条路径都不用改,目标机也不需要安装 VFW 编解码器。
BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算。

注意:服务端不要引用 turbojpeg 的 tj* 符号——会拉进与 Debug 构建不兼容的
运行时,启动即 0xC0000005;解码统一走项目已在用的 jpeg_* API。

验证:模拟器虚拟相机出图,窗口标题 127.0.0.1 - Video 1280x720;关闭对话框
后 camera stopped,服务端稳定。

Model: deepseek-flash
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
yuanyuanxiangandClaude Code committed 2026-11-16 18:36:48 +01:00
1 parent 28c0c9a2a6
commit 998c16edc5
11 files changed
+739 -1

No files matched your search

@@ -0,0 +1,259 @@
package com.yama.client
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.graphics.ImageFormat
import android.hardware.camera2.CameraCaptureSession
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraDevice
import android.hardware.camera2.CameraManager
import android.hardware.camera2.CaptureRequest
import android.media.ImageReader
import android.os.Build
import android.os.Handler
import android.os.HandlerThread
import android.os.SystemClock
import android.util.Log
import android.view.Surface
import android.view.WindowManager
import androidx.core.content.ContextCompat
/**
* 摄像头管理(COMMAND_WEBCAM)的 Android 采集端。
*
* 走 Camera2 的 JPEG 输出:ImageReader 直接拿一帧完整 JPEG,交给 native 侧的
* AndroidVideoHandler 按 [TOKEN_WEBCAM_DIB][isCompress=1][fccHandler=JPGD][JPEG]
* 发给服务端;服务端用 turbojpeg 解码显示(不需要装任何 VFW 编解码器)。
*
* 为什么不用 MediaCodec H.264:服务端摄像头对话框原本走 VFW ICM 解码,H.264
* 需要系统装 x264vfw;JPEG 直解改动最小、且 Android 相机原生支持 JPEG 输出。
*
* 生命周期:C++ 收到 COMMAND_WEBCAM 后调 start(),子连接断开时调 stop()。
* start() 同步返回所选分辨率(可能为 null=无相机/无权限),真正开流是异步的:
* 服务端已按返回的分辨率发 BITMAPINFO,第一帧通常比对话框打开晚几百毫秒。
*/
object CameraCapture {
private const val TAG = "YAMA_CAM"
private const val TARGET_FPS = 8
private const val MIN_INTERVAL_MS = 1000L / TARGET_FPS
// 720p 足够看清,且 8fps 的 JPEG 上行在弱网下也不至于把主连接挤死
private const val MAX_W = 1280
private const val MAX_H = 720
private var thread: HandlerThread? = null
private var handler: Handler? = null
private var camera: CameraDevice? = null
private var session: CameraCaptureSession? = null
private var reader: ImageReader? = null
private var lastSent = 0L
@Volatile private var running = false
@Volatile private var currentSize: IntArray? = null
// 每次 start() 自增;旧相机的回调靠它识别「自己已经是上一代了」
@Volatile private var generation = 0
/**
* 打开摄像头。成功返回 [width, height];失败返回 null(无权限 / 无相机)。
* 重复调用安全(已在采集时直接返回当前分辨率)。
*/
@JvmStatic
@Synchronized
fun start(): IntArray? {
if (running) return currentSize
val ctx = CaptureService.instance ?: CaptureService.appContext ?: return null
if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.CAMERA)
!= PackageManager.PERMISSION_GRANTED) {
Log.w(TAG, "start: CAMERA permission not granted")
return null
}
val manager = ctx.getSystemService(Context.CAMERA_SERVICE) as? CameraManager ?: return null
val cameraId = pickCamera(manager) ?: return null
val size = pickSize(manager, cameraId) ?: return null
running = true
val gen = ++generation
currentSize = intArrayOf(size.first, size.second)
lastSent = 0L
val ht = HandlerThread("yama-camera")
ht.start()
thread = ht
handler = Handler(ht.looper)
val h = handler!!
runCatching {
manager.openCamera(cameraId, object : CameraDevice.StateCallback() {
override fun onOpened(cam: CameraDevice) {
if (!running || gen != generation) { cam.close(); return }
camera = cam
openSession(ctx, cam, size.first, size.second, gen)
}
override fun onDisconnected(cam: CameraDevice) {
Log.w(TAG, "camera disconnected")
cam.close()
if (camera === cam) camera = null
}
override fun onError(cam: CameraDevice, error: Int) {
Log.e(TAG, "camera error=$error")
cam.close()
if (camera === cam) camera = null
// 让 native 侧知道这路会话没有画面,避免服务端对话框一直黑着
runCatching { YamaBridge.nativeOnCameraError("Camera error $error") }
}
}, h)
}.onFailure {
Log.e(TAG, "openCamera failed: $it")
stop()
return null
}
Log.i(TAG, "start: camera=$cameraId size=${size.first}x${size.second}")
return currentSize
}
@JvmStatic
@Synchronized
fun stop() {
running = false
currentSize = null
// 先把当前这一代的句柄摘下来,避免删到下一次 start() 新建的 handler/camera:
// stop 是 post 到旧 HandlerThread 上执行的,执行前 start() 可能已经跑完了。
val h = handler
val s = session
val c = camera
val r = reader
val ht = thread
session = null
camera = null
reader = null
handler = null
thread = null
val closer = {
runCatching { s?.close() }
runCatching { c?.close() }
runCatching { r?.close() }
runCatching { ht?.quitSafely() }
Log.i(TAG, "camera stopped")
Unit
}
if (h != null) h.post(closer) else closer()
}
private fun openSession(ctx: Context, cam: CameraDevice, width: Int, height: Int, gen: Int) {
// 这些调用跑在相机 HandlerThread 上,异常逃出去会直接杀进程
val r = try {
ImageReader.newInstance(width, height, ImageFormat.JPEG, 3)
} catch (e: Exception) {
Log.e(TAG, "ImageReader.newInstance failed: $e")
YamaBridge.nativeOnCameraError("Camera init failed")
return
}
reader = r
r.setOnImageAvailableListener({ ir ->
val image = try { ir.acquireLatestImage() } catch (e: Exception) { null } ?: return@setOnImageAvailableListener
try {
if (!running || gen != generation) return@setOnImageAvailableListener
val now = SystemClock.elapsedRealtime()
if (now - lastSent < MIN_INTERVAL_MS) return@setOnImageAvailableListener
lastSent = now
val buf = image.planes[0].buffer
val bytes = ByteArray(buf.remaining())
buf.get(bytes)
YamaBridge.nativeOnCameraJpeg(bytes)
} catch (e: Exception) {
Log.w(TAG, "frame failed: $e")
} finally {
image.close()
}
}, handler)
val surfaces = listOf(r.surface)
val callback = object : CameraCaptureSession.StateCallback() {
override fun onConfigured(s: CameraCaptureSession) {
if (!running || gen != generation) { s.close(); return }
session = s
runCatching {
val req = cam.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
req.addTarget(r.surface)
req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
req.set(CaptureRequest.JPEG_ORIENTATION, orientation(ctx, cam))
s.setRepeatingRequest(req.build(), null, handler)
Log.i(TAG, "repeating request started")
}.onFailure {
Log.e(TAG, "setRepeatingRequest failed: $it")
YamaBridge.nativeOnCameraError("Camera stream failed")
}
}
override fun onConfigureFailed(s: CameraCaptureSession) {
Log.e(TAG, "capture session configure failed")
YamaBridge.nativeOnCameraError("Camera session failed")
}
}
// createCaptureSession 在相机线程上执行,抛异常同样会杀进程
try {
@Suppress("DEPRECATION")
cam.createCaptureSession(surfaces, callback, handler)
} catch (e: Exception) {
Log.e(TAG, "createCaptureSession failed: $e")
YamaBridge.nativeOnCameraError("Camera session failed")
}
}
/** 让 JPEG 按当前屏幕方向摆正(手机竖屏时传感器画面是横的)。 */
private fun orientation(ctx: Context, cam: CameraDevice): Int {
val chars = runCatching {
(ctx.getSystemService(Context.CAMERA_SERVICE) as CameraManager)
.getCameraCharacteristics(cam.id)
}.getOrNull() ?: return 0
val sensor = chars.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
val facing = chars.get(CameraCharacteristics.LENS_FACING)
@Suppress("DEPRECATION")
val wm = ctx.getSystemService(Context.WINDOW_SERVICE) as? WindowManager
@Suppress("DEPRECATION")
val rotation = runCatching { wm?.defaultDisplay?.rotation }.getOrNull() ?: Surface.ROTATION_0
val device = when (rotation) {
Surface.ROTATION_90 -> 90
Surface.ROTATION_180 -> 180
Surface.ROTATION_270 -> 270
else -> 0
}
return if (facing == CameraCharacteristics.LENS_FACING_FRONT)
(sensor + device) % 360
else
(sensor - device + 360) % 360
}
private fun pickCamera(manager: CameraManager): String? {
val ids = runCatching { manager.cameraIdList }.getOrNull() ?: return null
if (ids.isEmpty()) return null
// 优先后摄;电视盒子/无后摄设备退到任意一个
for (id in ids) {
val facing = runCatching {
manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING)
}.getOrNull()
if (facing == CameraCharacteristics.LENS_FACING_BACK) return id
}
return ids.first()
}
private fun pickSize(manager: CameraManager, id: String): Pair<Int, Int>? {
val chars = runCatching { manager.getCameraCharacteristics(id) }.getOrNull() ?: return null
val map = chars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) ?: return null
val sizes = runCatching { map.getOutputSizes(ImageFormat.JPEG) }.getOrNull() ?: return null
if (sizes.isEmpty()) return null
val fit = sizes.filter { it.width <= MAX_W && it.height <= MAX_H }
// 没有 ≤720p 的档位时选最小的一档:直接挑最大档会以 5-12 MP 推流,
// 8fps 就能把控制连接挤死
val best = if (fit.isNotEmpty()) fit.maxByOrNull { it.width * it.height }
else sizes.minByOrNull { it.width * it.height }
best ?: return null
return best.width to best.height
}
}
@@ -555,6 +555,12 @@ class CaptureService : Service() {
checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
}
// 摄像头管理(COMMAND_WEBCAM)要在后台用相机:Android 11+ 要求前台服务带 camera 类型。
// 没授权时不能带,否则 startForeground 直接抛 SecurityException。
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R &&
checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA
}
try {
startForeground(NOTIF_ID, buildNotification(), types)
} catch (e: Exception) {
@@ -128,6 +128,10 @@ class MainActivity : AppCompatActivity() {
// 主控打开语音监听时客户端会回一条「麦克风不可用」的提示
missing.add(Manifest.permission.RECORD_AUDIO)
}
if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
// 摄像头管理(COMMAND_WEBCAM);拒绝后主控打开视频管理会收到「摄像头不可用」
missing.add(Manifest.permission.CAMERA)
}
if (missing.isNotEmpty()) {
requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS)
return
@@ -41,4 +41,13 @@ object YamaBridge {
* GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。
*/
external fun nativeAudioSendPcm(samples: ShortArray)
/**
* 摄像头管理:Camera2 每输出一帧 JPEG,交给 native 封装成
* [TOKEN_WEBCAM_DIB][isCompress][fcc][JPEG] 发给服务端。
*/
external fun nativeOnCameraJpeg(data: ByteArray)
/** 摄像头采集失败(无权限 / 被占用 / 会话配置失败),回报主控 */
external fun nativeOnCameraError(msg: String)
}