Feature: Implement H.264 and AV1 hardware encoding for remote control

Remark: Need to update FFmpeg static libraries to take effort
2026-05-28 11:41:33 +02:00
parent d1aa7a2c02
commit 8c7f612449
30 changed files with 2113 additions and 68 deletions
--- a/server/2015Remote/ScreenSpyDlg.cpp
+++ b/server/2015Remote/ScreenSpyDlg.cpp
@@ -43,6 +43,66 @@ IMPLEMENT_DYNAMIC(CScreenSpyDlg, CDialog)

 #define TIMER_ID 132

+// H.264 Annex B keyframe 探测：扫描 start code (00 00 01 / 00 00 00 01)，
+// 取后续 NAL header low 5 bits，命中 5 (IDR) / 7 (SPS) / 8 (PPS) 即认定为关键帧。
+static bool IsH264Keyframe(const uint8_t* data, size_t len)
+{
+    for (size_t i = 0; i + 4 < len; ++i) {
+        size_t nalOffset = 0;
+        if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1) {
+            nalOffset = i + 4;
+        } else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
+            nalOffset = i + 3;
+        } else {
+            continue;
+        }
+        if (nalOffset >= len) continue;
+        uint8_t nalType = data[nalOffset] & 0x1F;
+        if (nalType == 5 || nalType == 7 || nalType == 8) return true;
+    }
+    return false;
+}
+
+// AV1 OBU keyframe 探测：扫描 OBU 链，遇到 OBU_SEQUENCE_HEADER (type 1) 即认定为关键帧。
+// FFmpeg AV1 编码器在每个 IDR 前必定插入 SEQ HDR，因此该判定与 H.264 NAL 5/7/8 语义对齐。
+static bool IsAv1Keyframe(const uint8_t* data, size_t len)
+{
+    size_t pos = 0;
+    while (pos < len) {
+        uint8_t hdr = data[pos];
+        uint8_t obu_type = (hdr >> 3) & 0x0F;
+        bool has_ext  = (hdr & 0x04) != 0;
+        bool has_size = (hdr & 0x02) != 0;
+        if (obu_type == 1 /*OBU_SEQUENCE_HEADER*/) return true;
+        pos++;
+        if (has_ext) {
+            if (pos >= len) return false;
+            pos++;
+        }
+        if (!has_size) return false;  // 无 size 字段：OBU 占满到包尾，无法继续解析
+        // LEB128 size
+        uint64_t sz = 0;
+        for (int i = 0; i < 8; ++i) {
+            if (pos >= len) return false;
+            uint8_t b = data[pos++];
+            sz |= (uint64_t)(b & 0x7F) << (7 * i);
+            if ((b & 0x80) == 0) break;
+        }
+        if (pos + sz > len) return false;
+        pos += (size_t)sz;
+    }
+    return false;
+}
+
+// 首字节嗅探：H.264 Annex B 首字节恒为 0x00（起始码）；AV1 OBU header 首字节
+// bit7=0、bits[3:6]=obu_type 1-15，典型值 0x08-0x78，绝不为 0x00。
+// 一字节即可干净区分两套码流，无需协议字段或编码端协商。
+static bool IsAnyKeyframe(const uint8_t* data, size_t len)
+{
+    if (len == 0) return false;
+    return data[0] == 0x00 ? IsH264Keyframe(data, len) : IsAv1Keyframe(data, len);
+}
+
 // 静态成员变量定义
 int CScreenSpyDlg::s_nFastStretch = -1;  // -1 表示未初始化

@@ -675,6 +735,12 @@ BOOL CScreenSpyDlg::OnInitDialog()
        // 音频菜单项
        SysMenu->AppendMenuL(MF_STRING, IDM_AUDIO_TOGGLE, "系统音频(&U)");
        SysMenu->CheckMenuItem(IDM_AUDIO_TOGGLE, m_Settings.AudioEnabled ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->AppendMenuL(MF_STRING, IDM_ENABLE_H264_HARD, "启用 H264 硬编码");
+        SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
+        SysMenu->AppendMenuL(MF_STRING, IDM_ENABLE_AV1_HARD, "启用 AV1 硬编码");
+        SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);

        // 初始化勾选状态
        UpdateQualityMenuCheck(SysMenu);
@@ -1410,27 +1476,11 @@ VOID CScreenSpyDlg::DrawNextScreenDiff(bool keyFrame)
                    bChange = TRUE;
                }
            }
-            // Broadcast H264 frame to web clients (only for Web session dialogs)
-            // Format: [DeviceID:4][FrameType:1][DataLen:4][H264Data:N]
+            // Broadcast video frame to web clients (only for Web session dialogs)
+            // Format: [DeviceID:4][FrameType:1][DataLen:4][VideoData:N]
+            // 浏览器侧按首字节嗅探区分 H.264 / AV1，因此 packet 内不需要 codec 字段。
            if (m_bIsWebSession && NextScreenLength > 0 && WebService().IsRunning()) {
-                // Detect H264 keyframe by checking NAL unit type
-                // NAL type 5 = IDR slice (keyframe), NAL type 7 = SPS, NAL type 8 = PPS
-                bool isKeyFrame = false;
-                LPBYTE h264Data = (LPBYTE)NextScreenData;
-                for (ULONG i = 0; i + 4 < NextScreenLength; i++) {
-                    // Look for start code: 0x00 0x00 0x00 0x01 or 0x00 0x00 0x01
-                    if ((h264Data[i] == 0 && h264Data[i+1] == 0 && h264Data[i+2] == 0 && h264Data[i+3] == 1) ||
-                        (h264Data[i] == 0 && h264Data[i+1] == 0 && h264Data[i+2] == 1)) {
-                        int nalOffset = (h264Data[i+2] == 1) ? i + 3 : i + 4;
-                        if (nalOffset < (int)NextScreenLength) {
-                            int nalType = h264Data[nalOffset] & 0x1F;
-                            if (nalType == 5 || nalType == 7 || nalType == 8) {
-                                isKeyFrame = true;
-                                break;
-                            }
-                        }
-                    }
-                }
+                bool isKeyFrame = IsAnyKeyframe((const uint8_t*)NextScreenData, NextScreenLength);

                std::vector<uint8_t> packet(4 + 1 + 4 + NextScreenLength);
                uint32_t deviceIdLow = (uint32_t)(m_ClientID & 0xFFFFFFFF);
@@ -2134,6 +2184,26 @@ void CScreenSpyDlg::OnSysCommand(UINT nID, LPARAM lParam)
        }
        break;
    }
+    case IDM_ENABLE_H264_HARD: {
+        m_Settings.EncodeLevel = m_Settings.EncodeLevel ? LEVEL_H264_SOFT : LEVEL_H264_HARD;
+        SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);
+        BYTE bToken[] = {COMMAND_ENCODE_LEVEL, m_Settings.EncodeLevel };
+        m_ContextObject->Send2Client(bToken, sizeof(bToken));
+        break;
+    }
+    case IDM_ENABLE_AV1_HARD: {
+        m_Settings.EncodeLevel = m_Settings.EncodeLevel ? LEVEL_H264_SOFT : LEVEL_AV1_HARD;
+        SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
+        SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);
+        BYTE bToken[] = { COMMAND_ENCODE_LEVEL, m_Settings.EncodeLevel };
+        m_ContextObject->Send2Client(bToken, sizeof(bToken));
+        break;
+    }
    }

    __super::OnSysCommand(nID, lParam);
--- a/server/2015Remote/ScreenSpyDlg.h
+++ b/server/2015Remote/ScreenSpyDlg.h
@@ -135,6 +135,8 @@ enum {
    IDM_RESTORE_CONSOLE,        // RDP会话归位
    IDM_RESET_VIRTUAL_DESKTOP,  // 重置虚拟桌面
    IDM_AUDIO_TOGGLE,           // 音频开关
+    IDM_ENABLE_H264_HARD,
+    IDM_ENABLE_AV1_HARD,
 };

 // 状态信息窗口 - 全屏时显示帧率/速度/质量
--- a/server/2015Remote/lang/en_US.ini
+++ b/server/2015Remote/lang/en_US.ini
@@ -1892,3 +1892,5 @@ FRPC Զ
 不支持的位深度，需要24位或32位=Bitmap depth is unsupported
 未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=x264 Encoder is required \nDownload via：https://sourceforge.net/projects/x264vfw
 创建AVI文件失败=Create AVI file failed
+启用 H264 硬编码=Enable HW H264 Encoding
+启用 AV1 硬编码=Enable HW AV1 Encoding
--- a/server/2015Remote/lang/zh_TW.ini
+++ b/server/2015Remote/lang/zh_TW.ini
@@ -1883,3 +1883,5 @@ FRPC Զ
 不支持的位深度，需要24位或32位=不支持的位深度，需要24位或32位
 未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw
 创建AVI文件失败=创建AVI文件失败
+启用 H264 硬编码=启用 H264 硬编码
+启用 AV1 硬编码=启用 AV1 硬编码