Feature: Implement H.264 and AV1 hardware encoding for remote control
Remark: Need to update FFmpeg static libraries to take effort
This commit is contained in:
yuanyuanxiang
1
.gitignore
vendored
1
.gitignore
vendored
@@ -81,6 +81,7 @@ Releases/*
|
|||||||
linux/Makefile
|
linux/Makefile
|
||||||
linux/cmake_install.cmake
|
linux/cmake_install.cmake
|
||||||
.vs
|
.vs
|
||||||
|
client/ghost_vs2015.vcxproj.user
|
||||||
docs/macOS_Support_Design.md
|
docs/macOS_Support_Design.md
|
||||||
settings.local.json
|
settings.local.json
|
||||||
*.zip
|
*.zip
|
||||||
|
|||||||
243
client/CFFmpegAV1Encoder.cpp
Normal file
243
client/CFFmpegAV1Encoder.cpp
Normal file
@@ -0,0 +1,243 @@
|
|||||||
|
#include "CFFmpegAV1Encoder.h"
|
||||||
|
#include "common/config.h"
|
||||||
|
#include "common/logger.h"
|
||||||
|
|
||||||
|
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时整个实现移出编译单元（FFmpeg lib 已在
|
||||||
|
// CFFmpegH264Encoder.cpp 用同条件链接，此处不重复 #pragma comment）
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
|
||||||
|
extern "C" {
|
||||||
|
#include <libavcodec/avcodec.h>
|
||||||
|
#include <libavutil/opt.h>
|
||||||
|
#include <libavutil/imgutils.h>
|
||||||
|
#include <libyuv/libyuv.h>
|
||||||
|
}
|
||||||
|
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
// FFmpeg / 系统库已经由 CFFmpegH264Encoder.cpp 的 #pragma comment(lib) 引入。
|
||||||
|
// 这里不再重复声明（重复 #pragma comment 在同一 link 单元不冲突但冗余）。
|
||||||
|
|
||||||
|
// av_opt_set 包装：拼错的参数值会被 FFmpeg 静默忽略，包一层日志便于发现。
|
||||||
|
// 实现与 CFFmpegH264Encoder 内的 helper 相同；放成 static 文件内可见即可。
|
||||||
|
static void setOpt(void* obj, const char* name, const char* val, const char* backend) {
|
||||||
|
int rc = av_opt_set(obj, name, val, 0);
|
||||||
|
if (rc < 0) {
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("[WARN] av_opt_set('%s'='%s') on %s failed (%d): %s\n",
|
||||||
|
name, val, backend, rc, errbuf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
static void setOptInt(void* obj, const char* name, int64_t val, const char* backend) {
|
||||||
|
int rc = av_opt_set_int(obj, name, val, 0);
|
||||||
|
if (rc < 0) {
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("[WARN] av_opt_set_int('%s'=%lld) on %s failed (%d): %s\n",
|
||||||
|
name, (long long)val, backend, rc, errbuf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AV1 硬编后端探测顺序，没有 av1_mf 兜底（FFmpeg 7.1 不支持）。
|
||||||
|
// 全失败时 EncoderFactory 自动回退到 H.264 路径，行为对称。
|
||||||
|
static const char* kAV1Backends[] = {
|
||||||
|
"av1_nvenc", // NVIDIA RTX 40 / 50 系（Ada Lovelace+）
|
||||||
|
"av1_amf", // AMD RX 7000+（RDNA 3+）
|
||||||
|
"av1_qsv", // Intel Arc 独显 / 部分 11 代+ 核显
|
||||||
|
};
|
||||||
|
|
||||||
|
CFFmpegAV1Encoder::CFFmpegAV1Encoder() = default;
|
||||||
|
|
||||||
|
CFFmpegAV1Encoder::~CFFmpegAV1Encoder() {
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegAV1Encoder::cleanupCodec() {
|
||||||
|
if (m_packet) { av_packet_free(&m_packet); m_packet = nullptr; }
|
||||||
|
if (m_frame) { av_frame_free(&m_frame); m_frame = nullptr; }
|
||||||
|
if (m_ctx) { avcodec_free_context(&m_ctx); m_ctx = nullptr; }
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegAV1Encoder::close() {
|
||||||
|
cleanupCodec();
|
||||||
|
m_backend.clear();
|
||||||
|
m_pts = 0;
|
||||||
|
m_forceIDR = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool CFFmpegAV1Encoder::open(const EncoderParams& params) {
|
||||||
|
close();
|
||||||
|
for (const char* name : kAV1Backends) {
|
||||||
|
if (tryOpenBackend(name, params)) {
|
||||||
|
m_backend = name;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cleanupCodec();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool CFFmpegAV1Encoder::tryOpenBackend(const char* name, const EncoderParams& p) {
|
||||||
|
const AVCodec* codec = avcodec_find_encoder_by_name(name);
|
||||||
|
if (!codec) {
|
||||||
|
// AV1 硬编没注册 = 老 ffmpeg lib 不含 AV1 encoder（compress\ffmpeg 没启用 av1）
|
||||||
|
Mprintf("=> FFmpeg: AV1 encoder '%s' NOT in linked lib\n", name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_ctx = avcodec_alloc_context3(codec);
|
||||||
|
if (!m_ctx) {
|
||||||
|
Mprintf("=> FFmpeg: avcodec_alloc_context3('%s') failed\n", name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_ctx->width = p.width & ~1;
|
||||||
|
m_ctx->height = p.height & ~1;
|
||||||
|
m_ctx->time_base = AVRational{1, p.fps};
|
||||||
|
m_ctx->framerate = AVRational{p.fps, 1};
|
||||||
|
m_ctx->pix_fmt = AV_PIX_FMT_NV12;
|
||||||
|
m_ctx->gop_size = p.fps * (p.gop_seconds > 0 ? p.gop_seconds : 15);
|
||||||
|
m_ctx->max_b_frames = 0;
|
||||||
|
m_ctx->bit_rate = (int64_t)p.bitrate_kbps * 1000;
|
||||||
|
m_ctx->rc_max_rate = (int64_t)p.bitrate_kbps * 1500;
|
||||||
|
m_ctx->rc_buffer_size = (int)(p.bitrate_kbps * 1000);
|
||||||
|
|
||||||
|
// RC 策略与 H.264 路径对齐：peak-constrained VBR，远控静态画面省带宽。
|
||||||
|
if (strcmp(name, "av1_nvenc") == 0) {
|
||||||
|
// av1_nvenc preset p1~p7；远控 p5 兼顾质量与速度。
|
||||||
|
// tile-columns=1 把帧切两列，解码端并行更友好（浏览器 AV1 解码常用 SIMD/多线程）
|
||||||
|
setOpt(m_ctx->priv_data, "preset", "p5", name);
|
||||||
|
setOpt(m_ctx->priv_data, "tune", "ll", name);
|
||||||
|
setOpt(m_ctx->priv_data, "rc", "vbr", name);
|
||||||
|
setOpt(m_ctx->priv_data, "zerolatency", "1", name);
|
||||||
|
setOptInt(m_ctx->priv_data, "tile-columns", 1, name);
|
||||||
|
} else if (strcmp(name, "av1_amf") == 0) {
|
||||||
|
// av1_amf 选项命名与 h264_amf 大体一致，rc 同样支持 vbr_peak
|
||||||
|
// (见 ffmpeg -h encoder=av1_amf)。静态画面省码率四件套同 H.264 路径。
|
||||||
|
setOpt(m_ctx->priv_data, "usage", "lowlatency", name);
|
||||||
|
setOpt(m_ctx->priv_data, "quality", "quality", name);
|
||||||
|
setOpt(m_ctx->priv_data, "rc", "vbr_peak", name);
|
||||||
|
setOptInt(m_ctx->priv_data, "vbaq", 1, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "preanalysis", 1, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "filler_data", 0, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "enforce_hrd", 0, name);
|
||||||
|
} else if (strcmp(name, "av1_qsv") == 0) {
|
||||||
|
// av1_qsv：bit_rate < max_rate 时自动 VBR
|
||||||
|
setOpt(m_ctx->priv_data, "preset", "slow", name);
|
||||||
|
setOptInt(m_ctx->priv_data, "async_depth", 1, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "low_power", 0, name);
|
||||||
|
}
|
||||||
|
|
||||||
|
int ret = avcodec_open2(m_ctx, codec, nullptr);
|
||||||
|
if (ret < 0) {
|
||||||
|
// 找到了但开不起来：无对应 GPU / 驱动太旧 / 跨适配器
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(ret, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("=> FFmpeg: avcodec_open2('%s') failed (%d): %s\n", name, ret, errbuf);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_frame = av_frame_alloc();
|
||||||
|
if (!m_frame) return false;
|
||||||
|
m_frame->format = AV_PIX_FMT_NV12;
|
||||||
|
m_frame->width = m_ctx->width;
|
||||||
|
m_frame->height = m_ctx->height;
|
||||||
|
if (av_frame_get_buffer(m_frame, 32) < 0) {
|
||||||
|
Mprintf("=> FFmpeg: av_frame_get_buffer failed\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_packet = av_packet_alloc();
|
||||||
|
return m_packet != nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegAV1Encoder::setBitrate(int kbps) {
|
||||||
|
if (!m_ctx) return;
|
||||||
|
m_ctx->bit_rate = (int64_t)kbps * 1000;
|
||||||
|
m_ctx->rc_max_rate = (int64_t)kbps * 1500;
|
||||||
|
m_ctx->rc_buffer_size = (int)(kbps * 1000);
|
||||||
|
// 同 H.264 路径：多数硬编不支持运行时改 bit_rate 让 ctx 立刻生效；
|
||||||
|
// 这里仅更新数值，下次 open 时生效。
|
||||||
|
}
|
||||||
|
|
||||||
|
int CFFmpegAV1Encoder::convertRGB24ToNV12(uint8_t* rgb, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
int direction)
|
||||||
|
{
|
||||||
|
int signed_height = direction * (int)height;
|
||||||
|
int w = (int)width;
|
||||||
|
int h = (int)height;
|
||||||
|
int y_size = w * h;
|
||||||
|
int uv_size = (w / 2) * (h / 2);
|
||||||
|
m_i420Scratch.resize(y_size + 2 * uv_size);
|
||||||
|
|
||||||
|
uint8_t* y = m_i420Scratch.data();
|
||||||
|
uint8_t* u = y + y_size;
|
||||||
|
uint8_t* v = u + uv_size;
|
||||||
|
|
||||||
|
if (libyuv::RGB24ToI420(rgb, stride, y, w, u, w / 2, v, w / 2, w, signed_height) != 0)
|
||||||
|
return -1;
|
||||||
|
if (libyuv::I420ToNV12(y, w, u, w / 2, v, w / 2,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
w, h) != 0)
|
||||||
|
return -1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int CFFmpegAV1Encoder::encode(
|
||||||
|
uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
uint8_t** lppData, uint32_t* lpSize, int direction)
|
||||||
|
{
|
||||||
|
if (!m_ctx || !m_frame || !m_packet) return -1;
|
||||||
|
if (av_frame_make_writable(m_frame) < 0) return -1;
|
||||||
|
|
||||||
|
int w = (int)width;
|
||||||
|
int h = (int)height;
|
||||||
|
int signed_height = direction * h;
|
||||||
|
|
||||||
|
if (bpp == 32) {
|
||||||
|
if (libyuv::ARGBToNV12(
|
||||||
|
rgb, stride,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
w, signed_height) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else if (bpp == 24) {
|
||||||
|
if (convertRGB24ToNV12(rgb, stride, width, height, direction) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return -2;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_frame->pts = m_pts++;
|
||||||
|
if (m_forceIDR) {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_I;
|
||||||
|
m_forceIDR = false;
|
||||||
|
} else {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
int ret = avcodec_send_frame(m_ctx, m_frame);
|
||||||
|
if (ret < 0) return -3;
|
||||||
|
|
||||||
|
ret = avcodec_receive_packet(m_ctx, m_packet);
|
||||||
|
if (ret == AVERROR(EAGAIN)) {
|
||||||
|
*lppData = nullptr;
|
||||||
|
*lpSize = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (ret < 0) return -4;
|
||||||
|
|
||||||
|
m_outputBuffer.assign(m_packet->data, m_packet->data + m_packet->size);
|
||||||
|
*lppData = m_outputBuffer.data();
|
||||||
|
*lpSize = (uint32_t)m_outputBuffer.size();
|
||||||
|
av_packet_unref(m_packet);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif // _WIN64 && !DISABLE_FFMPEG_FOR_TEST
|
||||||
62
client/CFFmpegAV1Encoder.h
Normal file
62
client/CFFmpegAV1Encoder.h
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include "common/config.h"
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时整类移出编译单元，避免 GPL 传染（与 c0a632a 对齐）
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
|
||||||
|
struct AVCodecContext;
|
||||||
|
struct AVFrame;
|
||||||
|
struct AVPacket;
|
||||||
|
|
||||||
|
// FFmpeg 硬编 AV1 实现。
|
||||||
|
// 后端探测顺序：av1_nvenc (NVIDIA RTX 40+) → av1_amf (AMD RX 7000+) → av1_qsv
|
||||||
|
// (Intel Arc / 11 代+ 部分核显)。AV1 硬编硬件门槛比 H.264 高得多 —— 没合适
|
||||||
|
// 硬件时 open 全部失败，由 EncoderFactory 自动回退到 H.264 路径。
|
||||||
|
//
|
||||||
|
// 注意：FFmpeg 7.1 没有 av1_mf 兜底，因此本类的探测列表比 H.264 短一项。
|
||||||
|
class CFFmpegAV1Encoder : public VideoEncoderBase
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
CFFmpegAV1Encoder();
|
||||||
|
~CFFmpegAV1Encoder() override;
|
||||||
|
|
||||||
|
bool open(const EncoderParams& params) override;
|
||||||
|
void close() override;
|
||||||
|
|
||||||
|
int encode(
|
||||||
|
uint8_t* rgb,
|
||||||
|
uint8_t bpp,
|
||||||
|
uint32_t stride,
|
||||||
|
uint32_t width,
|
||||||
|
uint32_t height,
|
||||||
|
uint8_t** lppData,
|
||||||
|
uint32_t* lpSize,
|
||||||
|
int direction = 1
|
||||||
|
) override;
|
||||||
|
|
||||||
|
void forceIDR() override { m_forceIDR = true; }
|
||||||
|
void setBitrate(int kbps) override;
|
||||||
|
VideoCodec codec() const override { return VideoCodec::AV1; }
|
||||||
|
const char* backendName() const override { return m_backend.c_str(); }
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool tryOpenBackend(const char* name, const EncoderParams& p);
|
||||||
|
void cleanupCodec();
|
||||||
|
int convertRGB24ToNV12(uint8_t* rgb, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height, int direction);
|
||||||
|
|
||||||
|
AVCodecContext* m_ctx = nullptr;
|
||||||
|
AVFrame* m_frame = nullptr;
|
||||||
|
AVPacket* m_packet = nullptr;
|
||||||
|
std::vector<uint8_t> m_outputBuffer;
|
||||||
|
std::vector<uint8_t> m_i420Scratch;
|
||||||
|
int64_t m_pts = 0;
|
||||||
|
bool m_forceIDR = false;
|
||||||
|
std::string m_backend;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // _WIN64 && !DISABLE_FFMPEG_FOR_TEST
|
||||||
299
client/CFFmpegH264Encoder.cpp
Normal file
299
client/CFFmpegH264Encoder.cpp
Normal file
@@ -0,0 +1,299 @@
|
|||||||
|
#include "CFFmpegH264Encoder.h"
|
||||||
|
#include "common/config.h"
|
||||||
|
#include "common/logger.h"
|
||||||
|
|
||||||
|
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时整个实现 + 所有 #pragma comment(lib,"ffmpeg/...")
|
||||||
|
// 都不进编译单元，FFmpeg 静态库不会被链接进二进制
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
|
||||||
|
extern "C" {
|
||||||
|
#include <libavcodec/avcodec.h>
|
||||||
|
#include <libavutil/opt.h>
|
||||||
|
#include <libavutil/imgutils.h>
|
||||||
|
#include <libyuv/libyuv.h>
|
||||||
|
}
|
||||||
|
|
||||||
|
#include <string.h>
|
||||||
|
#include <cstdlib>
|
||||||
|
|
||||||
|
// FFmpeg 静态库 + 必要的 Windows 系统库。x86 build 不引入，由 _WIN64 守护。
|
||||||
|
// FFmpeg 三个核心库是纯 C，CRT 中性，Debug/Release 共用一份。
|
||||||
|
#pragma comment(lib,"ffmpeg/libavcodec_x64.lib")
|
||||||
|
#pragma comment(lib,"ffmpeg/libavutil_x64.lib")
|
||||||
|
#pragma comment(lib,"ffmpeg/libswresample_x64.lib")
|
||||||
|
// dav1d (AV1 软解，C 项目) —— 不分 Debug/Release。
|
||||||
|
// build 时启用了 --enable-libdav1d，libavcodec 内部 av1 decoder 引用了 dav1d 符号。
|
||||||
|
#pragma comment(lib,"ffmpeg/dav1d_x64.lib")
|
||||||
|
// libvpl (Intel QSV, C++ 项目) —— 强制 CRT 一致，必须按 _DEBUG 切。
|
||||||
|
// build 时启用了 --enable-libvpl，libavcodec 内部 h264_qsv / av1_qsv encoder 引用 MFX 符号。
|
||||||
|
#ifdef _DEBUG
|
||||||
|
#pragma comment(lib,"ffmpeg/vpl_x64d.lib")
|
||||||
|
#else
|
||||||
|
#pragma comment(lib,"ffmpeg/vpl_x64.lib")
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#pragma comment(lib, "mfplat.lib")
|
||||||
|
#pragma comment(lib, "mfuuid.lib")
|
||||||
|
#pragma comment(lib, "strmiids.lib")
|
||||||
|
#pragma comment(lib, "secur32.lib")
|
||||||
|
#pragma comment(lib, "bcrypt.lib")
|
||||||
|
#pragma comment(lib, "advapi32.lib")
|
||||||
|
#pragma comment(lib, "ole32.lib")
|
||||||
|
// ws2_32 在 IOCPClient.h 已 link，重复不冲突
|
||||||
|
#pragma comment(lib, "ws2_32.lib")
|
||||||
|
|
||||||
|
// av_opt_set wrappers：FFmpeg 在选项名/值拼错时 silently 返回 AVERROR_OPTION_NOT_FOUND
|
||||||
|
// 不报错，导致 encoder 退回默认行为且没人察觉（实际踩过：AMF rc=vbr_peak_constrained
|
||||||
|
// 拼成全名，FFmpeg 实际只接受 vbr_peak，没设上去就退回 CBR）。
|
||||||
|
// 包一层 helper，任何设置失败 Mprintf 警告。
|
||||||
|
static void setOpt(void* obj, const char* name, const char* val, const char* backend) {
|
||||||
|
int rc = av_opt_set(obj, name, val, 0);
|
||||||
|
if (rc < 0) {
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("[WARN] av_opt_set('%s'='%s') on %s failed (%d): %s\n",
|
||||||
|
name, val, backend, rc, errbuf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
static void setOptInt(void* obj, const char* name, int64_t val, const char* backend) {
|
||||||
|
int rc = av_opt_set_int(obj, name, val, 0);
|
||||||
|
if (rc < 0) {
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("[WARN] av_opt_set_int('%s'=%lld) on %s failed (%d): %s\n",
|
||||||
|
name, (long long)val, backend, rc, errbuf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 后端探测顺序：NVIDIA > Intel > AMD > Windows MF 兜底。
|
||||||
|
// open() 主循环按顺序试，第一个 avcodec_open2 成功的就用。
|
||||||
|
// h264_mf 质量/稳定性一般，但是 Windows 系统级 hwaccel，任何 GPU 都能尝试，作最后兜底。
|
||||||
|
static const char* kH264Backends[] = {
|
||||||
|
"h264_nvenc", // NVIDIA NVENC
|
||||||
|
"h264_qsv", // Intel Quick Sync Video
|
||||||
|
"h264_amf", // AMD AMF
|
||||||
|
"h264_mf", // Windows Media Foundation
|
||||||
|
};
|
||||||
|
|
||||||
|
CFFmpegH264Encoder::CFFmpegH264Encoder() = default;
|
||||||
|
|
||||||
|
CFFmpegH264Encoder::~CFFmpegH264Encoder() {
|
||||||
|
close();
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegH264Encoder::cleanupCodec() {
|
||||||
|
if (m_packet) { av_packet_free(&m_packet); m_packet = nullptr; }
|
||||||
|
if (m_frame) { av_frame_free(&m_frame); m_frame = nullptr; }
|
||||||
|
if (m_ctx) { avcodec_free_context(&m_ctx); m_ctx = nullptr; }
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegH264Encoder::close() {
|
||||||
|
cleanupCodec();
|
||||||
|
m_backend.clear();
|
||||||
|
m_pts = 0;
|
||||||
|
m_forceIDR = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool CFFmpegH264Encoder::open(const EncoderParams& params) {
|
||||||
|
close();
|
||||||
|
for (const char* name : kH264Backends) {
|
||||||
|
if (tryOpenBackend(name, params)) {
|
||||||
|
m_backend = name;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cleanupCodec(); // 释放本次失败的 ctx，准备下一次尝试
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool CFFmpegH264Encoder::tryOpenBackend(const char* name, const EncoderParams& p) {
|
||||||
|
const AVCodec* codec = avcodec_find_encoder_by_name(name);
|
||||||
|
if (!codec) {
|
||||||
|
// 失败 = lib 里没注册这个 encoder。几乎肯定是链到了老 ffmpeg lib。
|
||||||
|
Mprintf("=> FFmpeg: encoder '%s' NOT in linked lib (old ffmpeg?)\n", name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_ctx = avcodec_alloc_context3(codec);
|
||||||
|
if (!m_ctx) {
|
||||||
|
Mprintf("=> FFmpeg: avcodec_alloc_context3('%s') failed\n", name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 偶数对齐（与 x264 路径 i_width/i_height & 0xfffffffe 一致）
|
||||||
|
m_ctx->width = p.width & ~1;
|
||||||
|
m_ctx->height = p.height & ~1;
|
||||||
|
m_ctx->time_base = AVRational{1, p.fps};
|
||||||
|
m_ctx->framerate = AVRational{p.fps, 1};
|
||||||
|
m_ctx->pix_fmt = AV_PIX_FMT_NV12;
|
||||||
|
m_ctx->gop_size = p.fps * (p.gop_seconds > 0 ? p.gop_seconds : 4);
|
||||||
|
m_ctx->max_b_frames = 0;
|
||||||
|
m_ctx->bit_rate = (int64_t)p.bitrate_kbps * 1000;
|
||||||
|
m_ctx->rc_max_rate = (int64_t)p.bitrate_kbps * 1500;
|
||||||
|
m_ctx->rc_buffer_size = (int)(p.bitrate_kbps * 1000);
|
||||||
|
|
||||||
|
// RC 策略选择：远程办公 90% 时间是静态画面（文档/IDE/邮件），CBR 会强行
|
||||||
|
// 把目标码率填满（静态用不上的部分浪费带宽）。所有硬编后端统一改用 VBR，
|
||||||
|
// bit_rate 是平均目标、rc_max_rate (1.5x) 是峰值上限：静态时 encoder 自动
|
||||||
|
// 降码率省带宽，动态时回到目标 + 短暂上探到 1.5x 保证画质。
|
||||||
|
// 接近 x264 软编 CRF + VBV 的行为，但严格守住峰值不爆。
|
||||||
|
if (strcmp(name, "h264_nvenc") == 0) {
|
||||||
|
// NVENC preset: p1(最快/低质) ~ p7(最慢/高质)，远控低延迟 p5 兼顾。
|
||||||
|
// tune=ll low-latency；rc=vbr 配 max_rate 实现峰值受限的 VBR。
|
||||||
|
setOpt(m_ctx->priv_data, "preset", "p5", name);
|
||||||
|
setOpt(m_ctx->priv_data, "tune", "ll", name);
|
||||||
|
setOpt(m_ctx->priv_data, "rc", "vbr", name);
|
||||||
|
setOpt(m_ctx->priv_data, "zerolatency", "1", name);
|
||||||
|
} else if (strcmp(name, "h264_qsv") == 0) {
|
||||||
|
// Intel Quick Sync Video。preset: veryfast/faster/fast/medium/slow/slower/veryslow
|
||||||
|
// QSV 当 bit_rate != rc_max_rate 时自动走 VBR，所以这里只需调 preset。
|
||||||
|
// preset=slow 比 medium 慢但画质好，async_depth=1 单帧立即出包。
|
||||||
|
// low_power=0 走 PAK 路径，部分集显不支持 low_power 模式。
|
||||||
|
setOpt(m_ctx->priv_data, "preset", "slow", name);
|
||||||
|
setOptInt(m_ctx->priv_data, "async_depth", 1, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "low_power", 0, name);
|
||||||
|
} else if (strcmp(name, "h264_amf") == 0) {
|
||||||
|
// AMD AMF 远控低延迟配置：
|
||||||
|
// usage=ultralowlatency 比 lowlatency 更激进，关闭一切 lookahead；
|
||||||
|
// quality=speed 选最快编码路径（vs balanced/quality）；
|
||||||
|
// rc=cbr 提供最可预测的输出节拍，避免 RC 切换抖动。
|
||||||
|
// 静态画面省码率交给应用层 skip 检测（ScreenCapture::GetNextScreenData
|
||||||
|
// 已经过 memcmp 把无变化帧直接拦在编码器之前），不再依赖 vbaq/preanalysis
|
||||||
|
// 这些会引入 30-100ms lookahead 的"省码率三件套"。
|
||||||
|
setOpt(m_ctx->priv_data, "usage", "ultralowlatency", name);
|
||||||
|
setOpt(m_ctx->priv_data, "quality", "speed", name);
|
||||||
|
setOpt(m_ctx->priv_data, "rc", "cbr", name);
|
||||||
|
setOptInt(m_ctx->priv_data, "filler_data", 0, name);
|
||||||
|
setOptInt(m_ctx->priv_data, "enforce_hrd", 0, name);
|
||||||
|
} else if (strcmp(name, "h264_mf") == 0) {
|
||||||
|
// Windows Media Foundation 兜底。rate_control 实际值（ffmpeg -h encoder=h264_mf）：
|
||||||
|
// default / cbr / pc_vbr / u_vbr / quality / ld_vbr / g_vbr / gld_vbr
|
||||||
|
// 远控用 pc_vbr (peak-constrained VBR) 与其他后端语义对齐。
|
||||||
|
setOptInt(m_ctx->priv_data, "hw_encoding", 1, name);
|
||||||
|
setOpt(m_ctx->priv_data, "rate_control", "pc_vbr", name);
|
||||||
|
}
|
||||||
|
|
||||||
|
int ret = avcodec_open2(m_ctx, codec, nullptr);
|
||||||
|
if (ret < 0) {
|
||||||
|
// 失败 = encoder 找到了但开不起来。常见：无 NVIDIA GPU / 驱动太旧 /
|
||||||
|
// NVENC session 占满 / 笔记本独显未唤醒 / 参数组合驱动不接受
|
||||||
|
char errbuf[128] = {0};
|
||||||
|
av_strerror(ret, errbuf, sizeof(errbuf));
|
||||||
|
Mprintf("=> FFmpeg: avcodec_open2('%s') failed (%d): %s\n", name, ret, errbuf);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_frame = av_frame_alloc();
|
||||||
|
if (!m_frame) return false;
|
||||||
|
m_frame->format = AV_PIX_FMT_NV12;
|
||||||
|
m_frame->width = m_ctx->width;
|
||||||
|
m_frame->height = m_ctx->height;
|
||||||
|
if (av_frame_get_buffer(m_frame, 32) < 0) {
|
||||||
|
Mprintf("=> FFmpeg: av_frame_get_buffer failed\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_packet = av_packet_alloc();
|
||||||
|
return m_packet != nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void CFFmpegH264Encoder::setBitrate(int kbps) {
|
||||||
|
if (!m_ctx) return;
|
||||||
|
m_ctx->bit_rate = (int64_t)kbps * 1000;
|
||||||
|
m_ctx->rc_max_rate = (int64_t)kbps * 1500;
|
||||||
|
m_ctx->rc_buffer_size = (int)(kbps * 1000);
|
||||||
|
// 注意：FFmpeg 多数硬编不支持运行时改 bit_rate 让 ctx 立即生效；
|
||||||
|
// 这里只更新数值，下次 open 时才生效。Step 1 不依赖动态调码率。
|
||||||
|
}
|
||||||
|
|
||||||
|
int CFFmpegH264Encoder::convertRGB24ToNV12(uint8_t* rgb, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
int direction)
|
||||||
|
{
|
||||||
|
int signed_height = direction * (int)height;
|
||||||
|
int w = (int)width;
|
||||||
|
int h = (int)height;
|
||||||
|
int y_size = w * h;
|
||||||
|
int uv_size = (w / 2) * (h / 2);
|
||||||
|
m_i420Scratch.resize(y_size + 2 * uv_size);
|
||||||
|
|
||||||
|
uint8_t* y = m_i420Scratch.data();
|
||||||
|
uint8_t* u = y + y_size;
|
||||||
|
uint8_t* v = u + uv_size;
|
||||||
|
|
||||||
|
if (libyuv::RGB24ToI420(
|
||||||
|
rgb, stride,
|
||||||
|
y, w,
|
||||||
|
u, w / 2,
|
||||||
|
v, w / 2,
|
||||||
|
w, signed_height) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (libyuv::I420ToNV12(
|
||||||
|
y, w,
|
||||||
|
u, w / 2,
|
||||||
|
v, w / 2,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
w, h) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int CFFmpegH264Encoder::encode(
|
||||||
|
uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
uint8_t** lppData, uint32_t* lpSize, int direction)
|
||||||
|
{
|
||||||
|
if (!m_ctx || !m_frame || !m_packet) return -1;
|
||||||
|
if (av_frame_make_writable(m_frame) < 0) return -1;
|
||||||
|
|
||||||
|
int w = (int)width;
|
||||||
|
int h = (int)height;
|
||||||
|
int signed_height = direction * h;
|
||||||
|
|
||||||
|
if (bpp == 32) {
|
||||||
|
if (libyuv::ARGBToNV12(
|
||||||
|
rgb, stride,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
w, signed_height) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else if (bpp == 24) {
|
||||||
|
if (convertRGB24ToNV12(rgb, stride, width, height, direction) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return -2;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_frame->pts = m_pts++;
|
||||||
|
if (m_forceIDR) {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_I;
|
||||||
|
m_forceIDR = false;
|
||||||
|
} else {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
int ret = avcodec_send_frame(m_ctx, m_frame);
|
||||||
|
if (ret < 0) return -3;
|
||||||
|
|
||||||
|
ret = avcodec_receive_packet(m_ctx, m_packet);
|
||||||
|
if (ret == AVERROR(EAGAIN)) {
|
||||||
|
// 首帧延迟：本次没出包，调用方按 lpSize==0 跳过本帧
|
||||||
|
*lppData = nullptr;
|
||||||
|
*lpSize = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (ret < 0) return -4;
|
||||||
|
|
||||||
|
m_outputBuffer.assign(m_packet->data, m_packet->data + m_packet->size);
|
||||||
|
*lppData = m_outputBuffer.data();
|
||||||
|
*lpSize = (uint32_t)m_outputBuffer.size();
|
||||||
|
av_packet_unref(m_packet);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif // _WIN64 && !DISABLE_FFMPEG_FOR_TEST
|
||||||
62
client/CFFmpegH264Encoder.h
Normal file
62
client/CFFmpegH264Encoder.h
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include "common/config.h"
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时整类移出编译单元，避免 GPL 传染（与 c0a632a 对齐）
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
|
||||||
|
struct AVCodecContext;
|
||||||
|
struct AVFrame;
|
||||||
|
struct AVPacket;
|
||||||
|
|
||||||
|
// FFmpeg 硬编 H.264 实现。
|
||||||
|
// Step 1: 仅探测 h264_nvenc 单后端，足以验证 FFmpeg 静态库集成链路。
|
||||||
|
// Step 2: 扩展 h264_qsv / h264_amf / h264_mf。
|
||||||
|
//
|
||||||
|
// 输入像素：BGRA (bpp=32) / RGB24 (bpp=24)，与 CX264Encoder 完全一致；
|
||||||
|
// 内部转 NV12 喂给 FFmpeg encoder。
|
||||||
|
class CFFmpegH264Encoder : public VideoEncoderBase
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
CFFmpegH264Encoder();
|
||||||
|
~CFFmpegH264Encoder() override;
|
||||||
|
|
||||||
|
bool open(const EncoderParams& params) override;
|
||||||
|
void close() override;
|
||||||
|
|
||||||
|
int encode(
|
||||||
|
uint8_t* rgb,
|
||||||
|
uint8_t bpp,
|
||||||
|
uint32_t stride,
|
||||||
|
uint32_t width,
|
||||||
|
uint32_t height,
|
||||||
|
uint8_t** lppData,
|
||||||
|
uint32_t* lpSize,
|
||||||
|
int direction = 1
|
||||||
|
) override;
|
||||||
|
|
||||||
|
void forceIDR() override { m_forceIDR = true; }
|
||||||
|
void setBitrate(int kbps) override;
|
||||||
|
VideoCodec codec() const override { return VideoCodec::H264; }
|
||||||
|
const char* backendName() const override { return m_backend.c_str(); }
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool tryOpenBackend(const char* name, const EncoderParams& p);
|
||||||
|
void cleanupCodec();
|
||||||
|
int convertRGB24ToNV12(uint8_t* rgb, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height, int direction);
|
||||||
|
|
||||||
|
AVCodecContext* m_ctx = nullptr;
|
||||||
|
AVFrame* m_frame = nullptr;
|
||||||
|
AVPacket* m_packet = nullptr;
|
||||||
|
std::vector<uint8_t> m_outputBuffer; // encode 返回给调用方的缓冲（持有到下一次 encode）
|
||||||
|
std::vector<uint8_t> m_i420Scratch; // RGB24 路径的中间缓冲
|
||||||
|
int64_t m_pts = 0;
|
||||||
|
bool m_forceIDR = false;
|
||||||
|
std::string m_backend; // 实际选中的后端名（"h264_nvenc" / ...）
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // _WIN64 && !DISABLE_FFMPEG_FOR_TEST
|
||||||
@@ -124,7 +124,7 @@
|
|||||||
<AdditionalDependencies>zlib\zlib_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
<AdditionalDependencies>zlib\zlib_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||||
<IgnoreSpecificDefaultLibraries>libcmt.lib</IgnoreSpecificDefaultLibraries>
|
<IgnoreSpecificDefaultLibraries>libcmt.lib</IgnoreSpecificDefaultLibraries>
|
||||||
<AdditionalOptions>/ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
<AdditionalOptions>/ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
||||||
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin</AdditionalLibraryDirectories>
|
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin;$(SolutionDir)..\ffmpeg-7.1\install-win64\lib</AdditionalLibraryDirectories>
|
||||||
</Link>
|
</Link>
|
||||||
</ItemDefinitionGroup>
|
</ItemDefinitionGroup>
|
||||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||||
@@ -167,7 +167,7 @@
|
|||||||
<OptimizeReferences>true</OptimizeReferences>
|
<OptimizeReferences>true</OptimizeReferences>
|
||||||
<AdditionalDependencies>zlib\zlib_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
<AdditionalDependencies>zlib\zlib_x64.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||||
<AdditionalOptions> /SAFESEH:NO /ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
<AdditionalOptions> /SAFESEH:NO /ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
||||||
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin</AdditionalLibraryDirectories>
|
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin;$(SolutionDir)..\ffmpeg-7.1\install-win64\lib</AdditionalLibraryDirectories>
|
||||||
</Link>
|
</Link>
|
||||||
</ItemDefinitionGroup>
|
</ItemDefinitionGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@@ -205,6 +205,9 @@
|
|||||||
<ClCompile Include="ShellManager.cpp" />
|
<ClCompile Include="ShellManager.cpp" />
|
||||||
<ClCompile Include="StdAfx.cpp" />
|
<ClCompile Include="StdAfx.cpp" />
|
||||||
<ClCompile Include="SystemManager.cpp" />
|
<ClCompile Include="SystemManager.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegAV1Encoder.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegH264Encoder.cpp" />
|
||||||
|
<ClCompile Include="EncoderFactory.cpp" />
|
||||||
<ClCompile Include="TalkManager.cpp" />
|
<ClCompile Include="TalkManager.cpp" />
|
||||||
<ClCompile Include="VideoManager.cpp" />
|
<ClCompile Include="VideoManager.cpp" />
|
||||||
<ClCompile Include="X264Encoder.cpp" />
|
<ClCompile Include="X264Encoder.cpp" />
|
||||||
@@ -228,6 +231,10 @@
|
|||||||
<ClInclude Include="IOCPClient.h" />
|
<ClInclude Include="IOCPClient.h" />
|
||||||
<ClInclude Include="IOCPKCPClient.h" />
|
<ClInclude Include="IOCPKCPClient.h" />
|
||||||
<ClInclude Include="IOCPUDPClient.h" />
|
<ClInclude Include="IOCPUDPClient.h" />
|
||||||
|
<ClInclude Include="CFFmpegAV1Encoder.h" />
|
||||||
|
<ClInclude Include="CFFmpegH264Encoder.h" />
|
||||||
|
<ClInclude Include="EncoderFactory.h" />
|
||||||
|
<ClInclude Include="VideoEncoderBase.h" />
|
||||||
<ClInclude Include="KernelManager.h" />
|
<ClInclude Include="KernelManager.h" />
|
||||||
<ClInclude Include="KeyboardManager.h" />
|
<ClInclude Include="KeyboardManager.h" />
|
||||||
<ClInclude Include="keylogger.h" />
|
<ClInclude Include="keylogger.h" />
|
||||||
|
|||||||
@@ -36,6 +36,9 @@
|
|||||||
<ClCompile Include="TalkManager.cpp" />
|
<ClCompile Include="TalkManager.cpp" />
|
||||||
<ClCompile Include="VideoManager.cpp" />
|
<ClCompile Include="VideoManager.cpp" />
|
||||||
<ClCompile Include="X264Encoder.cpp" />
|
<ClCompile Include="X264Encoder.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegH264Encoder.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegAV1Encoder.cpp" />
|
||||||
|
<ClCompile Include="EncoderFactory.cpp" />
|
||||||
<ClCompile Include="..\common\file_upload.cpp" />
|
<ClCompile Include="..\common\file_upload.cpp" />
|
||||||
<ClCompile Include="ConPTYManager.cpp" />
|
<ClCompile Include="ConPTYManager.cpp" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
@@ -81,6 +84,10 @@
|
|||||||
<ClInclude Include="VideoCodec.h" />
|
<ClInclude Include="VideoCodec.h" />
|
||||||
<ClInclude Include="VideoManager.h" />
|
<ClInclude Include="VideoManager.h" />
|
||||||
<ClInclude Include="X264Encoder.h" />
|
<ClInclude Include="X264Encoder.h" />
|
||||||
|
<ClInclude Include="VideoEncoderBase.h" />
|
||||||
|
<ClInclude Include="CFFmpegH264Encoder.h" />
|
||||||
|
<ClInclude Include="CFFmpegAV1Encoder.h" />
|
||||||
|
<ClInclude Include="EncoderFactory.h" />
|
||||||
<ClInclude Include="ConPTYManager.h" />
|
<ClInclude Include="ConPTYManager.h" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
71
client/EncoderFactory.cpp
Normal file
71
client/EncoderFactory.cpp
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
#include "EncoderFactory.h"
|
||||||
|
#include "common/config.h"
|
||||||
|
#include "common/logger.h"
|
||||||
|
#include "X264Encoder.h"
|
||||||
|
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时硬编实现整体移出工程，仅保留 x264 软编路径
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
#include "CFFmpegH264Encoder.h"
|
||||||
|
#include "CFFmpegAV1Encoder.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
// 与 ScreenCapture::BitRateToCRF 同步：码率越高 CRF 越低（质量更好）。
|
||||||
|
// 仅 x264 软编路径用，硬编路径直接用 bitrate_kbps 走 CBR。
|
||||||
|
int BitRateToCRF(int bitRate) {
|
||||||
|
if (bitRate <= 0) return 23;
|
||||||
|
if (bitRate >= 3000) return 20;
|
||||||
|
if (bitRate >= 2000) return 20 + (3000 - bitRate) * 3 / 1000;
|
||||||
|
if (bitRate >= 800) return 23 + (2000 - bitRate) * 7 / 1200;
|
||||||
|
return 32;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<VideoEncoderBase> CreateEncoder(const EncoderRequest& req) {
|
||||||
|
EncoderParams p;
|
||||||
|
p.width = req.width;
|
||||||
|
p.height = req.height;
|
||||||
|
p.fps = req.fps;
|
||||||
|
|
||||||
|
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||||
|
// AV1 硬编路径（仅当客户端声明支持 AV1 解码）
|
||||||
|
// 硬件门槛高：仅 RTX 40+ / RX 7000+ / Intel Arc 才有 av1 encoder ASIC；
|
||||||
|
// 没合适硬件时 open() 全部失败，自然 fall through 到下面 H.264 路径。
|
||||||
|
if (req.encodeLevel >= LEVEL_AV1_HARD) {
|
||||||
|
auto enc = std::make_unique<CFFmpegAV1Encoder>();
|
||||||
|
p.rc = RateControl::BITRATE;
|
||||||
|
p.bitrate_kbps = req.bitrate_kbps;
|
||||||
|
if (enc->open(p)) {
|
||||||
|
Mprintf("=> encoder: %s (HW AV1, bitrate=%dk)\n", enc->backendName(), req.bitrate_kbps);
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
Mprintf("=> all AV1 HW backends failed, falling back to H.264\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// H.264 硬编：CFFmpegH264Encoder 内部按 nvenc/qsv/amf/mf 顺序探
|
||||||
|
if (req.encodeLevel >= LEVEL_H264_HARD) {
|
||||||
|
auto enc = std::make_unique<CFFmpegH264Encoder>();
|
||||||
|
p.rc = RateControl::BITRATE;
|
||||||
|
p.bitrate_kbps = req.bitrate_kbps;
|
||||||
|
if (enc->open(p)) {
|
||||||
|
Mprintf("=> encoder: %s (HW, bitrate=%dk)\n", enc->backendName(), req.bitrate_kbps);
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
Mprintf("=> all H.264 HW backends failed, falling back to x264\n");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// x264 软编兜底（无硬件 / 全失败 / 虚拟机 / 远程桌面会话场景）
|
||||||
|
if (req.encodeLevel >= LEVEL_H264_SOFT) {
|
||||||
|
auto enc = std::make_unique<CX264Encoder>();
|
||||||
|
p.rc = RateControl::CRF;
|
||||||
|
p.crf = BitRateToCRF(req.bitrate_kbps);
|
||||||
|
if (enc->open(p)) {
|
||||||
|
Mprintf("=> encoder: %s (SW, crf=%d)\n", enc->backendName(), p.crf);
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Mprintf("=> ERROR: no encoder could be opened\n");
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
25
client/EncoderFactory.h
Normal file
25
client/EncoderFactory.h
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include "common/commands.h"
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
|
||||||
|
// 创建编码器的请求参数。
|
||||||
|
struct EncoderRequest {
|
||||||
|
int width = 0;
|
||||||
|
int height = 0;
|
||||||
|
int fps = 30;
|
||||||
|
int bitrate_kbps = 4000;
|
||||||
|
int encodeLevel = LEVEL_H264_SOFT;
|
||||||
|
};
|
||||||
|
|
||||||
|
// 按客户端能力 + 本机硬件能力创建一个 VideoEncoderBase。
|
||||||
|
//
|
||||||
|
// 探测顺序（第一个 open 成功的就用）：
|
||||||
|
// AV1 硬编路径
|
||||||
|
// H.264 硬编（CFFmpegH264Encoder 内部按 nvenc/qsv/amf/mf 探）
|
||||||
|
// x264 软编（CX264Encoder，CPU 兜底）
|
||||||
|
//
|
||||||
|
// 失败路径在日志中可见（Mprintf）。返回 nullptr 仅在 x264 也开不起来时（极少见）。
|
||||||
|
std::unique_ptr<VideoEncoderBase> CreateEncoder(const EncoderRequest& req);
|
||||||
@@ -13,8 +13,11 @@
|
|||||||
#include <condition_variable>
|
#include <condition_variable>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <future>
|
#include <future>
|
||||||
|
#include <memory>
|
||||||
#include <emmintrin.h> // SSE2
|
#include <emmintrin.h> // SSE2
|
||||||
#include "X264Encoder.h"
|
#include "common/config.h"
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include "EncoderFactory.h"
|
||||||
#include "ScrollDetector.h"
|
#include "ScrollDetector.h"
|
||||||
#include "common/file_upload.h"
|
#include "common/file_upload.h"
|
||||||
|
|
||||||
@@ -126,6 +129,7 @@ public:
|
|||||||
ULONG* m_BlockSizes; // 分块差异像素数
|
ULONG* m_BlockSizes; // 分块差异像素数
|
||||||
int m_BlockNum; // 分块个数
|
int m_BlockNum; // 分块个数
|
||||||
int m_SendQuality; // 发送质量
|
int m_SendQuality; // 发送质量
|
||||||
|
int m_EncodeLevel; // 编码级别
|
||||||
|
|
||||||
LPBITMAPINFO m_BitmapInfor_Full; // BMP信息
|
LPBITMAPINFO m_BitmapInfor_Full; // BMP信息
|
||||||
LPBITMAPINFO m_BitmapInfor_Send; // 发送的BMP信息
|
LPBITMAPINFO m_BitmapInfor_Send; // 发送的BMP信息
|
||||||
@@ -145,7 +149,7 @@ public:
|
|||||||
int m_FrameID; // 帧序号
|
int m_FrameID; // 帧序号
|
||||||
int m_GOP; // 关键帧间隔
|
int m_GOP; // 关键帧间隔
|
||||||
bool m_SendKeyFrame; // 发送关键帧
|
bool m_SendKeyFrame; // 发送关键帧
|
||||||
CX264Encoder *m_encoder; // 编码器
|
std::unique_ptr<VideoEncoderBase> m_encoder; // 编码器，ensureEncoder() lazy 创建，走 EncoderFactory 探测
|
||||||
int m_nScreenCount; // 屏幕数量
|
int m_nScreenCount; // 屏幕数量
|
||||||
BOOL m_bEnableMultiScreen;// 多显示器支持
|
BOOL m_bEnableMultiScreen;// 多显示器支持
|
||||||
|
|
||||||
@@ -182,14 +186,14 @@ protected:
|
|||||||
int m_nVScreenHeight = GetSystemMetrics(SM_CYVIRTUALSCREEN);
|
int m_nVScreenHeight = GetSystemMetrics(SM_CYVIRTUALSCREEN);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
ScreenCapture(int n = 32, BYTE algo = ALGORITHM_DIFF, BOOL all = FALSE) :
|
ScreenCapture(int n = 32, BYTE algo = ALGORITHM_DIFF, BOOL all = FALSE, int level = LEVEL_H264_SOFT) :
|
||||||
m_ThreadPool(nullptr), m_FirstBuffer(nullptr), m_RectBuffer(nullptr),
|
m_ThreadPool(nullptr), m_FirstBuffer(nullptr), m_RectBuffer(nullptr),
|
||||||
m_BitmapInfor_Full(nullptr), m_bAlgorithm(algo), m_SendQuality(100),
|
m_BitmapInfor_Full(nullptr), m_bAlgorithm(algo), m_SendQuality(100),
|
||||||
m_ulFullWidth(0), m_ulFullHeight(0), m_bZoomed(false), m_wZoom(1), m_hZoom(1),
|
m_ulFullWidth(0), m_ulFullHeight(0), m_bZoomed(false), m_wZoom(1), m_hZoom(1),
|
||||||
m_FrameID(0), m_GOP(DEFAULT_GOP), m_iScreenX(0), m_iScreenY(0), m_biBitCount(n),
|
m_FrameID(0), m_GOP(DEFAULT_GOP), m_iScreenX(0), m_iScreenY(0), m_biBitCount(n),
|
||||||
m_SendKeyFrame(false), m_encoder(nullptr),
|
m_SendKeyFrame(false), m_encoder(nullptr),
|
||||||
m_pScrollDetector(nullptr), m_bEnableScrollDetect(false), m_bServerSupportsScroll(false),
|
m_pScrollDetector(nullptr), m_bEnableScrollDetect(false), m_bServerSupportsScroll(false),
|
||||||
m_bLastFrameWasScroll(false), m_nScrollDetectInterval(1)
|
m_bLastFrameWasScroll(false), m_nScrollDetectInterval(1), m_EncodeLevel(level)
|
||||||
{
|
{
|
||||||
SetAlgorithm(algo);
|
SetAlgorithm(algo);
|
||||||
m_BitmapInfor_Send = nullptr;
|
m_BitmapInfor_Send = nullptr;
|
||||||
@@ -256,7 +260,6 @@ public:
|
|||||||
SAFE_DELETE_ARRAY(m_BlockSizes);
|
SAFE_DELETE_ARRAY(m_BlockSizes);
|
||||||
|
|
||||||
SAFE_DELETE(m_ThreadPool);
|
SAFE_DELETE(m_ThreadPool);
|
||||||
SAFE_DELETE(m_encoder);
|
|
||||||
SAFE_DELETE(m_pScrollDetector);
|
SAFE_DELETE(m_pScrollDetector);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -839,6 +842,19 @@ public:
|
|||||||
return bmpInfo;
|
return bmpInfo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 编码器 lazy 创建。委托 EncoderFactory 完成"硬编探测 + 软编 fallback"。
|
||||||
|
void ensureEncoder(int width, int height)
|
||||||
|
{
|
||||||
|
if (m_encoder) return;
|
||||||
|
EncoderRequest req;
|
||||||
|
req.width = width;
|
||||||
|
req.height = height;
|
||||||
|
req.fps = 20;
|
||||||
|
req.bitrate_kbps = (m_nBitRate > 0) ? m_nBitRate : (width * height / 1266);
|
||||||
|
req.encodeLevel = m_EncodeLevel;
|
||||||
|
m_encoder = CreateEncoder(req);
|
||||||
|
}
|
||||||
|
|
||||||
// 算法+光标位置+光标类型
|
// 算法+光标位置+光标类型
|
||||||
virtual LPBYTE GetNextScreenData(ULONG* ulNextSendLength)
|
virtual LPBYTE GetNextScreenData(ULONG* ulNextSendLength)
|
||||||
{
|
{
|
||||||
@@ -923,13 +939,12 @@ public:
|
|||||||
uint8_t* encoded_data = nullptr;
|
uint8_t* encoded_data = nullptr;
|
||||||
uint32_t encoded_size = 0;
|
uint32_t encoded_size = 0;
|
||||||
int width = m_BitmapInfor_Send->bmiHeader.biWidth, height = m_BitmapInfor_Send->bmiHeader.biHeight;
|
int width = m_BitmapInfor_Send->bmiHeader.biWidth, height = m_BitmapInfor_Send->bmiHeader.biHeight;
|
||||||
if (m_encoder == nullptr) {
|
ensureEncoder(width, height);
|
||||||
m_encoder = new CX264Encoder();
|
if (!m_encoder) return nullptr;
|
||||||
int br = (m_nBitRate > 0) ? m_nBitRate : (width * height / 1266);
|
m_encoder->forceIDR(); // 协议层 keyframe → 编码器强制 IDR，与 TOKEN_KEYFRAME 语义对齐
|
||||||
m_encoder->open(width, height, 20, BitRateToCRF(br));
|
|
||||||
}
|
|
||||||
int err = m_encoder->encode(nextData, 32, 4 * width, width, height, &encoded_data, &encoded_size);
|
int err = m_encoder->encode(nextData, 32, 4 * width, width, height, &encoded_data, &encoded_size);
|
||||||
if (err) {
|
// encoded_size == 0：硬编首帧延迟（avcodec_receive_packet 返回 EAGAIN），本帧无码流，按失败跳过
|
||||||
|
if (err || encoded_size == 0) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
*ulNextSendLength = 1 + offset + encoded_size;
|
*ulNextSendLength = 1 + offset + encoded_size;
|
||||||
@@ -953,13 +968,11 @@ public:
|
|||||||
uint8_t* encoded_data = nullptr;
|
uint8_t* encoded_data = nullptr;
|
||||||
uint32_t encoded_size = 0;
|
uint32_t encoded_size = 0;
|
||||||
int width = m_BitmapInfor_Send->bmiHeader.biWidth, height = m_BitmapInfor_Send->bmiHeader.biHeight;
|
int width = m_BitmapInfor_Send->bmiHeader.biWidth, height = m_BitmapInfor_Send->bmiHeader.biHeight;
|
||||||
if (m_encoder == nullptr) {
|
ensureEncoder(width, height);
|
||||||
m_encoder = new CX264Encoder();
|
if (!m_encoder) return nullptr;
|
||||||
int br = (m_nBitRate > 0) ? m_nBitRate : (width * height / 1266);
|
|
||||||
m_encoder->open(width, height, 20, BitRateToCRF(br));
|
|
||||||
}
|
|
||||||
int err = m_encoder->encode(nextData, 32, 4 * width, width, height, &encoded_data, &encoded_size);
|
int err = m_encoder->encode(nextData, 32, 4 * width, width, height, &encoded_data, &encoded_size);
|
||||||
if (err) {
|
// encoded_size == 0：硬编首帧延迟，本帧无码流，按失败跳过
|
||||||
|
if (err || encoded_size == 0) {
|
||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
*ulNextSendLength = 1 + offset + encoded_size;
|
*ulNextSendLength = 1 + offset + encoded_size;
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ private:
|
|||||||
BYTE* m_NextBuffer = nullptr;
|
BYTE* m_NextBuffer = nullptr;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
ScreenCapturerDXGI(BYTE algo, int gop = DEFAULT_GOP, BOOL all = FALSE) : ScreenCapture(32, algo, all)
|
ScreenCapturerDXGI(BYTE algo, int gop = DEFAULT_GOP, BOOL all = FALSE, int level = LEVEL_H264_SOFT)
|
||||||
|
: ScreenCapture(32, algo, all, level)
|
||||||
{
|
{
|
||||||
m_GOP = gop;
|
m_GOP = gop;
|
||||||
InitDXGI(all);
|
InitDXGI(all);
|
||||||
|
|||||||
@@ -154,6 +154,7 @@ CScreenManager::CScreenManager(IOCPClient* ClientObject, int n, void* user, BOOL
|
|||||||
m_ScreenSettings.QualityLevel = cfg.GetInt("settings", "QualityLevel", quality);
|
m_ScreenSettings.QualityLevel = cfg.GetInt("settings", "QualityLevel", quality);
|
||||||
m_ScreenSettings.CpuSpeedup = cfg.GetInt("settings", "CpuSpeedup", 0);
|
m_ScreenSettings.CpuSpeedup = cfg.GetInt("settings", "CpuSpeedup", 0);
|
||||||
m_ScreenSettings.AudioEnabled = cfg.GetInt("settings", "AudioEnabled", 0); // 默认禁用音频
|
m_ScreenSettings.AudioEnabled = cfg.GetInt("settings", "AudioEnabled", 0); // 默认禁用音频
|
||||||
|
m_ScreenSettings.EncodeLevel = cfg.GetInt("settings", "EncodeLevel", LEVEL_H264_SOFT);
|
||||||
|
|
||||||
LoadQualityProfiles(); // 加载质量配置
|
LoadQualityProfiles(); // 加载质量配置
|
||||||
|
|
||||||
@@ -519,18 +520,18 @@ void CScreenManager::InitScreenSpy()
|
|||||||
SAFE_DELETE(m_ScreenSpyObject);
|
SAFE_DELETE(m_ScreenSpyObject);
|
||||||
if ((USING_DXGI == DXGI && IsWindows8orHigher())) {
|
if ((USING_DXGI == DXGI && IsWindows8orHigher())) {
|
||||||
m_isGDI = FALSE;
|
m_isGDI = FALSE;
|
||||||
auto s = new ScreenCapturerDXGI(algo, DEFAULT_GOP, all);
|
auto s = new ScreenCapturerDXGI(algo, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel);
|
||||||
if (s->IsInitSucceed()) {
|
if (s->IsInitSucceed()) {
|
||||||
m_ScreenSpyObject = s;
|
m_ScreenSpyObject = s;
|
||||||
} else {
|
} else {
|
||||||
SAFE_DELETE(s);
|
SAFE_DELETE(s);
|
||||||
m_isGDI = TRUE;
|
m_isGDI = TRUE;
|
||||||
m_ScreenSpyObject = new CScreenSpy(32, algo, FALSE, DEFAULT_GOP, all);
|
m_ScreenSpyObject = new CScreenSpy(32, algo, FALSE, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel);
|
||||||
Mprintf("CScreenManager: DXGI SPY init failed!!! Using GDI instead.\n");
|
Mprintf("CScreenManager: DXGI SPY init failed!!! Using GDI instead.\n");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
m_isGDI = TRUE;
|
m_isGDI = TRUE;
|
||||||
m_ScreenSpyObject = new CScreenSpy(32, algo, DXGI == USING_VIRTUAL, DEFAULT_GOP, all);
|
m_ScreenSpyObject = new CScreenSpy(32, algo, DXGI == USING_VIRTUAL, DEFAULT_GOP, all, m_ScreenSettings.EncodeLevel);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -817,6 +818,14 @@ VOID CScreenManager::OnReceive(PBYTE szBuffer, ULONG ulLength)
|
|||||||
m_ClientObject->StopRunning();
|
m_ClientObject->StopRunning();
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case COMMAND_ENCODE_LEVEL: {
|
||||||
|
int encodeLevel = szBuffer[1];
|
||||||
|
iniFile cfg(CLIENT_PATH);
|
||||||
|
cfg.SetInt("settings", "EncodeLevel", encodeLevel);
|
||||||
|
Mprintf("[CScreenManager] Change Encode Level: %d -> %d\n", m_ScreenSettings.EncodeLevel, encodeLevel);
|
||||||
|
m_ScreenSettings.EncodeLevel = encodeLevel;
|
||||||
|
break;
|
||||||
|
}
|
||||||
case COMMAND_SWITCH_SCREEN: {
|
case COMMAND_SWITCH_SCREEN: {
|
||||||
SwitchScreen();
|
SwitchScreen();
|
||||||
break;
|
break;
|
||||||
|
|||||||
@@ -12,8 +12,8 @@
|
|||||||
// Construction/Destruction
|
// Construction/Destruction
|
||||||
//////////////////////////////////////////////////////////////////////
|
//////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
CScreenSpy::CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk, int gop, BOOL all) :
|
CScreenSpy::CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk, int gop, BOOL all, int level) :
|
||||||
ScreenCapture(ulbiBitCount, algo, all)
|
ScreenCapture(ulbiBitCount, algo, all, level)
|
||||||
{
|
{
|
||||||
m_GOP = gop;
|
m_GOP = gop;
|
||||||
|
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ protected:
|
|||||||
EnumHwndsPrintData m_data;
|
EnumHwndsPrintData m_data;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk = FALSE, int gop = DEFAULT_GOP, BOOL all = FALSE);
|
CScreenSpy(ULONG ulbiBitCount, BYTE algo, BOOL vDesk = FALSE, int gop = DEFAULT_GOP, BOOL all = FALSE, int level = LEVEL_H264_SOFT);
|
||||||
|
|
||||||
virtual ~CScreenSpy();
|
virtual ~CScreenSpy();
|
||||||
|
|
||||||
|
|||||||
59
client/VideoEncoderBase.h
Normal file
59
client/VideoEncoderBase.h
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
#pragma once
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
// 视频编码器抽象接口
|
||||||
|
// Step 0: 仅 CX264Encoder 实现；后续 CFFmpegH264Encoder / CFFmpegAV1Encoder 接入
|
||||||
|
// 详见 docs/HardwareEncoding_Design.md
|
||||||
|
|
||||||
|
enum class VideoCodec {
|
||||||
|
H264,
|
||||||
|
AV1,
|
||||||
|
};
|
||||||
|
|
||||||
|
enum class RateControl {
|
||||||
|
CRF, // x264 软编用 CRF (0-51, 越小越好)
|
||||||
|
BITRATE, // 硬编路径用目标码率 (kbps)
|
||||||
|
};
|
||||||
|
|
||||||
|
struct EncoderParams {
|
||||||
|
int width = 0;
|
||||||
|
int height = 0;
|
||||||
|
int fps = 30;
|
||||||
|
RateControl rc = RateControl::BITRATE;
|
||||||
|
int crf = 23; // 当 rc == CRF
|
||||||
|
int bitrate_kbps = 4000; // 当 rc == BITRATE
|
||||||
|
int gop_seconds = 15; // 关键帧间隔（秒），与 x264 i_keyint_max=fps*15 对齐
|
||||||
|
};
|
||||||
|
|
||||||
|
class VideoEncoderBase {
|
||||||
|
public:
|
||||||
|
virtual ~VideoEncoderBase() = default;
|
||||||
|
|
||||||
|
virtual bool open(const EncoderParams& params) = 0;
|
||||||
|
virtual void close() = 0;
|
||||||
|
|
||||||
|
// 编码一帧
|
||||||
|
// rgb : 输入像素数据
|
||||||
|
// bpp : 24 (RGB) / 32 (BGRA)
|
||||||
|
// stride : 源行字节数
|
||||||
|
// width/height : 图像尺寸
|
||||||
|
// lppData : 输出指针，指向编码后码流（生命周期归编码器，下一次 encode 失效）
|
||||||
|
// lpSize : 输出码流字节数；返回 0 表示成功但本帧无输出（硬编首帧延迟）
|
||||||
|
// direction : 1 = 上下不翻转，-1 = 翻转（适配 Windows BMP bottom-up）
|
||||||
|
// 返回 0 = 成功；< 0 = 失败
|
||||||
|
virtual int encode(
|
||||||
|
uint8_t* rgb,
|
||||||
|
uint8_t bpp,
|
||||||
|
uint32_t stride,
|
||||||
|
uint32_t width,
|
||||||
|
uint32_t height,
|
||||||
|
uint8_t** lppData,
|
||||||
|
uint32_t* lpSize,
|
||||||
|
int direction = 1
|
||||||
|
) = 0;
|
||||||
|
|
||||||
|
virtual void forceIDR() = 0;
|
||||||
|
virtual void setBitrate(int kbps) {} // 可选实现，默认 no-op
|
||||||
|
virtual VideoCodec codec() const = 0;
|
||||||
|
virtual const char* backendName() const = 0; // "x264" / "h264_nvenc" / "av1_amf" ...
|
||||||
|
};
|
||||||
@@ -3,10 +3,11 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#if DISABLE_X264_FOR_TEST
|
#if DISABLE_X264_FOR_TEST
|
||||||
CX264Encoder::CX264Encoder() { memset(&m_Param, 0, sizeof(m_Param)); m_pCodec = NULL; m_pPicIn = NULL; m_pPicOut = NULL; }
|
CX264Encoder::CX264Encoder() { memset(&m_Param, 0, sizeof(m_Param)); m_pCodec = NULL; m_pPicIn = NULL; m_pPicOut = NULL; m_forceIDR = false; }
|
||||||
CX264Encoder::~CX264Encoder() {}
|
CX264Encoder::~CX264Encoder() {}
|
||||||
bool CX264Encoder::open(int, int, int, int) { return false; }
|
bool CX264Encoder::open(int, int, int, int) { return false; }
|
||||||
bool CX264Encoder::open(x264_param_t*) { return false; }
|
bool CX264Encoder::open(x264_param_t*) { return false; }
|
||||||
|
bool CX264Encoder::open(const EncoderParams&) { return false; }
|
||||||
void CX264Encoder::close() {}
|
void CX264Encoder::close() {}
|
||||||
int CX264Encoder::encode(uint8_t*, uint8_t, uint32_t, uint32_t, uint32_t, uint8_t**, uint32_t*, int) { return -1; }
|
int CX264Encoder::encode(uint8_t*, uint8_t, uint32_t, uint32_t, uint32_t, uint8_t**, uint32_t*, int) { return -1; }
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ CX264Encoder::CX264Encoder()
|
|||||||
m_pCodec = NULL;
|
m_pCodec = NULL;
|
||||||
m_pPicIn = NULL;
|
m_pPicIn = NULL;
|
||||||
m_pPicOut = NULL;
|
m_pPicOut = NULL;
|
||||||
|
m_forceIDR = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -88,6 +90,14 @@ bool CX264Encoder::open(x264_param_t * param)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
bool CX264Encoder::open(const EncoderParams& params)
|
||||||
|
{
|
||||||
|
// x264 软编只支持 CRF；调用方走 BITRATE 时降级为 CRF=23（与 BitRateToCRF 默认一致）
|
||||||
|
int crf = (params.rc == RateControl::CRF) ? params.crf : 23;
|
||||||
|
return open(params.width, params.height, params.fps, crf);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void CX264Encoder::close()
|
void CX264Encoder::close()
|
||||||
{
|
{
|
||||||
if (m_pCodec) {
|
if (m_pCodec) {
|
||||||
@@ -146,6 +156,12 @@ int CX264Encoder::encode(
|
|||||||
return -2;
|
return -2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (m_forceIDR) {
|
||||||
|
m_pPicIn->i_type = X264_TYPE_IDR;
|
||||||
|
m_forceIDR = false;
|
||||||
|
} else {
|
||||||
|
m_pPicIn->i_type = X264_TYPE_AUTO;
|
||||||
|
}
|
||||||
|
|
||||||
encode_size = x264_encoder_encode(
|
encode_size = x264_encoder_encode(
|
||||||
m_pCodec,
|
m_pCodec,
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#include <libyuv\libyuv.h>
|
#include <libyuv\libyuv.h>
|
||||||
#include <x264\x264.h>
|
#include <x264\x264.h>
|
||||||
@@ -7,19 +9,22 @@ extern "C" {
|
|||||||
|
|
||||||
#include "common/config.h"
|
#include "common/config.h"
|
||||||
|
|
||||||
class CX264Encoder
|
class CX264Encoder : public VideoEncoderBase
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
x264_t* m_pCodec; //编码器实例
|
x264_t* m_pCodec; //编码器实例
|
||||||
x264_picture_t *m_pPicIn;
|
x264_picture_t *m_pPicIn;
|
||||||
x264_picture_t *m_pPicOut;
|
x264_picture_t *m_pPicOut;
|
||||||
x264_param_t m_Param;
|
x264_param_t m_Param;
|
||||||
|
bool m_forceIDR; // 下一次 encode 强制 IDR
|
||||||
public:
|
public:
|
||||||
|
// 旧签名保留：被 ScreenCapture 临时直接调；新增 EncoderParams overload 走接口路径
|
||||||
bool open(int width, int height, int fps, int crf);
|
bool open(int width, int height, int fps, int crf);
|
||||||
bool open(x264_param_t * param);
|
bool open(x264_param_t * param);
|
||||||
|
|
||||||
void close();
|
// VideoEncoderBase
|
||||||
|
bool open(const EncoderParams& params) override;
|
||||||
|
void close() override;
|
||||||
int encode(
|
int encode(
|
||||||
uint8_t * rgb,
|
uint8_t * rgb,
|
||||||
uint8_t bpp,
|
uint8_t bpp,
|
||||||
@@ -29,9 +34,11 @@ public:
|
|||||||
uint8_t ** lppData,
|
uint8_t ** lppData,
|
||||||
uint32_t * lpSize,
|
uint32_t * lpSize,
|
||||||
int direction = 1
|
int direction = 1
|
||||||
);
|
) override;
|
||||||
|
void forceIDR() override { m_forceIDR = true; }
|
||||||
|
VideoCodec codec() const override { return VideoCodec::H264; }
|
||||||
|
const char* backendName() const override { return "x264"; }
|
||||||
|
|
||||||
CX264Encoder();
|
CX264Encoder();
|
||||||
~CX264Encoder();
|
~CX264Encoder() override;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -130,7 +130,7 @@
|
|||||||
</EntryPointSymbol>
|
</EntryPointSymbol>
|
||||||
<SubSystem>Console</SubSystem>
|
<SubSystem>Console</SubSystem>
|
||||||
<AdditionalOptions>/ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
<AdditionalOptions>/ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
||||||
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin</AdditionalLibraryDirectories>
|
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin;$(SolutionDir)..\ffmpeg-7.1\install-win64\lib</AdditionalLibraryDirectories>
|
||||||
</Link>
|
</Link>
|
||||||
</ItemDefinitionGroup>
|
</ItemDefinitionGroup>
|
||||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
|
||||||
@@ -177,7 +177,7 @@
|
|||||||
<AdditionalOptions> /SAFESEH:NO /ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
<AdditionalOptions> /SAFESEH:NO /ignore:4099 %(AdditionalOptions)</AdditionalOptions>
|
||||||
<SubSystem>Windows</SubSystem>
|
<SubSystem>Windows</SubSystem>
|
||||||
<EntryPointSymbol>mainCRTStartup</EntryPointSymbol>
|
<EntryPointSymbol>mainCRTStartup</EntryPointSymbol>
|
||||||
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin</AdditionalLibraryDirectories>
|
<AdditionalLibraryDirectories>$(SolutionDir)..\SimplePlugins\bin;$(SolutionDir)..\ffmpeg-7.1\install-win64\lib</AdditionalLibraryDirectories>
|
||||||
</Link>
|
</Link>
|
||||||
</ItemDefinitionGroup>
|
</ItemDefinitionGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@@ -218,6 +218,9 @@
|
|||||||
<ClCompile Include="ConPTYManager.cpp" />
|
<ClCompile Include="ConPTYManager.cpp" />
|
||||||
<ClCompile Include="StdAfx.cpp" />
|
<ClCompile Include="StdAfx.cpp" />
|
||||||
<ClCompile Include="SystemManager.cpp" />
|
<ClCompile Include="SystemManager.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegAV1Encoder.cpp" />
|
||||||
|
<ClCompile Include="CFFmpegH264Encoder.cpp" />
|
||||||
|
<ClCompile Include="EncoderFactory.cpp" />
|
||||||
<ClCompile Include="TalkManager.cpp" />
|
<ClCompile Include="TalkManager.cpp" />
|
||||||
<ClCompile Include="VideoManager.cpp" />
|
<ClCompile Include="VideoManager.cpp" />
|
||||||
<ClCompile Include="X264Encoder.cpp" />
|
<ClCompile Include="X264Encoder.cpp" />
|
||||||
@@ -266,7 +269,11 @@
|
|||||||
<ClInclude Include="ShellManager.h" />
|
<ClInclude Include="ShellManager.h" />
|
||||||
<ClInclude Include="ConPTYManager.h" />
|
<ClInclude Include="ConPTYManager.h" />
|
||||||
<ClInclude Include="StdAfx.h" />
|
<ClInclude Include="StdAfx.h" />
|
||||||
|
<ClInclude Include="CFFmpegAV1Encoder.h" />
|
||||||
|
<ClInclude Include="CFFmpegH264Encoder.h" />
|
||||||
|
<ClInclude Include="EncoderFactory.h" />
|
||||||
<ClInclude Include="SystemManager.h" />
|
<ClInclude Include="SystemManager.h" />
|
||||||
|
<ClInclude Include="VideoEncoderBase.h" />
|
||||||
<ClInclude Include="TalkManager.h" />
|
<ClInclude Include="TalkManager.h" />
|
||||||
<ClInclude Include="VideoCodec.h" />
|
<ClInclude Include="VideoCodec.h" />
|
||||||
<ClInclude Include="VideoManager.h" />
|
<ClInclude Include="VideoManager.h" />
|
||||||
|
|||||||
@@ -255,6 +255,7 @@ enum {
|
|||||||
CMD_AUDIO_CTRL = 95, // 音频控制: [cmd:1][enable:1][persist:1]
|
CMD_AUDIO_CTRL = 95, // 音频控制: [cmd:1][enable:1][persist:1]
|
||||||
TOKEN_SCREEN_AUDIO = 96, // 音频数据: [token:1][hasFormat:1][AudioFormat?][data]
|
TOKEN_SCREEN_AUDIO = 96, // 音频数据: [token:1][hasFormat:1][AudioFormat?][data]
|
||||||
COMMAND_SHARE_CANCEL = 97,
|
COMMAND_SHARE_CANCEL = 97,
|
||||||
|
COMMAND_ENCODE_LEVEL = 98,
|
||||||
|
|
||||||
TOKEN_SCROLL_FRAME = 99, // 滚动优化帧
|
TOKEN_SCROLL_FRAME = 99, // 滚动优化帧
|
||||||
// 服务端发出的标识
|
// 服务端发出的标识
|
||||||
@@ -1188,6 +1189,12 @@ enum QualityLevel {
|
|||||||
#define ALGORITHM_RGB565 3 // RGB565 压缩
|
#define ALGORITHM_RGB565 3 // RGB565 压缩
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
enum EncodeLevel {
|
||||||
|
LEVEL_H264_SOFT = 0,
|
||||||
|
LEVEL_H264_HARD = 1,
|
||||||
|
LEVEL_AV1_HARD = 2,
|
||||||
|
};
|
||||||
|
|
||||||
/* 质量配置(与 QualityLevel 对应)
|
/* 质量配置(与 QualityLevel 对应)
|
||||||
- strategy = 0：1080p 限制
|
- strategy = 0：1080p 限制
|
||||||
- strategy = 1：原始分辨率
|
- strategy = 1：原始分辨率
|
||||||
@@ -1272,7 +1279,8 @@ typedef struct ScreenSettings {
|
|||||||
int CpuSpeedup; // 偏移 36, 指令集加速(0: 无, 1: SSE2)
|
int CpuSpeedup; // 偏移 36, 指令集加速(0: 无, 1: SSE2)
|
||||||
int ScreenType; // 偏移 40, 屏幕类型(0: GDI, 1: DXGI, 2: Virtual)
|
int ScreenType; // 偏移 40, 屏幕类型(0: GDI, 1: DXGI, 2: Virtual)
|
||||||
int AudioEnabled; // 偏移 44, 音频传输(0: 禁用, 1: 启用)
|
int AudioEnabled; // 偏移 44, 音频传输(0: 禁用, 1: 启用)
|
||||||
char Reserved[48]; // 偏移 48, 保留字段（新能力参数从此处扩展）
|
int EncodeLevel; // 偏移 48, 编码等级
|
||||||
|
char Reserved[44]; // 偏移 52, 保留字段（新能力参数从此处扩展）
|
||||||
uint32_t Capabilities; // 偏移 96, 能力位标志（放最后）
|
uint32_t Capabilities; // 偏移 96, 能力位标志（放最后）
|
||||||
} ScreenSettings; // 总大小 100 字节
|
} ScreenSettings; // 总大小 100 字节
|
||||||
|
|
||||||
|
|||||||
BIN
compress/ffmpeg/dav1d_x64.lib
Normal file
BIN
compress/ffmpeg/dav1d_x64.lib
Normal file
Binary file not shown.
BIN
compress/ffmpeg/vpl_x64.lib
Normal file
BIN
compress/ffmpeg/vpl_x64.lib
Normal file
Binary file not shown.
BIN
compress/ffmpeg/vpl_x64d.lib
Normal file
BIN
compress/ffmpeg/vpl_x64d.lib
Normal file
Binary file not shown.
977
docs/HardwareEncoding_Design.md
Normal file
977
docs/HardwareEncoding_Design.md
Normal file
@@ -0,0 +1,977 @@
|
|||||||
|
# 视频编码硬件加速实现指导文档
|
||||||
|
|
||||||
|
本文档供 AI 编码助手参考，用于在现有 C++ 远程控制程序中实现 H.264 硬件编码 + AV1 编码路径。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 项目背景
|
||||||
|
|
||||||
|
### 1.1 当前状态
|
||||||
|
|
||||||
|
- C++ Windows 远程控制程序
|
||||||
|
- 已实现 H.264 编码，基于 x264 软编（`CX264Encoder`），preset = `ultrafast + zerolatency`
|
||||||
|
- 视频管线：桌面捕获（RGB/BGRA）→ 编码 → 网络传输 → 客户端解码显示
|
||||||
|
- 当前架构：每个主控端连接对应一个独立编码器实例
|
||||||
|
- **分发模式**：单 exe，FFmpeg 静态链接
|
||||||
|
|
||||||
|
### 1.2 目标
|
||||||
|
|
||||||
|
分两阶段渐进推进，**始终保留 x264 软编作为兜底**：
|
||||||
|
|
||||||
|
**阶段一（H.264 硬编加速）**
|
||||||
|
- 新增 H.264 硬编（NVENC / QSV / AMF），按 GPU 能力探测优先走硬编
|
||||||
|
- x264 软编在无 GPU / 虚拟机 / 远程桌面会话等环境下兜底
|
||||||
|
- 浏览器解码零兼容性风险（H.264 全平台原生支持）
|
||||||
|
|
||||||
|
**阶段二（AV1 路径）**
|
||||||
|
- 新增 AV1 硬编（`av1_nvenc` / `av1_qsv` / `av1_amf`）
|
||||||
|
- 客户端浏览器握手时声明 AV1 能力
|
||||||
|
- 双方都能用就走 AV1，否则回落 H.264
|
||||||
|
|
||||||
|
**最终产物仍为单 exe**，体积增量可接受 6–10 MB。
|
||||||
|
|
||||||
|
### 1.3 关键决策记录
|
||||||
|
|
||||||
|
#### 1.3.1 为什么跳过 HEVC
|
||||||
|
|
||||||
|
经评估，HEVC 在本项目目标场景下没有独占价值：
|
||||||
|
|
||||||
|
| 维度 | 现状 |
|
||||||
|
|---|---|
|
||||||
|
| **浏览器解码** | Firefox 完全不支持；Chrome/Edge 需 Win11 + 商店付费的 HEVC Video Extensions |
|
||||||
|
| **专利授权** | 商用涉及 MPEG-LA / Access Advance / Velos Media 三个专利池 |
|
||||||
|
| **替代方案** | AV1 压缩效率更高、AOMedia 免专利、浏览器原生支持广 |
|
||||||
|
|
||||||
|
HEVC 编码端硬件普及度好（几乎所有 2015+ GPU）这个优势，被解码端短板完全抵消。
|
||||||
|
|
||||||
|
#### 1.3.2 为什么 H.264 硬编先于 AV1
|
||||||
|
|
||||||
|
- **AV1 硬编硬件门槛高**：仅 NVIDIA RTX 40+ / AMD RX 7000+ / Intel Arc 才有
|
||||||
|
- **"多机混杂"场景**下大部分编码端 GPU 没有 AV1 硬编
|
||||||
|
- **H.264 硬编**（NVENC/QSV/AMF）几乎所有现代 GPU 都有，覆盖面广
|
||||||
|
- **客户端浏览器解 H.264** 是零兼容性问题，跨浏览器/跨平台 100% 通用
|
||||||
|
|
||||||
|
H.264 硬编是先把"地板抬起来"，AV1 是"在新硬件上的天花板"。
|
||||||
|
|
||||||
|
### 1.4 设计约束
|
||||||
|
|
||||||
|
- **平台**：仅 Windows（macOS/Linux 未来另行设计）
|
||||||
|
- **GPU 不确定**：NVIDIA / AMD / Intel / 无独显 / 虚拟机无 GPU 都需支持
|
||||||
|
- **延迟要求**：不敏感（不追求极致低延迟）
|
||||||
|
- **并发模型**：通常 1 对 1，少数 1 对多（每个连接独立编码器）
|
||||||
|
- **客户端**：浏览器（WebCodecs 优先，`<video>` 次之），未来集成
|
||||||
|
- **工具链**：Visual Studio 2019
|
||||||
|
- **属性**：个人项目，暂不商用，专利问题搁置但仍优先选免专利方案
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 技术方案总览
|
||||||
|
|
||||||
|
### 2.1 编码器优先级链
|
||||||
|
|
||||||
|
```
|
||||||
|
新连接进入（带客户端能力）
|
||||||
|
│
|
||||||
|
├─ 客户端声明支持 AV1？─── 否 ────┐
|
||||||
|
│ 是 │
|
||||||
|
│ ↓ │
|
||||||
|
├─ av1_nvenc/qsv/amf 能开？──┐ │
|
||||||
|
│ │ │ │
|
||||||
|
│ 成功 → 用 AV1 │ │
|
||||||
|
│ ↓ ↓
|
||||||
|
└─ h264_nvenc/qsv/amf/mf 能开？──┐
|
||||||
|
│ │
|
||||||
|
成功 → 用 H.264 硬编 │
|
||||||
|
↓
|
||||||
|
x264 软编（始终可用）
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 编码器后端表
|
||||||
|
|
||||||
|
| 类型 | FFmpeg 编码器名 | 硬件要求 | 备注 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| AV1 硬编 | `av1_nvenc` | NVIDIA RTX 40+（Ada Lovelace） | 2022 Q4 起 |
|
||||||
|
| AV1 硬编 | `av1_amf` | AMD RX 7000+（RDNA 3） | 2022 Q4 起 |
|
||||||
|
| AV1 硬编 | `av1_qsv` | Intel Arc / 部分新 Iris Xe | 2022 起 |
|
||||||
|
| H.264 硬编 | `h264_nvenc` | 几乎所有 NVIDIA GPU（GTX 650+） | 2012 起 |
|
||||||
|
| H.264 硬编 | `h264_qsv` | 几乎所有 Intel 核显（HD 4000+） | 2012 起 |
|
||||||
|
| H.264 硬编 | `h264_amf` | 几乎所有 AMD GPU | |
|
||||||
|
| H.264 硬编 | `h264_mf` | Windows Media Foundation | 兜底，质量/稳定性一般 |
|
||||||
|
| H.264 软编 | `libx264`（现有 `CX264Encoder`） | 任意 CPU | 始终兜底 |
|
||||||
|
|
||||||
|
**不使用 `libx265` / `libaom-av1` / `libsvtav1`**（CPU 软编），原因：
|
||||||
|
- 远控产品对 CPU 占用敏感，AV1/HEVC 软编实时编码压力大
|
||||||
|
- `libx265` 会让 FFmpeg 切到 GPL，`libaom-av1` 编码速度也不够
|
||||||
|
|
||||||
|
### 2.3 类结构
|
||||||
|
|
||||||
|
```
|
||||||
|
VideoEncoderBase（新增抽象接口）
|
||||||
|
├── CX264Encoder （改造现有类继承接口，保留软编兜底）
|
||||||
|
├── CFFmpegH264Encoder （新增，封装 h264_nvenc/qsv/amf/mf）
|
||||||
|
└── CFFmpegAV1Encoder （新增，封装 av1_nvenc/qsv/amf）
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.4 协商流程
|
||||||
|
|
||||||
|
```
|
||||||
|
握手阶段：
|
||||||
|
- 客户端（浏览器）在 WebSocket 握手时上报能力：
|
||||||
|
{ "codecs": ["av1", "h264"] } // 浏览器实际能解的，按优先级排
|
||||||
|
- 服务端取「客户端能力 ∩ 自己硬件能力」选 codec
|
||||||
|
|
||||||
|
会话阶段：
|
||||||
|
- 选定 codec 后创建对应编码器，整个连接生命周期不变
|
||||||
|
- 运行中不切换 codec（保持简单，需要切换就重连）
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 硬编 vs 现有 x264 软编对比
|
||||||
|
|
||||||
|
### 3.1 CPU 占用（最大收益）
|
||||||
|
|
||||||
|
| 编码器 | 1080p @ 30fps CPU 占用 |
|
||||||
|
|---|---|
|
||||||
|
| x264 `ultrafast`（现状） | 单核 15–30% |
|
||||||
|
| x264 `medium`（同画质基准） | 单核 60–100% |
|
||||||
|
| `h264_nvenc p4` | 总 **1–3%** |
|
||||||
|
| `h264_qsv medium` | 总 2–5% |
|
||||||
|
| `h264_amf balanced` | 总 2–5% |
|
||||||
|
|
||||||
|
被控端是用户的主力工作机，他自己还在干活。CPU 让出来意味着远控对他几乎不可感。
|
||||||
|
|
||||||
|
### 3.2 同 CPU 预算下画质更高
|
||||||
|
|
||||||
|
x264 的 preset 排序（同码率下画质）：
|
||||||
|
|
||||||
|
```
|
||||||
|
ultrafast < superfast < veryfast < faster < fast < medium < slow ...
|
||||||
|
↑ 现状 ↑ 标准基准
|
||||||
|
```
|
||||||
|
|
||||||
|
NVENC `p4` 预设大致对应 x264 `fast` ~ `medium`，**画质明显优于当前 ultrafast，且 CPU 占用低一个数量级**。
|
||||||
|
|
||||||
|
### 3.3 其他收益
|
||||||
|
|
||||||
|
- **编码延迟稳定**：ASIC 不受 CPU 调度影响，单帧 1–5 ms
|
||||||
|
- **笔记本电池/温度**：ASIC 几瓦，键盘不烫、风扇不转
|
||||||
|
- **可拉高分辨率/帧率**：4K@30 / 多屏拼接软编扛不住，硬编轻松
|
||||||
|
|
||||||
|
### 3.4 代价（必须接受）
|
||||||
|
|
||||||
|
- **二进制 +6–10 MB**（FFmpeg 静态库，可接受）
|
||||||
|
- **编译复杂度上升**：vcpkg 或自编 FFmpeg
|
||||||
|
- **不同后端参数语义有差异**：rc 模式、preset 名字、bitrate 表现都不一样
|
||||||
|
- **必须保留 x264 软编兜底**：无 GPU / 远程桌面 / 虚拟机 / NVENC session 满 等场景
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 现有 H.264 编码器现状
|
||||||
|
|
||||||
|
`CX264Encoder` 签名（`client/X264Encoder.cpp`）：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
class CX264Encoder
|
||||||
|
{
|
||||||
|
private:
|
||||||
|
x264_t* m_pCodec;
|
||||||
|
x264_picture_t* m_pPicIn;
|
||||||
|
x264_picture_t* m_pPicOut;
|
||||||
|
x264_param_t m_Param;
|
||||||
|
|
||||||
|
public:
|
||||||
|
bool open(int width, int height, int fps, int crf);
|
||||||
|
bool open(x264_param_t* param);
|
||||||
|
void close();
|
||||||
|
int encode(uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
uint8_t** lppData, uint32_t* lpSize,
|
||||||
|
int direction = 1);
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
集成点（`client/ScreenCapture.h`）：
|
||||||
|
|
||||||
|
| 位置 | 内容 |
|
||||||
|
|---|---|
|
||||||
|
| `L148` | `CX264Encoder* m_encoder;`（持有具体类，需改为接口） |
|
||||||
|
| `L926-930` | 关键帧路径硬编码 `new CX264Encoder()` |
|
||||||
|
| `L956-960` | 增量帧路径硬编码 `new CX264Encoder()` |
|
||||||
|
| `L170-176` | `BitRateToCRF(bitRate)` 码率→CRF 映射 |
|
||||||
|
|
||||||
|
参数现状：
|
||||||
|
- `param.i_threads = 1`
|
||||||
|
- `preset = "ultrafast", tune = "zerolatency"`
|
||||||
|
- `i_keyint_max = fps * 15`（15 秒一个 IDR）
|
||||||
|
- `i_bframe = 0`、`b_open_gop = 0`
|
||||||
|
- `rc.i_rc_method = X264_RC_CRF`（CRF 模式，未设 VBV）
|
||||||
|
|
||||||
|
需要的改造（详见 §6）：
|
||||||
|
1. `ScreenCapture::m_encoder` 改 `std::unique_ptr<VideoEncoderBase>`
|
||||||
|
2. 编码器创建走 `CreateEncoder` 工厂
|
||||||
|
3. 接口的 quality 语义解耦（CRF vs kbps，详见 §6.2）
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. FFmpeg 静态库准备
|
||||||
|
|
||||||
|
### 5.1 推荐方案：vcpkg
|
||||||
|
|
||||||
|
VS2019 + vcpkg 是最稳的路径：
|
||||||
|
|
||||||
|
```
|
||||||
|
vcpkg install ffmpeg[core,nvcodec,amf,qsv]:x64-windows-static-md
|
||||||
|
```
|
||||||
|
|
||||||
|
要点：
|
||||||
|
- **三元组选 `x64-windows-static-md`**：链接静态 FFmpeg 但用动态 CRT（`/MD`），与本项目当前工程一致
|
||||||
|
- 如果工程是 `/MT` 改用 `x64-windows-static`
|
||||||
|
- `nvcodec` feature 引入 NVENC 头文件，`amf` 引入 AMF，`qsv` 引入 libmfx
|
||||||
|
- 阶段一只需要 H.264，可以先不带这些 feature，但建议一次到位
|
||||||
|
|
||||||
|
阶段二需要 AV1，FFmpeg 较新版本默认已支持 `av1_nvenc` / `av1_amf` / `av1_qsv`，无需额外 feature 名。
|
||||||
|
|
||||||
|
### 5.2 备选方案：自编
|
||||||
|
|
||||||
|
MSYS2 + MinGW-w64 + `--toolchain=msvc` 产出 MSVC 兼容 `.lib`：
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./configure \
|
||||||
|
--prefix=/path/to/install \
|
||||||
|
--arch=x86_64 \
|
||||||
|
--target-os=mingw64 \
|
||||||
|
--toolchain=msvc \
|
||||||
|
--disable-shared --enable-static \
|
||||||
|
--disable-everything \
|
||||||
|
--disable-autodetect \
|
||||||
|
--disable-network \
|
||||||
|
--disable-doc \
|
||||||
|
--disable-programs \
|
||||||
|
--disable-debug \
|
||||||
|
--enable-small \
|
||||||
|
--enable-encoder=h264_nvenc \
|
||||||
|
--enable-encoder=h264_amf \
|
||||||
|
--enable-encoder=h264_qsv \
|
||||||
|
--enable-encoder=h264_mf \
|
||||||
|
--enable-encoder=av1_nvenc \
|
||||||
|
--enable-encoder=av1_amf \
|
||||||
|
--enable-encoder=av1_qsv \
|
||||||
|
--enable-protocol=file
|
||||||
|
```
|
||||||
|
|
||||||
|
**不要** `--enable-gpl` 或 `--enable-libx265 / --enable-libaom`，保持 LGPL 且避免软编 H.265/AV1。
|
||||||
|
|
||||||
|
### 5.3 工程链接配置（MSVC）
|
||||||
|
|
||||||
|
**附加包含目录**（C/C++ → 常规）：
|
||||||
|
```
|
||||||
|
$(VcpkgRoot)\installed\x64-windows-static-md\include
|
||||||
|
```
|
||||||
|
|
||||||
|
**附加库目录**（链接器 → 常规）：
|
||||||
|
```
|
||||||
|
$(VcpkgRoot)\installed\x64-windows-static-md\lib
|
||||||
|
```
|
||||||
|
|
||||||
|
**附加依赖项**（链接器 → 输入）：
|
||||||
|
```
|
||||||
|
avcodec.lib
|
||||||
|
avutil.lib
|
||||||
|
swresample.lib
|
||||||
|
|
||||||
|
# FFmpeg 静态链接依赖的 Windows 系统库
|
||||||
|
mfplat.lib
|
||||||
|
mfuuid.lib
|
||||||
|
strmiids.lib
|
||||||
|
ws2_32.lib
|
||||||
|
secur32.lib
|
||||||
|
bcrypt.lib
|
||||||
|
```
|
||||||
|
|
||||||
|
**预处理器定义**：
|
||||||
|
```
|
||||||
|
ENABLE_HW_ENCODER
|
||||||
|
__STDC_CONSTANT_MACROS # FFmpeg C 头文件在 C++ 中需要
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 实现任务清单
|
||||||
|
|
||||||
|
### 6.1 文件清单
|
||||||
|
|
||||||
|
| 文件 | 操作 | 说明 |
|
||||||
|
|---|---|---|
|
||||||
|
| `VideoEncoderBase.h` | 新增 | 抽象基类 + EncoderParams |
|
||||||
|
| `X264Encoder.h/.cpp` | 修改 | 继承 `VideoEncoderBase`，新增 `forceIDR()` / `codec()` / `backendName()` |
|
||||||
|
| `CFFmpegH264Encoder.h/.cpp` | 新增（阶段一） | 封装 `h264_nvenc/qsv/amf/mf` |
|
||||||
|
| `CFFmpegAV1Encoder.h/.cpp` | 新增（阶段二） | 封装 `av1_nvenc/qsv/amf` |
|
||||||
|
| `EncoderFactory.h/.cpp` | 新增 | 工厂 + 多后端探测 |
|
||||||
|
| `EncoderProbe.h/.cpp` | 新增 | 启动时一次性探测可用后端，缓存结果 |
|
||||||
|
| `ScreenCapture.h` | 修改 | `m_encoder` 改 `std::unique_ptr<VideoEncoderBase>` |
|
||||||
|
| 握手协议代码 | 修改（阶段二） | 加 `codecs` 能力字段 |
|
||||||
|
| 工程配置 | 修改 | FFmpeg 静态库链接（详见 §5.3） |
|
||||||
|
|
||||||
|
### 6.2 抽象接口定义
|
||||||
|
|
||||||
|
`VideoEncoderBase.h`：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
#pragma once
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
enum class VideoCodec { H264, AV1 };
|
||||||
|
enum class RateControl { CRF, BITRATE };
|
||||||
|
|
||||||
|
struct EncoderParams {
|
||||||
|
int width;
|
||||||
|
int height;
|
||||||
|
int fps;
|
||||||
|
RateControl rc = RateControl::BITRATE;
|
||||||
|
int crf = 23; // 当 rc == CRF 时使用（x264 路径）
|
||||||
|
int bitrate_kbps = 4000; // 当 rc == BITRATE 时使用（硬编路径）
|
||||||
|
int gop_seconds = 4; // 关键帧间隔（秒），编码器内部转 frames
|
||||||
|
};
|
||||||
|
|
||||||
|
class VideoEncoderBase {
|
||||||
|
public:
|
||||||
|
virtual ~VideoEncoderBase() = default;
|
||||||
|
|
||||||
|
virtual bool open(const EncoderParams& params) = 0;
|
||||||
|
virtual void close() = 0;
|
||||||
|
|
||||||
|
virtual int encode(
|
||||||
|
uint8_t* rgb,
|
||||||
|
uint8_t bpp,
|
||||||
|
uint32_t stride,
|
||||||
|
uint32_t width,
|
||||||
|
uint32_t height,
|
||||||
|
uint8_t** lppData,
|
||||||
|
uint32_t* lpSize,
|
||||||
|
int direction = 1
|
||||||
|
) = 0;
|
||||||
|
|
||||||
|
virtual void forceIDR() = 0;
|
||||||
|
virtual void setBitrate(int kbps) {} // 默认空实现
|
||||||
|
virtual VideoCodec codec() const = 0;
|
||||||
|
virtual const char* backendName() const = 0; // "x264" / "h264_nvenc" / "av1_amf" ...
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
设计要点：
|
||||||
|
- **抛弃** `open(w, h, fps, quality)` 的设计 —— `quality` 在 H.264 是 CRF、在硬编是 kbps，语义不清楚是坑
|
||||||
|
- 改用 `EncoderParams` 结构体 + `RateControl` 枚举，明确指定速率控制模式
|
||||||
|
- `backendName()` 返回实际后端名（"x264" / "h264_nvenc" / "av1_amf"），调用方可用于日志/监控
|
||||||
|
|
||||||
|
### 6.3 CFFmpegH264Encoder 设计
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
#pragma once
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include <vector>
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
extern "C" {
|
||||||
|
#include <libavcodec/avcodec.h>
|
||||||
|
#include <libavutil/opt.h>
|
||||||
|
#include <libavutil/imgutils.h>
|
||||||
|
}
|
||||||
|
|
||||||
|
class CFFmpegH264Encoder : public VideoEncoderBase {
|
||||||
|
public:
|
||||||
|
CFFmpegH264Encoder();
|
||||||
|
~CFFmpegH264Encoder() override;
|
||||||
|
|
||||||
|
bool open(const EncoderParams& params) override;
|
||||||
|
void close() override;
|
||||||
|
|
||||||
|
int encode(uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
uint8_t** lppData, uint32_t* lpSize,
|
||||||
|
int direction = 1) override;
|
||||||
|
|
||||||
|
void forceIDR() override;
|
||||||
|
void setBitrate(int kbps) override;
|
||||||
|
VideoCodec codec() const override { return VideoCodec::H264; }
|
||||||
|
const char* backendName() const override { return m_backend.c_str(); }
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool tryOpenBackend(const char* name, const EncoderParams& p);
|
||||||
|
void cleanupCodec();
|
||||||
|
int convertToNV12(uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height, int direction);
|
||||||
|
|
||||||
|
AVCodecContext* m_ctx = nullptr;
|
||||||
|
AVFrame* m_frame = nullptr;
|
||||||
|
AVPacket* m_packet = nullptr;
|
||||||
|
std::vector<uint8_t> m_outputBuffer;
|
||||||
|
std::vector<uint8_t> m_i420Scratch; // RGB24 路径用
|
||||||
|
int64_t m_pts = 0;
|
||||||
|
bool m_forceIDR = false;
|
||||||
|
std::string m_backend;
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 6.3.1 后端探测顺序
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
static const char* kH264Backends[] = {
|
||||||
|
"h264_nvenc", // NVIDIA：质量/速度/稳定性都最好
|
||||||
|
"h264_qsv", // Intel：核显普及度高
|
||||||
|
"h264_amf", // AMD
|
||||||
|
"h264_mf", // Media Foundation 兜底（可选，质量一般）
|
||||||
|
};
|
||||||
|
|
||||||
|
bool CFFmpegH264Encoder::open(const EncoderParams& p) {
|
||||||
|
for (auto name : kH264Backends) {
|
||||||
|
if (tryOpenBackend(name, p)) {
|
||||||
|
m_backend = name;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cleanupCodec();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 6.3.2 各后端参数差异
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
bool CFFmpegH264Encoder::tryOpenBackend(const char* name, const EncoderParams& p) {
|
||||||
|
const AVCodec* codec = avcodec_find_encoder_by_name(name);
|
||||||
|
if (!codec) return false;
|
||||||
|
|
||||||
|
m_ctx = avcodec_alloc_context3(codec);
|
||||||
|
if (!m_ctx) return false;
|
||||||
|
|
||||||
|
// 通用参数
|
||||||
|
m_ctx->width = p.width;
|
||||||
|
m_ctx->height = p.height;
|
||||||
|
m_ctx->time_base = {1, p.fps};
|
||||||
|
m_ctx->framerate = {p.fps, 1};
|
||||||
|
m_ctx->pix_fmt = AV_PIX_FMT_NV12;
|
||||||
|
m_ctx->gop_size = p.fps * p.gop_seconds;
|
||||||
|
m_ctx->max_b_frames = 0;
|
||||||
|
m_ctx->bit_rate = (int64_t)p.bitrate_kbps * 1000;
|
||||||
|
m_ctx->rc_max_rate = (int64_t)p.bitrate_kbps * 1500;
|
||||||
|
m_ctx->rc_buffer_size = (int)(p.bitrate_kbps * 1000);
|
||||||
|
|
||||||
|
if (strcmp(name, "h264_nvenc") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "preset", "p4", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "tune", "ll", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "rc", "cbr", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "zerolatency", "1", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "delay", "0", 0);
|
||||||
|
} else if (strcmp(name, "h264_qsv") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "preset", "medium", 0);
|
||||||
|
av_opt_set_int(m_ctx->priv_data, "async_depth", 1, 0);
|
||||||
|
av_opt_set_int(m_ctx->priv_data, "low_power", 0, 0);
|
||||||
|
} else if (strcmp(name, "h264_amf") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "usage", "lowlatency", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "quality", "balanced", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "rc", "cbr", 0);
|
||||||
|
} else if (strcmp(name, "h264_mf") == 0) {
|
||||||
|
av_opt_set_int(m_ctx->priv_data, "hw_encoding", 1, 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "rate_control", "cbr", 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (avcodec_open2(m_ctx, codec, nullptr) < 0) return false;
|
||||||
|
|
||||||
|
m_frame = av_frame_alloc();
|
||||||
|
m_frame->format = AV_PIX_FMT_NV12;
|
||||||
|
m_frame->width = p.width;
|
||||||
|
m_frame->height = p.height;
|
||||||
|
if (av_frame_get_buffer(m_frame, 32) < 0) return false;
|
||||||
|
|
||||||
|
m_packet = av_packet_alloc();
|
||||||
|
return m_packet != nullptr;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 6.3.3 encode 实现
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
int CFFmpegH264Encoder::encode(
|
||||||
|
uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||||
|
uint32_t width, uint32_t height,
|
||||||
|
uint8_t** lppData, uint32_t* lpSize, int direction)
|
||||||
|
{
|
||||||
|
if (av_frame_make_writable(m_frame) < 0) return -1;
|
||||||
|
|
||||||
|
// 像素格式转换（直接用 libyuv，与现有 x264 路径保持一致，不引入 sws_scale）
|
||||||
|
int signed_height = direction * (int)height;
|
||||||
|
if (bpp == 32) {
|
||||||
|
libyuv::ARGBToNV12(
|
||||||
|
rgb, stride,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
width, signed_height
|
||||||
|
);
|
||||||
|
} else if (bpp == 24) {
|
||||||
|
if (convertToNV12(rgb, bpp, stride, width, height, direction) != 0)
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
|
return -2;
|
||||||
|
}
|
||||||
|
|
||||||
|
m_frame->pts = m_pts++;
|
||||||
|
|
||||||
|
if (m_forceIDR) {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_I;
|
||||||
|
m_frame->key_frame = 1;
|
||||||
|
m_forceIDR = false;
|
||||||
|
} else {
|
||||||
|
m_frame->pict_type = AV_PICTURE_TYPE_NONE;
|
||||||
|
m_frame->key_frame = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (avcodec_send_frame(m_ctx, m_frame) < 0) return -3;
|
||||||
|
|
||||||
|
int ret = avcodec_receive_packet(m_ctx, m_packet);
|
||||||
|
if (ret == AVERROR(EAGAIN)) {
|
||||||
|
// 首帧延迟正常情况：返回成功但本次无输出
|
||||||
|
*lpSize = 0;
|
||||||
|
*lppData = nullptr;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (ret < 0) return -4;
|
||||||
|
|
||||||
|
m_outputBuffer.assign(m_packet->data, m_packet->data + m_packet->size);
|
||||||
|
*lppData = m_outputBuffer.data();
|
||||||
|
*lpSize = (uint32_t)m_outputBuffer.size();
|
||||||
|
av_packet_unref(m_packet);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 6.3.4 RGB24 → NV12（libyuv 无直接 API，两步走）
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
int CFFmpegH264Encoder::convertToNV12(uint8_t* rgb, uint8_t /*bpp*/,
|
||||||
|
uint32_t stride, uint32_t width, uint32_t height,
|
||||||
|
int direction)
|
||||||
|
{
|
||||||
|
int signed_height = direction * (int)height;
|
||||||
|
int y_size = width * height;
|
||||||
|
int uv_size = (width / 2) * (height / 2);
|
||||||
|
m_i420Scratch.resize(y_size + 2 * uv_size);
|
||||||
|
|
||||||
|
uint8_t* y = m_i420Scratch.data();
|
||||||
|
uint8_t* u = y + y_size;
|
||||||
|
uint8_t* v = u + uv_size;
|
||||||
|
|
||||||
|
libyuv::RGB24ToI420(
|
||||||
|
rgb, stride,
|
||||||
|
y, width,
|
||||||
|
u, width / 2,
|
||||||
|
v, width / 2,
|
||||||
|
width, signed_height
|
||||||
|
);
|
||||||
|
libyuv::I420ToNV12(
|
||||||
|
y, width,
|
||||||
|
u, width / 2,
|
||||||
|
v, width / 2,
|
||||||
|
m_frame->data[0], m_frame->linesize[0],
|
||||||
|
m_frame->data[1], m_frame->linesize[1],
|
||||||
|
width, height
|
||||||
|
);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6.4 CFFmpegAV1Encoder 设计
|
||||||
|
|
||||||
|
结构与 `CFFmpegH264Encoder` **完全对称**，仅 backend 名换：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
static const char* kAV1Backends[] = {
|
||||||
|
"av1_nvenc", // RTX 40+
|
||||||
|
"av1_amf", // RX 7000+
|
||||||
|
"av1_qsv", // Intel Arc / 部分 11 代+ 核显
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
参数差异（av1_nvenc 与 h264_nvenc 略有不同）：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
if (strcmp(name, "av1_nvenc") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "preset", "p4", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "tune", "ll", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "rc", "cbr", 0);
|
||||||
|
// AV1 特有：tile-columns/rows 可调，多核解码更友好
|
||||||
|
av_opt_set_int(m_ctx->priv_data, "tile-columns", 1, 0);
|
||||||
|
} else if (strcmp(name, "av1_amf") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "usage", "lowlatency", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "quality", "balanced", 0);
|
||||||
|
av_opt_set(m_ctx->priv_data, "rc", "cbr", 0);
|
||||||
|
} else if (strcmp(name, "av1_qsv") == 0) {
|
||||||
|
av_opt_set(m_ctx->priv_data, "preset", "medium", 0);
|
||||||
|
av_opt_set_int(m_ctx->priv_data, "async_depth", 1, 0);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
实现建议：先把 `CFFmpegH264Encoder` 跑通稳定，**再把它复制改名做 AV1 版本**。同步两个类的逻辑可以后续考虑抽公共基类，但不要为了 DRY 提前抽。
|
||||||
|
|
||||||
|
### 6.5 EncoderFactory 与探测
|
||||||
|
|
||||||
|
`EncoderFactory.h`：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
#pragma once
|
||||||
|
#include "VideoEncoderBase.h"
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
struct ClientCapability {
|
||||||
|
bool supportAV1 = false;
|
||||||
|
bool supportH264 = true; // 假定都支持
|
||||||
|
};
|
||||||
|
|
||||||
|
struct EncoderRequest {
|
||||||
|
int width, height, fps;
|
||||||
|
int bitrate_kbps;
|
||||||
|
ClientCapability client;
|
||||||
|
};
|
||||||
|
|
||||||
|
std::unique_ptr<VideoEncoderBase> CreateEncoder(const EncoderRequest& req);
|
||||||
|
```
|
||||||
|
|
||||||
|
`EncoderFactory.cpp`：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
#include "EncoderFactory.h"
|
||||||
|
#include "EncoderProbe.h"
|
||||||
|
#include "CFFmpegAV1Encoder.h"
|
||||||
|
#include "CFFmpegH264Encoder.h"
|
||||||
|
#include "X264Encoder.h"
|
||||||
|
|
||||||
|
std::unique_ptr<VideoEncoderBase> CreateEncoder(const EncoderRequest& req) {
|
||||||
|
EncoderParams p;
|
||||||
|
p.width = req.width;
|
||||||
|
p.height = req.height;
|
||||||
|
p.fps = req.fps;
|
||||||
|
|
||||||
|
// 1. AV1 路径（仅当客户端支持且启动探测确认硬件可用）
|
||||||
|
if (req.client.supportAV1 && EncoderProbe::HasAV1Hw()) {
|
||||||
|
auto enc = std::make_unique<CFFmpegAV1Encoder>();
|
||||||
|
p.rc = RateControl::BITRATE;
|
||||||
|
p.bitrate_kbps = req.bitrate_kbps;
|
||||||
|
if (enc->open(p)) {
|
||||||
|
LOG_INFO("encoder: AV1 backend=%s", enc->backendName());
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
LOG_WARN("encoder: AV1 open failed, falling back to H.264");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. H.264 硬编路径
|
||||||
|
if (EncoderProbe::HasH264Hw()) {
|
||||||
|
auto enc = std::make_unique<CFFmpegH264Encoder>();
|
||||||
|
p.rc = RateControl::BITRATE;
|
||||||
|
p.bitrate_kbps = req.bitrate_kbps;
|
||||||
|
if (enc->open(p)) {
|
||||||
|
LOG_INFO("encoder: H264-HW backend=%s", enc->backendName());
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
LOG_WARN("encoder: H264 HW open failed, falling back to x264");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. H.264 软编兜底（始终可用）
|
||||||
|
{
|
||||||
|
auto enc = std::make_unique<CX264Encoder>();
|
||||||
|
p.rc = RateControl::CRF;
|
||||||
|
p.crf = BitRateToCRF(req.bitrate_kbps);
|
||||||
|
if (enc->open(p)) {
|
||||||
|
LOG_INFO("encoder: H264-SW (libx264)");
|
||||||
|
return enc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG_ERROR("encoder: all backends failed");
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
要点：
|
||||||
|
- **不要**引入会话池（`HEVCSessionPool` 那套）—— 个人项目场景下不需要
|
||||||
|
- NVENC session 限制由 FFmpeg 自己报错，工厂捕获后自动降级
|
||||||
|
- 失败路径都打日志，方便定位
|
||||||
|
|
||||||
|
### 6.6 启动时一次性后端探测
|
||||||
|
|
||||||
|
避免每个新连接都重复尝试每个 backend：
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
// EncoderProbe.h
|
||||||
|
class EncoderProbe {
|
||||||
|
public:
|
||||||
|
static void RunOnce(); // 程序启动时调用一次
|
||||||
|
static bool HasAV1Hw();
|
||||||
|
static bool HasH264Hw();
|
||||||
|
static const char* PreferredAV1Backend(); // 第一个能用的 AV1 后端名
|
||||||
|
static const char* PreferredH264Backend();
|
||||||
|
};
|
||||||
|
|
||||||
|
// EncoderProbe.cpp 实现思路：
|
||||||
|
// 对每个候选后端尝试 alloc_context → open2 → free
|
||||||
|
// 用最低分辨率（如 640x480 @30fps）减少探测开销
|
||||||
|
// 结果缓存在静态变量，加 std::once_flag 保证线程安全
|
||||||
|
```
|
||||||
|
|
||||||
|
启动时探测 1 次，运行时 `CreateEncoder` 直接读结果。
|
||||||
|
|
||||||
|
### 6.7 ScreenCapture.h 改造
|
||||||
|
|
||||||
|
**当前**（`client/ScreenCapture.h:148, 926-930, 956-960`）：
|
||||||
|
```cpp
|
||||||
|
CX264Encoder* m_encoder;
|
||||||
|
// ...
|
||||||
|
m_encoder = new CX264Encoder();
|
||||||
|
int br = (m_nBitRate > 0) ? m_nBitRate : (width * height / 1266);
|
||||||
|
m_encoder->open(width, height, 20, BitRateToCRF(br));
|
||||||
|
```
|
||||||
|
|
||||||
|
**改造后**：
|
||||||
|
```cpp
|
||||||
|
std::unique_ptr<VideoEncoderBase> m_encoder;
|
||||||
|
ClientCapability m_clientCap; // 握手阶段填入
|
||||||
|
// ...
|
||||||
|
if (!m_encoder) {
|
||||||
|
EncoderRequest req{
|
||||||
|
(int)width, (int)height, 20,
|
||||||
|
m_nBitRate > 0 ? m_nBitRate : (int)(width * height / 1266),
|
||||||
|
m_clientCap
|
||||||
|
};
|
||||||
|
m_encoder = CreateEncoder(req);
|
||||||
|
if (!m_encoder) return nullptr;
|
||||||
|
}
|
||||||
|
int err = m_encoder->encode(nextData, 32, 4 * width, width, height,
|
||||||
|
&encoded_data, &encoded_size);
|
||||||
|
```
|
||||||
|
|
||||||
|
注意：两处 `new CX264Encoder()` 提取成一个 `ensureEncoder()` 私有方法，避免重复。
|
||||||
|
|
||||||
|
### 6.8 协议握手（阶段二）
|
||||||
|
|
||||||
|
客户端浏览器 WebSocket 连接时上报：
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"type": "client_capability",
|
||||||
|
"codecs": ["av1", "h264"]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
浏览器端探测脚本（JS）：
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
async function probeBrowserCodecs() {
|
||||||
|
const codecs = [];
|
||||||
|
if (typeof VideoDecoder !== 'undefined') {
|
||||||
|
// AV1 Main Profile, Level 4.0, 8-bit
|
||||||
|
const av1 = await VideoDecoder.isConfigSupported({ codec: 'av01.0.04M.08' });
|
||||||
|
if (av1.supported) codecs.push('av1');
|
||||||
|
}
|
||||||
|
codecs.push('h264'); // 兜底假定支持，<video> 标签也支持
|
||||||
|
return codecs;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**向后兼容**：老版本客户端不发 `codecs` 字段 → 服务端按 H.264 处理。`ClientCapability::supportAV1` 默认 false。
|
||||||
|
|
||||||
|
可参考 `docs/hevc_browser_decode_test.html`（改造一份 AV1 版本）做浏览器解码端到端验证。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 像素格式与转换
|
||||||
|
|
||||||
|
保持与现有 x264 路径完全一致的做法：
|
||||||
|
|
||||||
|
- **硬编内部格式**：`AV_PIX_FMT_NV12`（NVENC/QSV/AMF 通用）
|
||||||
|
- **转换库**：libyuv（不引入 `sws_scale`）
|
||||||
|
- **BGRA → NV12**：`libyuv::ARGBToNV12` 直接
|
||||||
|
- **RGB24 → NV12**：libyuv 无直接 API，分两步 RGB24 → I420 → NV12
|
||||||
|
- **direction 参数**：沿用现有 `X264Encoder.cpp:136` 的写法（`direction * height` 作为乘子）
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 测试要求
|
||||||
|
|
||||||
|
### 8.1 单元测试
|
||||||
|
|
||||||
|
- `VideoEncoderBase` 各实现的 `open / encode / close` 生命周期
|
||||||
|
- `CFFmpegH264Encoder` 在缺失各后端时降级到下一个
|
||||||
|
- `EncoderFactory` 在不同 `ClientCapability` × 硬件能力 矩阵下返回正确后端
|
||||||
|
- `EncoderProbe::RunOnce()` 在不同 GPU 上的结果一致性
|
||||||
|
|
||||||
|
### 8.2 集成测试
|
||||||
|
|
||||||
|
| 场景 | 期望 |
|
||||||
|
|---|---|
|
||||||
|
| 客户端支持 AV1 + 编码端 RTX 40 | 使用 AV1（`av1_nvenc`） |
|
||||||
|
| 客户端支持 AV1 + 编码端 GTX 1080 | 降级 H.264 硬编（`h264_nvenc`） |
|
||||||
|
| 客户端不支持 AV1 + 编码端任意 | H.264（硬编优先） |
|
||||||
|
| 编码端无 GPU / 虚拟机 | x264 软编 |
|
||||||
|
| 编码端集显 + 独显 | 优先级中第一个成功的后端 |
|
||||||
|
| 中途客户端能力变化 | 当前连接不变；下次握手按新能力 |
|
||||||
|
| H264 硬编创建失败 | 自动回落 x264 软编，连接不断 |
|
||||||
|
|
||||||
|
### 8.3 硬件验证矩阵
|
||||||
|
|
||||||
|
| 编码端 | 客户端浏览器 | 期望 |
|
||||||
|
|---|---|---|
|
||||||
|
| RTX 40 / 50 | Chrome / Firefox / Edge | AV1 |
|
||||||
|
| GTX 10/16 / RTX 20/30 | Chrome / Firefox / Edge | H.264 NVENC |
|
||||||
|
| Intel Arc | Chrome / Firefox | AV1（`av1_qsv`） |
|
||||||
|
| Intel 12 代+ 核显 | Chrome / Firefox | H.264 QSV |
|
||||||
|
| AMD RX 7000+ | Chrome / Firefox | AV1（`av1_amf`） |
|
||||||
|
| AMD 老卡 | Chrome / Firefox | H.264 AMF |
|
||||||
|
| 虚拟机 / 远程桌面会话 | 任意 | x264 软编 |
|
||||||
|
| iOS Safari < 17 | 任意编码端 | H.264 |
|
||||||
|
| iOS Safari ≥ 17（A17 Pro+） | 任意编码端 | AV1 优先 |
|
||||||
|
|
||||||
|
### 8.4 体积验证
|
||||||
|
|
||||||
|
- exe 体积增量 < 12 MB（vcpkg 静态链接，含 AV1+H264 全后端）
|
||||||
|
- 若超出明显，检查 vcpkg feature 是否引入了不需要的 codec
|
||||||
|
|
||||||
|
### 8.5 回归测试（关键）
|
||||||
|
|
||||||
|
每一步改造后必须验证：
|
||||||
|
- 现有 x264 软编通路完全可用（在禁用所有硬编后端的环境下）
|
||||||
|
- 现有客户端（不发 `codecs` 字段）可正常工作
|
||||||
|
- 编码码流向后兼容，老客户端能解
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 已知风险与注意事项
|
||||||
|
|
||||||
|
### 9.1 多 GPU 跨适配器
|
||||||
|
笔记本集显+独显场景，FFmpeg 默认走主显卡。可能报 "failed to create device"。**Catch 后回落到下一个后端**，不要直接终止。
|
||||||
|
|
||||||
|
### 9.2 第一帧延迟
|
||||||
|
FFmpeg 硬编可能在首次 `send_frame` 后 `receive_packet` 返回 `EAGAIN`。调用方代码必须能处理 `*lpSize == 0` 的情况（返回 0 表示成功但本次无输出）。`ScreenCapture::GetNextScreenData` 当前 `encoded_size == 0` 会怎么处理需要确认。
|
||||||
|
|
||||||
|
### 9.3 NVENC session 数限制
|
||||||
|
NVIDIA **消费级**卡（GeForce）有 NVENC session 上限（驱动 522.25+ 起 3 个，更新版可能放宽到 5）。多连接超限时 `open` 失败 → 工厂自动降级到 QSV/AMF/x264。**不要做"会话池"提前拦截** —— 让 FFmpeg 自己报错，工厂处理。
|
||||||
|
|
||||||
|
### 9.4 浏览器解 AV1 的硬件依赖
|
||||||
|
- 桌面 Chrome/Firefox/Edge：软解兜底（CPU 占用高，但能用）
|
||||||
|
- 移动端：iPhone 15 Pro+ / M3+ 才有 AV1 硬解，老设备只能软解（可能卡）
|
||||||
|
- `isConfigSupported` 报 true 不等于跑得流畅。**建议握手时也带上设备类型**，弱设备强制走 H.264
|
||||||
|
|
||||||
|
### 9.5 LGPL 静态链接合规
|
||||||
|
个人项目暂搁置。商用前需法务确认（FFmpeg 源代码提供、重新链接能力等）。
|
||||||
|
|
||||||
|
### 9.6 配置项（建议）
|
||||||
|
|
||||||
|
建议做成 INI/JSON：
|
||||||
|
```
|
||||||
|
encoder.prefer_av1 = true
|
||||||
|
encoder.h264.bitrate_default = 4000 (kbps)
|
||||||
|
encoder.fallback_to_x264 = true
|
||||||
|
encoder.probe_at_startup = true
|
||||||
|
encoder.disable_h264_mf = true (h264_mf 质量一般，可禁用)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 9.7 日志要求
|
||||||
|
|
||||||
|
关键路径必须有日志：
|
||||||
|
- 启动探测结果：每个后端是否可用 + 失败原因（`av_err2str`）
|
||||||
|
- 每个连接选定的 `codec` + `backend`（INFO）
|
||||||
|
- 后端打开失败 + 回落（WARN）
|
||||||
|
- 编码过程中的异常（ERROR）
|
||||||
|
|
||||||
|
### 9.8 线程安全
|
||||||
|
- 每个编码器实例不跨线程
|
||||||
|
- `EncoderProbe` 单例首次初始化加 `std::once_flag`
|
||||||
|
- FFmpeg 新版本不需要全局初始化（`av_register_all` 已废弃）
|
||||||
|
|
||||||
|
### 9.9 与现有 `DISABLE_X264_FOR_TEST` 编译开关协同
|
||||||
|
项目已有 `DISABLE_X264_FOR_TEST` 宏（见 `X264Encoder.cpp:5`）和最近 `c0a632a` 提交"Compliance: Add building option to disable x264 and ffmpeg"。新代码须遵循同样的可禁用约定：
|
||||||
|
- `ENABLE_HW_ENCODER` 关闭时整个 `CFFmpegH264Encoder` / `CFFmpegAV1Encoder` 编译为空实现或不参与链接
|
||||||
|
- 工厂在该宏关闭时直接返回 `CX264Encoder`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. 实现顺序建议
|
||||||
|
|
||||||
|
**每一步独立可合入**，每一步完成后 x264 通路必须可用、客户端无感知。
|
||||||
|
|
||||||
|
### Step 0：抽象层（零功能改动）
|
||||||
|
1. 新建 `VideoEncoderBase.h`，定义接口 + `EncoderParams` + `RateControl`
|
||||||
|
2. `CX264Encoder` 改造继承 `VideoEncoderBase`：
|
||||||
|
- 新增 `forceIDR()`（设置一个标志，下次 encode 时通过 `x264_picture_t::i_type = X264_TYPE_IDR`）
|
||||||
|
- 实现 `codec()` 返回 `H264`
|
||||||
|
- 实现 `backendName()` 返回 `"x264"`
|
||||||
|
- 旧 `open(w, h, fps, crf)` 签名保留，转调新的 `open(EncoderParams)`
|
||||||
|
3. `ScreenCapture::m_encoder` 改 `std::unique_ptr<VideoEncoderBase>`，但仍直接 `new CX264Encoder`
|
||||||
|
4. **不引 FFmpeg、不引工厂**
|
||||||
|
5. 验证：H.264 通路完全不变，对外行为零变化
|
||||||
|
|
||||||
|
### Step 1：FFmpeg 集成 + `h264_nvenc` 单后端
|
||||||
|
1. vcpkg 安装 `ffmpeg[core,nvcodec]:x64-windows-static-md`
|
||||||
|
2. 工程添加包含目录 / 库目录 / 系统库
|
||||||
|
3. 新建 `CFFmpegH264Encoder`，仅实现 `h264_nvenc`
|
||||||
|
4. 在 `ScreenCapture` 加临时开关：硬编码切到 `CFFmpegH264Encoder` 跑一下
|
||||||
|
5. 用浏览器解码 demo 验证码流能解
|
||||||
|
6. 体积验证（应 +4–6 MB）
|
||||||
|
|
||||||
|
### Step 2：扩展 H.264 硬编后端
|
||||||
|
1. `CFFmpegH264Encoder` 加 `h264_qsv` / `h264_amf` 探测
|
||||||
|
2. 顺序：`nvenc → qsv → amf`（`mf` 可暂不接）
|
||||||
|
3. 不同后端的参数适配（见 §6.3.2）
|
||||||
|
4. 测试 Intel 核显 + AMD 卡
|
||||||
|
|
||||||
|
### Step 3：工厂 + 软编兜底
|
||||||
|
1. 新建 `EncoderFactory` / `EncoderProbe`
|
||||||
|
2. 工厂按 `H264 硬编 → x264 软编` 顺序
|
||||||
|
3. `ScreenCapture` 改用工厂（消除两处 `new CX264Encoder`）
|
||||||
|
4. 测试无 GPU 环境降级到 x264
|
||||||
|
|
||||||
|
### Step 4：AV1 路径（独立闭环）
|
||||||
|
1. 重跑 vcpkg：`ffmpeg[core,nvcodec,amf,qsv]:x64-windows-static-md`
|
||||||
|
2. 新建 `CFFmpegAV1Encoder`，结构与 H.264 对称（直接 copy + 改 backend 名 + 调参）
|
||||||
|
3. `EncoderProbe` 加 AV1 探测
|
||||||
|
4. 工厂前置 AV1 路径
|
||||||
|
5. 硬件验证矩阵执行
|
||||||
|
|
||||||
|
### Step 5：握手协商 + 浏览器探测
|
||||||
|
1. 客户端 JS `isConfigSupported` 探测 AV1 / H.264
|
||||||
|
2. WebSocket 握手字段 `codecs` 上报
|
||||||
|
3. 服务端解析后填入 `ClientCapability`
|
||||||
|
4. 老客户端向后兼容（无 `codecs` 字段 → 默认 H.264）
|
||||||
|
5. 端到端验证：编码端 RTX 40 + 浏览器 Chrome 走 AV1，回落场景走 H.264
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. 不在本次范围
|
||||||
|
|
||||||
|
- HEVC 编码（决策已排除，见 §1.3.1）
|
||||||
|
- 软编 AV1（libaom / SVT-AV1，CPU 占用不适合远控）
|
||||||
|
- 运行中动态切换 codec（需要切换就重连）
|
||||||
|
- 转发流（1 路编码多路分发）
|
||||||
|
- 桌面捕获共享
|
||||||
|
- 客户端浏览器解码具体实现（可参考 `docs/hevc_browser_decode_test.html` 改 AV1 版本验证）
|
||||||
|
- Linux/macOS 移植
|
||||||
|
- FFmpeg DLL 形式分发
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. 参考资料
|
||||||
|
|
||||||
|
- FFmpeg 编码器列表：`ffmpeg -encoders | grep -E "h264|av1"`
|
||||||
|
- NVENC 参数：`ffmpeg -h encoder=h264_nvenc`、`ffmpeg -h encoder=av1_nvenc`
|
||||||
|
- QSV 参数：`ffmpeg -h encoder=h264_qsv`、`ffmpeg -h encoder=av1_qsv`
|
||||||
|
- AMF 参数：`ffmpeg -h encoder=h264_amf`、`ffmpeg -h encoder=av1_amf`
|
||||||
|
- NVIDIA Video Codec Support Matrix：https://developer.nvidia.com/video-encode-and-decode-gpu-support-matrix-new
|
||||||
|
- WebCodecs API：https://developer.mozilla.org/en-US/docs/Web/API/WebCodecs_API
|
||||||
|
- libyuv：https://chromium.googlesource.com/libyuv/libyuv/
|
||||||
|
- vcpkg ffmpeg port：https://github.com/microsoft/vcpkg/tree/master/ports/ffmpeg
|
||||||
|
- FFmpeg HWAccel Intro：https://trac.ffmpeg.org/wiki/HWAccelIntro
|
||||||
|
- AOMedia AV1：https://aomedia.org/
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**文档结束**
|
||||||
|
|
||||||
|
实现时如遇到本文档未覆盖的设计抉择，优先选择**简单、与现有 x264 通路对称、不破坏已有功能、不增加运行时外部依赖**的方案，并在代码注释中说明决策依据。
|
||||||
@@ -43,6 +43,66 @@ IMPLEMENT_DYNAMIC(CScreenSpyDlg, CDialog)
|
|||||||
|
|
||||||
#define TIMER_ID 132
|
#define TIMER_ID 132
|
||||||
|
|
||||||
|
// H.264 Annex B keyframe 探测：扫描 start code (00 00 01 / 00 00 00 01)，
|
||||||
|
// 取后续 NAL header low 5 bits，命中 5 (IDR) / 7 (SPS) / 8 (PPS) 即认定为关键帧。
|
||||||
|
static bool IsH264Keyframe(const uint8_t* data, size_t len)
|
||||||
|
{
|
||||||
|
for (size_t i = 0; i + 4 < len; ++i) {
|
||||||
|
size_t nalOffset = 0;
|
||||||
|
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 0 && data[i+3] == 1) {
|
||||||
|
nalOffset = i + 4;
|
||||||
|
} else if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
|
||||||
|
nalOffset = i + 3;
|
||||||
|
} else {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (nalOffset >= len) continue;
|
||||||
|
uint8_t nalType = data[nalOffset] & 0x1F;
|
||||||
|
if (nalType == 5 || nalType == 7 || nalType == 8) return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// AV1 OBU keyframe 探测：扫描 OBU 链，遇到 OBU_SEQUENCE_HEADER (type 1) 即认定为关键帧。
|
||||||
|
// FFmpeg AV1 编码器在每个 IDR 前必定插入 SEQ HDR，因此该判定与 H.264 NAL 5/7/8 语义对齐。
|
||||||
|
static bool IsAv1Keyframe(const uint8_t* data, size_t len)
|
||||||
|
{
|
||||||
|
size_t pos = 0;
|
||||||
|
while (pos < len) {
|
||||||
|
uint8_t hdr = data[pos];
|
||||||
|
uint8_t obu_type = (hdr >> 3) & 0x0F;
|
||||||
|
bool has_ext = (hdr & 0x04) != 0;
|
||||||
|
bool has_size = (hdr & 0x02) != 0;
|
||||||
|
if (obu_type == 1 /*OBU_SEQUENCE_HEADER*/) return true;
|
||||||
|
pos++;
|
||||||
|
if (has_ext) {
|
||||||
|
if (pos >= len) return false;
|
||||||
|
pos++;
|
||||||
|
}
|
||||||
|
if (!has_size) return false; // 无 size 字段：OBU 占满到包尾，无法继续解析
|
||||||
|
// LEB128 size
|
||||||
|
uint64_t sz = 0;
|
||||||
|
for (int i = 0; i < 8; ++i) {
|
||||||
|
if (pos >= len) return false;
|
||||||
|
uint8_t b = data[pos++];
|
||||||
|
sz |= (uint64_t)(b & 0x7F) << (7 * i);
|
||||||
|
if ((b & 0x80) == 0) break;
|
||||||
|
}
|
||||||
|
if (pos + sz > len) return false;
|
||||||
|
pos += (size_t)sz;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 首字节嗅探：H.264 Annex B 首字节恒为 0x00（起始码）；AV1 OBU header 首字节
|
||||||
|
// bit7=0、bits[3:6]=obu_type 1-15，典型值 0x08-0x78，绝不为 0x00。
|
||||||
|
// 一字节即可干净区分两套码流，无需协议字段或编码端协商。
|
||||||
|
static bool IsAnyKeyframe(const uint8_t* data, size_t len)
|
||||||
|
{
|
||||||
|
if (len == 0) return false;
|
||||||
|
return data[0] == 0x00 ? IsH264Keyframe(data, len) : IsAv1Keyframe(data, len);
|
||||||
|
}
|
||||||
|
|
||||||
// 静态成员变量定义
|
// 静态成员变量定义
|
||||||
int CScreenSpyDlg::s_nFastStretch = -1; // -1 表示未初始化
|
int CScreenSpyDlg::s_nFastStretch = -1; // -1 表示未初始化
|
||||||
|
|
||||||
@@ -675,6 +735,12 @@ BOOL CScreenSpyDlg::OnInitDialog()
|
|||||||
// 音频菜单项
|
// 音频菜单项
|
||||||
SysMenu->AppendMenuL(MF_STRING, IDM_AUDIO_TOGGLE, "系统音频(&U)");
|
SysMenu->AppendMenuL(MF_STRING, IDM_AUDIO_TOGGLE, "系统音频(&U)");
|
||||||
SysMenu->CheckMenuItem(IDM_AUDIO_TOGGLE, m_Settings.AudioEnabled ? MF_CHECKED : MF_UNCHECKED);
|
SysMenu->CheckMenuItem(IDM_AUDIO_TOGGLE, m_Settings.AudioEnabled ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->AppendMenuL(MF_STRING, IDM_ENABLE_H264_HARD, "启用 H264 硬编码");
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
SysMenu->AppendMenuL(MF_STRING, IDM_ENABLE_AV1_HARD, "启用 AV1 硬编码");
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
|
||||||
// 初始化勾选状态
|
// 初始化勾选状态
|
||||||
UpdateQualityMenuCheck(SysMenu);
|
UpdateQualityMenuCheck(SysMenu);
|
||||||
@@ -1410,27 +1476,11 @@ VOID CScreenSpyDlg::DrawNextScreenDiff(bool keyFrame)
|
|||||||
bChange = TRUE;
|
bChange = TRUE;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Broadcast H264 frame to web clients (only for Web session dialogs)
|
// Broadcast video frame to web clients (only for Web session dialogs)
|
||||||
// Format: [DeviceID:4][FrameType:1][DataLen:4][H264Data:N]
|
// Format: [DeviceID:4][FrameType:1][DataLen:4][VideoData:N]
|
||||||
|
// 浏览器侧按首字节嗅探区分 H.264 / AV1，因此 packet 内不需要 codec 字段。
|
||||||
if (m_bIsWebSession && NextScreenLength > 0 && WebService().IsRunning()) {
|
if (m_bIsWebSession && NextScreenLength > 0 && WebService().IsRunning()) {
|
||||||
// Detect H264 keyframe by checking NAL unit type
|
bool isKeyFrame = IsAnyKeyframe((const uint8_t*)NextScreenData, NextScreenLength);
|
||||||
// NAL type 5 = IDR slice (keyframe), NAL type 7 = SPS, NAL type 8 = PPS
|
|
||||||
bool isKeyFrame = false;
|
|
||||||
LPBYTE h264Data = (LPBYTE)NextScreenData;
|
|
||||||
for (ULONG i = 0; i + 4 < NextScreenLength; i++) {
|
|
||||||
// Look for start code: 0x00 0x00 0x00 0x01 or 0x00 0x00 0x01
|
|
||||||
if ((h264Data[i] == 0 && h264Data[i+1] == 0 && h264Data[i+2] == 0 && h264Data[i+3] == 1) ||
|
|
||||||
(h264Data[i] == 0 && h264Data[i+1] == 0 && h264Data[i+2] == 1)) {
|
|
||||||
int nalOffset = (h264Data[i+2] == 1) ? i + 3 : i + 4;
|
|
||||||
if (nalOffset < (int)NextScreenLength) {
|
|
||||||
int nalType = h264Data[nalOffset] & 0x1F;
|
|
||||||
if (nalType == 5 || nalType == 7 || nalType == 8) {
|
|
||||||
isKeyFrame = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<uint8_t> packet(4 + 1 + 4 + NextScreenLength);
|
std::vector<uint8_t> packet(4 + 1 + 4 + NextScreenLength);
|
||||||
uint32_t deviceIdLow = (uint32_t)(m_ClientID & 0xFFFFFFFF);
|
uint32_t deviceIdLow = (uint32_t)(m_ClientID & 0xFFFFFFFF);
|
||||||
@@ -2134,6 +2184,26 @@ void CScreenSpyDlg::OnSysCommand(UINT nID, LPARAM lParam)
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
case IDM_ENABLE_H264_HARD: {
|
||||||
|
m_Settings.EncodeLevel = m_Settings.EncodeLevel ? LEVEL_H264_SOFT : LEVEL_H264_HARD;
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
BYTE bToken[] = {COMMAND_ENCODE_LEVEL, m_Settings.EncodeLevel };
|
||||||
|
m_ContextObject->Send2Client(bToken, sizeof(bToken));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case IDM_ENABLE_AV1_HARD: {
|
||||||
|
m_Settings.EncodeLevel = m_Settings.EncodeLevel ? LEVEL_H264_SOFT : LEVEL_AV1_HARD;
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->CheckMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_CHECKED : MF_UNCHECKED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_H264_HARD, m_Settings.EncodeLevel == LEVEL_AV1_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
SysMenu->EnableMenuItem(IDM_ENABLE_AV1_HARD, m_Settings.EncodeLevel == LEVEL_H264_HARD ? MF_GRAYED : MF_ENABLED);
|
||||||
|
BYTE bToken[] = { COMMAND_ENCODE_LEVEL, m_Settings.EncodeLevel };
|
||||||
|
m_ContextObject->Send2Client(bToken, sizeof(bToken));
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
__super::OnSysCommand(nID, lParam);
|
__super::OnSysCommand(nID, lParam);
|
||||||
|
|||||||
@@ -135,6 +135,8 @@ enum {
|
|||||||
IDM_RESTORE_CONSOLE, // RDP会话归位
|
IDM_RESTORE_CONSOLE, // RDP会话归位
|
||||||
IDM_RESET_VIRTUAL_DESKTOP, // 重置虚拟桌面
|
IDM_RESET_VIRTUAL_DESKTOP, // 重置虚拟桌面
|
||||||
IDM_AUDIO_TOGGLE, // 音频开关
|
IDM_AUDIO_TOGGLE, // 音频开关
|
||||||
|
IDM_ENABLE_H264_HARD,
|
||||||
|
IDM_ENABLE_AV1_HARD,
|
||||||
};
|
};
|
||||||
|
|
||||||
// 状态信息窗口 - 全屏时显示帧率/速度/质量
|
// 状态信息窗口 - 全屏时显示帧率/速度/质量
|
||||||
|
|||||||
@@ -1892,3 +1892,5 @@ FRPC Զ
|
|||||||
不支持的位深度，需要24位或32位=Bitmap depth is unsupported
|
不支持的位深度，需要24位或32位=Bitmap depth is unsupported
|
||||||
未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=x264 Encoder is required \nDownload via：https://sourceforge.net/projects/x264vfw
|
未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=x264 Encoder is required \nDownload via：https://sourceforge.net/projects/x264vfw
|
||||||
创建AVI文件失败=Create AVI file failed
|
创建AVI文件失败=Create AVI file failed
|
||||||
|
启用 H264 硬编码=Enable HW H264 Encoding
|
||||||
|
启用 AV1 硬编码=Enable HW AV1 Encoding
|
||||||
|
|||||||
@@ -1883,3 +1883,5 @@ FRPC Զ
|
|||||||
不支持的位深度，需要24位或32位=不支持的位深度，需要24位或32位
|
不支持的位深度，需要24位或32位=不支持的位深度，需要24位或32位
|
||||||
未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw
|
未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw=未安装x264编解码器 \n下载地址：https://sourceforge.net/projects/x264vfw
|
||||||
创建AVI文件失败=创建AVI文件失败
|
创建AVI文件失败=创建AVI文件失败
|
||||||
|
启用 H264 硬编码=启用 H264 硬编码
|
||||||
|
启用 AV1 硬编码=启用 AV1 硬编码
|
||||||
|
|||||||
@@ -290,11 +290,13 @@ func (h *MyHandler) handleBitmapInfo(ctx *connection.Context, data []byte) {
|
|||||||
// handleScreenFrame relays one TOKEN_FIRSTSCREEN / TOKEN_NEXTSCREEN packet
|
// handleScreenFrame relays one TOKEN_FIRSTSCREEN / TOKEN_NEXTSCREEN packet
|
||||||
// to all browsers watching this device. The on-the-wire packet starts with
|
// to all browsers watching this device. The on-the-wire packet starts with
|
||||||
// the token byte then a small fixed header (algorithm, cursor pos, cursor
|
// the token byte then a small fixed header (algorithm, cursor pos, cursor
|
||||||
// index) before the H.264 NAL payload. The browser-facing WS packet uses
|
// index) before the video payload (H.264 Annex B or AV1 OBU). The browser-
|
||||||
// the C++-compatible layout: [deviceID:4 LE][frameType:1][dataLen:4 LE][H264:N].
|
// facing WS packet uses the C++-compatible layout:
|
||||||
|
// [deviceID:4 LE][frameType:1][dataLen:4 LE][Video:N].
|
||||||
//
|
//
|
||||||
// alwaysKey=true is used for TOKEN_FIRSTSCREEN (always IDR by construction);
|
// alwaysKey=true is used for TOKEN_FIRSTSCREEN (always IDR by construction);
|
||||||
// TOKEN_NEXTSCREEN is keyframe iff the NAL stream contains a 5/7/8 unit.
|
// TOKEN_NEXTSCREEN keyframe detection is delegated to protocol.IsAnyKeyframe
|
||||||
|
// which sniffs the codec from the first byte (0x00 → H.264, else AV1).
|
||||||
func (h *MyHandler) handleScreenFrame(ctx *connection.Context, data []byte, alwaysKey bool) {
|
func (h *MyHandler) handleScreenFrame(ctx *connection.Context, data []byte, alwaysKey bool) {
|
||||||
deviceID := h.hub.ScreenDeviceID(ctx)
|
deviceID := h.hub.ScreenDeviceID(ctx)
|
||||||
if deviceID == "" {
|
if deviceID == "" {
|
||||||
@@ -310,8 +312,10 @@ func (h *MyHandler) handleScreenFrame(ctx *connection.Context, data []byte, alwa
|
|||||||
// browser sees cursor updates even if we end up dropping frames later.
|
// browser sees cursor updates even if we end up dropping frames later.
|
||||||
h.hub.PublishCursor(deviceID, data[10])
|
h.hub.PublishCursor(deviceID, data[10])
|
||||||
|
|
||||||
h264 := data[skip:]
|
video := data[skip:]
|
||||||
isKey := alwaysKey || protocol.IsH264Keyframe(h264)
|
// 按首字节嗅探 H.264 / AV1，分发到对应的 keyframe 探测器。浏览器侧用同样方式
|
||||||
|
// 决定 VideoDecoder codec string，因此 server 不必感知客户端实际编码器。
|
||||||
|
isKey := alwaysKey || protocol.IsAnyKeyframe(video)
|
||||||
|
|
||||||
// Build the WS packet exactly as the C++ ScreenSpyDlg does — the front-end
|
// Build the WS packet exactly as the C++ ScreenSpyDlg does — the front-end
|
||||||
// decoder reads these offsets directly.
|
// decoder reads these offsets directly.
|
||||||
@@ -321,13 +325,13 @@ func (h *MyHandler) handleScreenFrame(ctx *connection.Context, data []byte, alwa
|
|||||||
if isKey {
|
if isKey {
|
||||||
frameType = 1
|
frameType = 1
|
||||||
}
|
}
|
||||||
dataLen := uint32(len(h264))
|
dataLen := uint32(len(video))
|
||||||
|
|
||||||
packet := make([]byte, 9+len(h264))
|
packet := make([]byte, 9+len(video))
|
||||||
binary.LittleEndian.PutUint32(packet[0:4], idLow)
|
binary.LittleEndian.PutUint32(packet[0:4], idLow)
|
||||||
packet[4] = frameType
|
packet[4] = frameType
|
||||||
binary.LittleEndian.PutUint32(packet[5:9], dataLen)
|
binary.LittleEndian.PutUint32(packet[5:9], dataLen)
|
||||||
copy(packet[9:], h264)
|
copy(packet[9:], video)
|
||||||
|
|
||||||
h.hub.PublishScreenFrame(deviceID, packet, isKey)
|
h.hub.PublishScreenFrame(deviceID, packet, isKey)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -350,6 +350,69 @@ func IsH264Keyframe(data []byte) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsAnyKeyframe sniffs the codec from the first byte then dispatches to the
|
||||||
|
// matching keyframe detector. H.264 Annex B always starts with 0x00 (start
|
||||||
|
// code prefix); AV1 OBU headers have bit7=0 and bits[3:6]=obu_type in [1,15]
|
||||||
|
// so the first byte is in [0x08,0x78] and never 0x00. Lets the server stay
|
||||||
|
// codec-agnostic so the browser can run H.264 and AV1 sessions side by side.
|
||||||
|
func IsAnyKeyframe(data []byte) bool {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if data[0] == 0x00 {
|
||||||
|
return IsH264Keyframe(data)
|
||||||
|
}
|
||||||
|
return IsAv1Keyframe(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsAv1Keyframe walks the OBU chain and returns true on the first
|
||||||
|
// OBU_SEQUENCE_HEADER (type 1). FFmpeg's AV1 encoders prepend SEQ HDR to
|
||||||
|
// every IDR, so seeing one is equivalent to "this packet contains a key
|
||||||
|
// frame". Mirrors the C++ IsAv1Keyframe helper in ScreenSpyDlg.cpp.
|
||||||
|
//
|
||||||
|
// AV1 OBU header byte layout: 0|type:4|ext:1|size:1|reserved:1
|
||||||
|
func IsAv1Keyframe(data []byte) bool {
|
||||||
|
n := len(data)
|
||||||
|
pos := 0
|
||||||
|
for pos < n {
|
||||||
|
hdr := data[pos]
|
||||||
|
obuType := (hdr >> 3) & 0x0F
|
||||||
|
hasExt := hdr&0x04 != 0
|
||||||
|
hasSize := hdr&0x02 != 0
|
||||||
|
if obuType == 1 { // OBU_SEQUENCE_HEADER
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
pos++
|
||||||
|
if hasExt {
|
||||||
|
if pos >= n {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
pos++
|
||||||
|
}
|
||||||
|
if !hasSize {
|
||||||
|
return false // unsized OBU runs to end of packet
|
||||||
|
}
|
||||||
|
// LEB128 size
|
||||||
|
var sz uint64
|
||||||
|
for i := range 8 {
|
||||||
|
if pos >= n {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b := data[pos]
|
||||||
|
pos++
|
||||||
|
sz |= uint64(b&0x7F) << (7 * i)
|
||||||
|
if b&0x80 == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if uint64(pos)+sz > uint64(n) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
pos += int(sz)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// LOGIN_INFOR structure size and offsets (matching C++ struct with default alignment)
|
// LOGIN_INFOR structure size and offsets (matching C++ struct with default alignment)
|
||||||
// Note: C++ struct uses default alignment (4-byte for uint32/int)
|
// Note: C++ struct uses default alignment (4-byte for uint32/int)
|
||||||
const (
|
const (
|
||||||
|
|||||||
@@ -1624,8 +1624,14 @@
|
|||||||
},
|
},
|
||||||
error: (e) => { console.error('Decoder error:', e); needKeyframe = true; }
|
error: (e) => { console.error('Decoder error:', e); needKeyframe = true; }
|
||||||
});
|
});
|
||||||
|
// codec string 由首帧嗅探得到的 currentCodec 决定：
|
||||||
|
// 'avc' → 'avc1.42E01E' (H.264 Constrained Baseline Level 3.0)
|
||||||
|
// 'av1' → 'av01.0.08M.08' (AV1 Main Profile Level 4.0 8-bit)
|
||||||
|
// 客户端硬件支持 AV1 编码时浏览器收到 AV1 流；fallback 到 H.264 时浏览器
|
||||||
|
// 收到 H.264 流。两条路径在同一前端代码中并存，运维侧无须感知。
|
||||||
|
const codecStr = currentCodec === 'av1' ? 'av01.0.08M.08' : 'avc1.42E01E';
|
||||||
decoder.configure({
|
decoder.configure({
|
||||||
codec: 'avc1.42E01E',
|
codec: codecStr,
|
||||||
codedWidth: width,
|
codedWidth: width,
|
||||||
codedHeight: height,
|
codedHeight: height,
|
||||||
optimizeForLatency: true
|
optimizeForLatency: true
|
||||||
@@ -1634,6 +1640,14 @@
|
|||||||
|
|
||||||
let decoderWidth = 0, decoderHeight = 0, needKeyframe = false;
|
let decoderWidth = 0, decoderHeight = 0, needKeyframe = false;
|
||||||
let decodeTimestamp = 0; // Monotonically increasing timestamp for decoder
|
let decodeTimestamp = 0; // Monotonically increasing timestamp for decoder
|
||||||
|
let currentCodec = null; // 'avc' | 'av1' | null（initDecoder 读取）
|
||||||
|
|
||||||
|
// 首字节嗅探：H.264 Annex B 起始码必以 0x00 开头；AV1 OBU header
|
||||||
|
// bit7=0 且 bits[3:6] = obu_type ∈ [1,15]，首字节落在 [0x08,0x78] 区间且
|
||||||
|
// 绝不为 0x00。单字节即可干净区分。
|
||||||
|
function detectCodec(videoBytes) {
|
||||||
|
return videoBytes[0] === 0x00 ? 'avc' : 'av1';
|
||||||
|
}
|
||||||
|
|
||||||
function handleBinaryFrame(data) {
|
function handleBinaryFrame(data) {
|
||||||
// 终端输出帧：4 字节 magic 'TRM1' (0x54 0x52 0x4D 0x31) → 转发到 xterm。
|
// 终端输出帧：4 字节 magic 'TRM1' (0x54 0x52 0x4D 0x31) → 转发到 xterm。
|
||||||
@@ -1650,11 +1664,26 @@
|
|||||||
const frameType = view.getUint8(4);
|
const frameType = view.getUint8(4);
|
||||||
const dataLen = view.getUint32(5, true);
|
const dataLen = view.getUint32(5, true);
|
||||||
const isKeyframe = frameType === 1;
|
const isKeyframe = frameType === 1;
|
||||||
|
const videoData = new Uint8Array(data, 9, dataLen);
|
||||||
|
const frameCodec = dataLen > 0 ? detectCodec(videoData) : currentCodec;
|
||||||
|
|
||||||
|
// codec 切换（客户端硬件 fallback、首次连接等）：必须等到 keyframe 才能
|
||||||
|
// 重建 decoder，delta 帧没有 SPS/PPS 或 SEQ HDR，无法独立初始化。
|
||||||
|
if (decoder && currentCodec && frameCodec !== currentCodec) {
|
||||||
|
if (!isKeyframe) {
|
||||||
|
needKeyframe = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try { decoder.close(); } catch (e) {}
|
||||||
|
decoder = null;
|
||||||
|
currentCodec = null;
|
||||||
|
}
|
||||||
|
|
||||||
// If decoder is closed or errored, wait for keyframe to reinitialize
|
// If decoder is closed or errored, wait for keyframe to reinitialize
|
||||||
if (!decoder || decoder.state === 'closed') {
|
if (!decoder || decoder.state === 'closed') {
|
||||||
if (isKeyframe && decoderWidth > 0) {
|
if (isKeyframe && decoderWidth > 0) {
|
||||||
console.log('Reinitializing decoder on keyframe');
|
currentCodec = frameCodec;
|
||||||
|
console.log('Reinitializing decoder on keyframe, codec=' + currentCodec);
|
||||||
initDecoder(decoderWidth, decoderHeight);
|
initDecoder(decoderWidth, decoderHeight);
|
||||||
needKeyframe = false;
|
needKeyframe = false;
|
||||||
} else {
|
} else {
|
||||||
@@ -1669,7 +1698,6 @@
|
|||||||
if (needKeyframe && !isKeyframe) return;
|
if (needKeyframe && !isKeyframe) return;
|
||||||
if (isKeyframe) needKeyframe = false;
|
if (isKeyframe) needKeyframe = false;
|
||||||
|
|
||||||
const h264Data = new Uint8Array(data, 9, dataLen);
|
|
||||||
try {
|
try {
|
||||||
// Check decoder queue to avoid overwhelming it (but never skip keyframes)
|
// Check decoder queue to avoid overwhelming it (but never skip keyframes)
|
||||||
if (!isKeyframe && decoder.decodeQueueSize > 10) {
|
if (!isKeyframe && decoder.decodeQueueSize > 10) {
|
||||||
@@ -1679,7 +1707,7 @@
|
|||||||
decoder.decode(new EncodedVideoChunk({
|
decoder.decode(new EncodedVideoChunk({
|
||||||
type: isKeyframe ? 'key' : 'delta',
|
type: isKeyframe ? 'key' : 'delta',
|
||||||
timestamp: decodeTimestamp++,
|
timestamp: decodeTimestamp++,
|
||||||
data: h264Data
|
data: videoData
|
||||||
}));
|
}));
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
console.error('Decode error:', e);
|
console.error('Decode error:', e);
|
||||||
|
|||||||
Reference in New Issue
Block a user