Feature: Implement H.264 and AV1 hardware encoding for remote control
Remark: Need to update FFmpeg static libraries to take effort
This commit is contained in:
yuanyuanxiang
299
client/CFFmpegH264Encoder.cpp
Normal file
299
client/CFFmpegH264Encoder.cpp
Normal file
@@ -0,0 +1,299 @@
|
||||
#include "CFFmpegH264Encoder.h"
|
||||
#include "common/config.h"
|
||||
#include "common/logger.h"
|
||||
|
||||
// 合规守护：DISABLE_FFMPEG_FOR_TEST=1 时整个实现 + 所有 #pragma comment(lib,"ffmpeg/...")
|
||||
// 都不进编译单元，FFmpeg 静态库不会被链接进二进制
|
||||
#if defined(_WIN64) && !DISABLE_FFMPEG_FOR_TEST
|
||||
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavutil/opt.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libyuv/libyuv.h>
|
||||
}
|
||||
|
||||
#include <string.h>
|
||||
#include <cstdlib>
|
||||
|
||||
// FFmpeg 静态库 + 必要的 Windows 系统库。x86 build 不引入，由 _WIN64 守护。
|
||||
// FFmpeg 三个核心库是纯 C，CRT 中性，Debug/Release 共用一份。
|
||||
#pragma comment(lib,"ffmpeg/libavcodec_x64.lib")
|
||||
#pragma comment(lib,"ffmpeg/libavutil_x64.lib")
|
||||
#pragma comment(lib,"ffmpeg/libswresample_x64.lib")
|
||||
// dav1d (AV1 软解，C 项目) —— 不分 Debug/Release。
|
||||
// build 时启用了 --enable-libdav1d，libavcodec 内部 av1 decoder 引用了 dav1d 符号。
|
||||
#pragma comment(lib,"ffmpeg/dav1d_x64.lib")
|
||||
// libvpl (Intel QSV, C++ 项目) —— 强制 CRT 一致，必须按 _DEBUG 切。
|
||||
// build 时启用了 --enable-libvpl，libavcodec 内部 h264_qsv / av1_qsv encoder 引用 MFX 符号。
|
||||
#ifdef _DEBUG
|
||||
#pragma comment(lib,"ffmpeg/vpl_x64d.lib")
|
||||
#else
|
||||
#pragma comment(lib,"ffmpeg/vpl_x64.lib")
|
||||
#endif
|
||||
|
||||
#pragma comment(lib, "mfplat.lib")
|
||||
#pragma comment(lib, "mfuuid.lib")
|
||||
#pragma comment(lib, "strmiids.lib")
|
||||
#pragma comment(lib, "secur32.lib")
|
||||
#pragma comment(lib, "bcrypt.lib")
|
||||
#pragma comment(lib, "advapi32.lib")
|
||||
#pragma comment(lib, "ole32.lib")
|
||||
// ws2_32 在 IOCPClient.h 已 link，重复不冲突
|
||||
#pragma comment(lib, "ws2_32.lib")
|
||||
|
||||
// av_opt_set wrappers：FFmpeg 在选项名/值拼错时 silently 返回 AVERROR_OPTION_NOT_FOUND
|
||||
// 不报错，导致 encoder 退回默认行为且没人察觉（实际踩过：AMF rc=vbr_peak_constrained
|
||||
// 拼成全名，FFmpeg 实际只接受 vbr_peak，没设上去就退回 CBR）。
|
||||
// 包一层 helper，任何设置失败 Mprintf 警告。
|
||||
static void setOpt(void* obj, const char* name, const char* val, const char* backend) {
|
||||
int rc = av_opt_set(obj, name, val, 0);
|
||||
if (rc < 0) {
|
||||
char errbuf[128] = {0};
|
||||
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||
Mprintf("[WARN] av_opt_set('%s'='%s') on %s failed (%d): %s\n",
|
||||
name, val, backend, rc, errbuf);
|
||||
}
|
||||
}
|
||||
static void setOptInt(void* obj, const char* name, int64_t val, const char* backend) {
|
||||
int rc = av_opt_set_int(obj, name, val, 0);
|
||||
if (rc < 0) {
|
||||
char errbuf[128] = {0};
|
||||
av_strerror(rc, errbuf, sizeof(errbuf));
|
||||
Mprintf("[WARN] av_opt_set_int('%s'=%lld) on %s failed (%d): %s\n",
|
||||
name, (long long)val, backend, rc, errbuf);
|
||||
}
|
||||
}
|
||||
|
||||
// 后端探测顺序：NVIDIA > Intel > AMD > Windows MF 兜底。
|
||||
// open() 主循环按顺序试，第一个 avcodec_open2 成功的就用。
|
||||
// h264_mf 质量/稳定性一般，但是 Windows 系统级 hwaccel，任何 GPU 都能尝试，作最后兜底。
|
||||
static const char* kH264Backends[] = {
|
||||
"h264_nvenc", // NVIDIA NVENC
|
||||
"h264_qsv", // Intel Quick Sync Video
|
||||
"h264_amf", // AMD AMF
|
||||
"h264_mf", // Windows Media Foundation
|
||||
};
|
||||
|
||||
CFFmpegH264Encoder::CFFmpegH264Encoder() = default;
|
||||
|
||||
CFFmpegH264Encoder::~CFFmpegH264Encoder() {
|
||||
close();
|
||||
}
|
||||
|
||||
void CFFmpegH264Encoder::cleanupCodec() {
|
||||
if (m_packet) { av_packet_free(&m_packet); m_packet = nullptr; }
|
||||
if (m_frame) { av_frame_free(&m_frame); m_frame = nullptr; }
|
||||
if (m_ctx) { avcodec_free_context(&m_ctx); m_ctx = nullptr; }
|
||||
}
|
||||
|
||||
void CFFmpegH264Encoder::close() {
|
||||
cleanupCodec();
|
||||
m_backend.clear();
|
||||
m_pts = 0;
|
||||
m_forceIDR = false;
|
||||
}
|
||||
|
||||
bool CFFmpegH264Encoder::open(const EncoderParams& params) {
|
||||
close();
|
||||
for (const char* name : kH264Backends) {
|
||||
if (tryOpenBackend(name, params)) {
|
||||
m_backend = name;
|
||||
return true;
|
||||
}
|
||||
cleanupCodec(); // 释放本次失败的 ctx，准备下一次尝试
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool CFFmpegH264Encoder::tryOpenBackend(const char* name, const EncoderParams& p) {
|
||||
const AVCodec* codec = avcodec_find_encoder_by_name(name);
|
||||
if (!codec) {
|
||||
// 失败 = lib 里没注册这个 encoder。几乎肯定是链到了老 ffmpeg lib。
|
||||
Mprintf("=> FFmpeg: encoder '%s' NOT in linked lib (old ffmpeg?)\n", name);
|
||||
return false;
|
||||
}
|
||||
|
||||
m_ctx = avcodec_alloc_context3(codec);
|
||||
if (!m_ctx) {
|
||||
Mprintf("=> FFmpeg: avcodec_alloc_context3('%s') failed\n", name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 偶数对齐（与 x264 路径 i_width/i_height & 0xfffffffe 一致）
|
||||
m_ctx->width = p.width & ~1;
|
||||
m_ctx->height = p.height & ~1;
|
||||
m_ctx->time_base = AVRational{1, p.fps};
|
||||
m_ctx->framerate = AVRational{p.fps, 1};
|
||||
m_ctx->pix_fmt = AV_PIX_FMT_NV12;
|
||||
m_ctx->gop_size = p.fps * (p.gop_seconds > 0 ? p.gop_seconds : 4);
|
||||
m_ctx->max_b_frames = 0;
|
||||
m_ctx->bit_rate = (int64_t)p.bitrate_kbps * 1000;
|
||||
m_ctx->rc_max_rate = (int64_t)p.bitrate_kbps * 1500;
|
||||
m_ctx->rc_buffer_size = (int)(p.bitrate_kbps * 1000);
|
||||
|
||||
// RC 策略选择：远程办公 90% 时间是静态画面（文档/IDE/邮件），CBR 会强行
|
||||
// 把目标码率填满（静态用不上的部分浪费带宽）。所有硬编后端统一改用 VBR，
|
||||
// bit_rate 是平均目标、rc_max_rate (1.5x) 是峰值上限：静态时 encoder 自动
|
||||
// 降码率省带宽，动态时回到目标 + 短暂上探到 1.5x 保证画质。
|
||||
// 接近 x264 软编 CRF + VBV 的行为，但严格守住峰值不爆。
|
||||
if (strcmp(name, "h264_nvenc") == 0) {
|
||||
// NVENC preset: p1(最快/低质) ~ p7(最慢/高质)，远控低延迟 p5 兼顾。
|
||||
// tune=ll low-latency；rc=vbr 配 max_rate 实现峰值受限的 VBR。
|
||||
setOpt(m_ctx->priv_data, "preset", "p5", name);
|
||||
setOpt(m_ctx->priv_data, "tune", "ll", name);
|
||||
setOpt(m_ctx->priv_data, "rc", "vbr", name);
|
||||
setOpt(m_ctx->priv_data, "zerolatency", "1", name);
|
||||
} else if (strcmp(name, "h264_qsv") == 0) {
|
||||
// Intel Quick Sync Video。preset: veryfast/faster/fast/medium/slow/slower/veryslow
|
||||
// QSV 当 bit_rate != rc_max_rate 时自动走 VBR，所以这里只需调 preset。
|
||||
// preset=slow 比 medium 慢但画质好，async_depth=1 单帧立即出包。
|
||||
// low_power=0 走 PAK 路径，部分集显不支持 low_power 模式。
|
||||
setOpt(m_ctx->priv_data, "preset", "slow", name);
|
||||
setOptInt(m_ctx->priv_data, "async_depth", 1, name);
|
||||
setOptInt(m_ctx->priv_data, "low_power", 0, name);
|
||||
} else if (strcmp(name, "h264_amf") == 0) {
|
||||
// AMD AMF 远控低延迟配置：
|
||||
// usage=ultralowlatency 比 lowlatency 更激进，关闭一切 lookahead；
|
||||
// quality=speed 选最快编码路径（vs balanced/quality）；
|
||||
// rc=cbr 提供最可预测的输出节拍，避免 RC 切换抖动。
|
||||
// 静态画面省码率交给应用层 skip 检测（ScreenCapture::GetNextScreenData
|
||||
// 已经过 memcmp 把无变化帧直接拦在编码器之前），不再依赖 vbaq/preanalysis
|
||||
// 这些会引入 30-100ms lookahead 的"省码率三件套"。
|
||||
setOpt(m_ctx->priv_data, "usage", "ultralowlatency", name);
|
||||
setOpt(m_ctx->priv_data, "quality", "speed", name);
|
||||
setOpt(m_ctx->priv_data, "rc", "cbr", name);
|
||||
setOptInt(m_ctx->priv_data, "filler_data", 0, name);
|
||||
setOptInt(m_ctx->priv_data, "enforce_hrd", 0, name);
|
||||
} else if (strcmp(name, "h264_mf") == 0) {
|
||||
// Windows Media Foundation 兜底。rate_control 实际值（ffmpeg -h encoder=h264_mf）：
|
||||
// default / cbr / pc_vbr / u_vbr / quality / ld_vbr / g_vbr / gld_vbr
|
||||
// 远控用 pc_vbr (peak-constrained VBR) 与其他后端语义对齐。
|
||||
setOptInt(m_ctx->priv_data, "hw_encoding", 1, name);
|
||||
setOpt(m_ctx->priv_data, "rate_control", "pc_vbr", name);
|
||||
}
|
||||
|
||||
int ret = avcodec_open2(m_ctx, codec, nullptr);
|
||||
if (ret < 0) {
|
||||
// 失败 = encoder 找到了但开不起来。常见：无 NVIDIA GPU / 驱动太旧 /
|
||||
// NVENC session 占满 / 笔记本独显未唤醒 / 参数组合驱动不接受
|
||||
char errbuf[128] = {0};
|
||||
av_strerror(ret, errbuf, sizeof(errbuf));
|
||||
Mprintf("=> FFmpeg: avcodec_open2('%s') failed (%d): %s\n", name, ret, errbuf);
|
||||
return false;
|
||||
}
|
||||
|
||||
m_frame = av_frame_alloc();
|
||||
if (!m_frame) return false;
|
||||
m_frame->format = AV_PIX_FMT_NV12;
|
||||
m_frame->width = m_ctx->width;
|
||||
m_frame->height = m_ctx->height;
|
||||
if (av_frame_get_buffer(m_frame, 32) < 0) {
|
||||
Mprintf("=> FFmpeg: av_frame_get_buffer failed\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
m_packet = av_packet_alloc();
|
||||
return m_packet != nullptr;
|
||||
}
|
||||
|
||||
void CFFmpegH264Encoder::setBitrate(int kbps) {
|
||||
if (!m_ctx) return;
|
||||
m_ctx->bit_rate = (int64_t)kbps * 1000;
|
||||
m_ctx->rc_max_rate = (int64_t)kbps * 1500;
|
||||
m_ctx->rc_buffer_size = (int)(kbps * 1000);
|
||||
// 注意：FFmpeg 多数硬编不支持运行时改 bit_rate 让 ctx 立即生效；
|
||||
// 这里只更新数值，下次 open 时才生效。Step 1 不依赖动态调码率。
|
||||
}
|
||||
|
||||
int CFFmpegH264Encoder::convertRGB24ToNV12(uint8_t* rgb, uint32_t stride,
|
||||
uint32_t width, uint32_t height,
|
||||
int direction)
|
||||
{
|
||||
int signed_height = direction * (int)height;
|
||||
int w = (int)width;
|
||||
int h = (int)height;
|
||||
int y_size = w * h;
|
||||
int uv_size = (w / 2) * (h / 2);
|
||||
m_i420Scratch.resize(y_size + 2 * uv_size);
|
||||
|
||||
uint8_t* y = m_i420Scratch.data();
|
||||
uint8_t* u = y + y_size;
|
||||
uint8_t* v = u + uv_size;
|
||||
|
||||
if (libyuv::RGB24ToI420(
|
||||
rgb, stride,
|
||||
y, w,
|
||||
u, w / 2,
|
||||
v, w / 2,
|
||||
w, signed_height) != 0) {
|
||||
return -1;
|
||||
}
|
||||
if (libyuv::I420ToNV12(
|
||||
y, w,
|
||||
u, w / 2,
|
||||
v, w / 2,
|
||||
m_frame->data[0], m_frame->linesize[0],
|
||||
m_frame->data[1], m_frame->linesize[1],
|
||||
w, h) != 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int CFFmpegH264Encoder::encode(
|
||||
uint8_t* rgb, uint8_t bpp, uint32_t stride,
|
||||
uint32_t width, uint32_t height,
|
||||
uint8_t** lppData, uint32_t* lpSize, int direction)
|
||||
{
|
||||
if (!m_ctx || !m_frame || !m_packet) return -1;
|
||||
if (av_frame_make_writable(m_frame) < 0) return -1;
|
||||
|
||||
int w = (int)width;
|
||||
int h = (int)height;
|
||||
int signed_height = direction * h;
|
||||
|
||||
if (bpp == 32) {
|
||||
if (libyuv::ARGBToNV12(
|
||||
rgb, stride,
|
||||
m_frame->data[0], m_frame->linesize[0],
|
||||
m_frame->data[1], m_frame->linesize[1],
|
||||
w, signed_height) != 0) {
|
||||
return -1;
|
||||
}
|
||||
} else if (bpp == 24) {
|
||||
if (convertRGB24ToNV12(rgb, stride, width, height, direction) != 0) {
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
return -2;
|
||||
}
|
||||
|
||||
m_frame->pts = m_pts++;
|
||||
if (m_forceIDR) {
|
||||
m_frame->pict_type = AV_PICTURE_TYPE_I;
|
||||
m_forceIDR = false;
|
||||
} else {
|
||||
m_frame->pict_type = AV_PICTURE_TYPE_NONE;
|
||||
}
|
||||
|
||||
int ret = avcodec_send_frame(m_ctx, m_frame);
|
||||
if (ret < 0) return -3;
|
||||
|
||||
ret = avcodec_receive_packet(m_ctx, m_packet);
|
||||
if (ret == AVERROR(EAGAIN)) {
|
||||
// 首帧延迟：本次没出包，调用方按 lpSize==0 跳过本帧
|
||||
*lppData = nullptr;
|
||||
*lpSize = 0;
|
||||
return 0;
|
||||
}
|
||||
if (ret < 0) return -4;
|
||||
|
||||
m_outputBuffer.assign(m_packet->data, m_packet->data + m_packet->size);
|
||||
*lppData = m_outputBuffer.data();
|
||||
*lpSize = (uint32_t)m_outputBuffer.size();
|
||||
av_packet_unref(m_packet);
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif // _WIN64 && !DISABLE_FFMPEG_FOR_TEST
|
||||
Reference in New Issue
Block a user