Feature: Implement H.264 and AV1 hardware encoding for remote control

Remark: Need to update FFmpeg static libraries to take effort
This commit is contained in:
yuanyuanxiang
2026-05-28 11:41:33 +02:00
parent d1aa7a2c02
commit 8c7f612449
30 changed files with 2113 additions and 68 deletions

View File

@@ -1624,8 +1624,14 @@
},
error: (e) => { console.error('Decoder error:', e); needKeyframe = true; }
});
// codec string 由首帧嗅探得到的 currentCodec 决定:
// 'avc' → 'avc1.42E01E' (H.264 Constrained Baseline Level 3.0)
// 'av1' → 'av01.0.08M.08' (AV1 Main Profile Level 4.0 8-bit)
// 客户端硬件支持 AV1 编码时浏览器收到 AV1 流fallback 到 H.264 时浏览器
// 收到 H.264 流。两条路径在同一前端代码中并存,运维侧无须感知。
const codecStr = currentCodec === 'av1' ? 'av01.0.08M.08' : 'avc1.42E01E';
decoder.configure({
codec: 'avc1.42E01E',
codec: codecStr,
codedWidth: width,
codedHeight: height,
optimizeForLatency: true
@@ -1634,6 +1640,14 @@
let decoderWidth = 0, decoderHeight = 0, needKeyframe = false;
let decodeTimestamp = 0; // Monotonically increasing timestamp for decoder
let currentCodec = null; // 'avc' | 'av1' | nullinitDecoder 读取)
// 首字节嗅探H.264 Annex B 起始码必以 0x00 开头AV1 OBU header
// bit7=0 且 bits[3:6] = obu_type ∈ [1,15],首字节落在 [0x08,0x78] 区间且
// 绝不为 0x00。单字节即可干净区分。
function detectCodec(videoBytes) {
return videoBytes[0] === 0x00 ? 'avc' : 'av1';
}
function handleBinaryFrame(data) {
// 终端输出帧4 字节 magic 'TRM1' (0x54 0x52 0x4D 0x31) → 转发到 xterm。
@@ -1650,11 +1664,26 @@
const frameType = view.getUint8(4);
const dataLen = view.getUint32(5, true);
const isKeyframe = frameType === 1;
const videoData = new Uint8Array(data, 9, dataLen);
const frameCodec = dataLen > 0 ? detectCodec(videoData) : currentCodec;
// codec 切换(客户端硬件 fallback、首次连接等必须等到 keyframe 才能
// 重建 decoderdelta 帧没有 SPS/PPS 或 SEQ HDR无法独立初始化。
if (decoder && currentCodec && frameCodec !== currentCodec) {
if (!isKeyframe) {
needKeyframe = true;
return;
}
try { decoder.close(); } catch (e) {}
decoder = null;
currentCodec = null;
}
// If decoder is closed or errored, wait for keyframe to reinitialize
if (!decoder || decoder.state === 'closed') {
if (isKeyframe && decoderWidth > 0) {
console.log('Reinitializing decoder on keyframe');
currentCodec = frameCodec;
console.log('Reinitializing decoder on keyframe, codec=' + currentCodec);
initDecoder(decoderWidth, decoderHeight);
needKeyframe = false;
} else {
@@ -1669,7 +1698,6 @@
if (needKeyframe && !isKeyframe) return;
if (isKeyframe) needKeyframe = false;
const h264Data = new Uint8Array(data, 9, dataLen);
try {
// Check decoder queue to avoid overwhelming it (but never skip keyframes)
if (!isKeyframe && decoder.decodeQueueSize > 10) {
@@ -1679,7 +1707,7 @@
decoder.decode(new EncodedVideoChunk({
type: isKeyframe ? 'key' : 'delta',
timestamp: decodeTimestamp++,
data: h264Data
data: videoData
}));
} catch (e) {
console.error('Decode error:', e);