yuanyuanxiang/SimpleRemoter
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 9 Wiki Activity

Feature: Play remote audio on iPhone via WebAudio + self-hosted Opus WASM

Browse Source 
iOS WebKit lacks Media Source Extensions with WebM/Opus, so remote audio
was silent on iPhone Safari/Chrome even though audio frames kept flowing.
Add a Web Audio API fallback path that decodes Opus packets with the
`opus-decoder` WASM library and schedules them onto an AudioContext.

The decoder bundle is now self-hosted — served at /static/opus-decoder.min.js
by both the Go and C++ servers — instead of pulled from a CDN, which was
unreachable from some networks.

Gotcha fixed during bring-up: the unscoped `opus-decoder` build's
`OpusDecoder.decodeFrame` is synchronous and returns {channelData,
samplesDecoded, sampleRate, errors} directly, not a Promise, so it must be
called in try/catch rather than via .then().

Also remove the on-screen diagnostic overlay (audio toast + frame counter)
used while debugging the phone without a console; diagnostics now go to
console.* only.

Co-Authored-By: deepseek-v4-pro
This commit is contained in:
yuanyuanxianganddeepseek-v4-pro committed 2026-10-03 18:23:10 +02:00
1 parent d118558eda
commit f3a05c14b1
10 files changed
+655 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/web/index.html
+180 -13
View File
@@ -1166,7 +1166,7 @@
<div class="login-footer">
<div class="security-notice">
<strong>&#x26A0; Security Notice</strong>
SimpleRemoter.com may be flagged as "dangerous" by browsers due to the word "Remote" in its name. This is a false positive. The software is fully open-source and safe to use.
This software is for lawful, authorized, and legitimate use only. You are solely responsible for your use and for complying with applicable law. The developer assumes no liability.
</div>
<div class="login-links">
<a href="https://simpleremoter.com/" target="_blank">&#x1F310; Website</a>
@@ -1352,8 +1352,11 @@
<script src="__BASE_PATH__/static/xterm.js"></script>
<script src="__BASE_PATH__/static/xterm-fit.js"></script>
<!-- Opus codec for audio decompression -->
<script src="https://cdn.jsdelivr.net/npm/opus.js@0.5.0/dist/opus.js"></script>
<!-- Opus WASM decoder (WebAudio fallback for browsers without MSE+WebM/Opus,
e.g. iOS Safari/Chrome where WebKit's MediaSource lacks WebM/Opus).
Self-hosted (served from /static/) so audio works without reaching the
internet — a CDN copy is unreliable behind GFW/offline LANs. -->
<script src="__BASE_PATH__/static/opus-decoder.min.js" charset="UTF-8"></script>
<script>
let ws = null, token = null, decoder = null, devices = [], currentDevice = null;
@@ -1387,11 +1390,11 @@
// channels:2 sampleRate:4 bitsPerSample:2 blockAlign:2
// compression:1 (0=PCM unsupported by web, 1=Opus) reserved:1
// MSE + WebM/Opus playback. Raw Opus packets arrive over WS; we wrap
// each one in a minimal WebM container in JS and feed it to a
// SourceBuffer attached to a hidden <audio> element. The browser
// decodes Opus natively. Tested on desktop Chrome; mobile playback
// is a known follow-up (see commit notes).
// MSE + WebM/Opus playback (desktop Chrome/Firefox/Android). Raw Opus
// packets arrive over WS; we wrap each one in a minimal WebM container
// and feed it to a SourceBuffer attached to a hidden <audio> element.
// Browsers without MSE+WebM/Opus (iOS Safari/Chrome — WebKit has no
// WebM/Opus in MediaSource) fall back to the WebAudio + WASM path below.
let audioFormat = null; // { compression, channels, sampleRate, bitsPerSample, blockAlign }
let audioEnabled = true; // Audio on/off flag (set by UI)
let syncDrift = 0; // A/V sync monitoring (milliseconds)
@@ -1405,13 +1408,35 @@
const OPUS_FRAME_MS = 20; // 960 samples @ 48k — matches client encoder
const _pendingOpusPackets = []; // packets received before SourceBuffer is ready
// ---- WebAudio + WASM Opus fallback path ----
// Used when MSE + WebM/Opus is unsupported (iOS Safari/Chrome — WebKit's
// MediaSource has no WebM/Opus). Raw Opus packets are decoded to PCM
// with a WASM decoder (opus-decoder) and played via AudioContext. The
// decision is made on kernel capability, not UA, so it covers every iOS
// browser at once.
const _mseOpusSupported = !!(window.MediaSource &&
MediaSource.isTypeSupported('audio/webm; codecs="opus"'));
const _useWebAudioPath = !_mseOpusSupported;
let _audioCtx = null; // AudioContext (created/resumed inside a gesture)
let _opusDecoder = null; // WASM OpusDecoder (created when format is known)
let _decoderReady = false; // _opusDecoder.ready resolved
let _waNextTime = null; // next AudioBufferSourceNode start time (ctx clock)
let _waDropLogged = false; // one-shot console.warn for per-frame drop reasons
let _waDecodeFailLogged = false; // one-shot console.error for decode failures
const WA_LOOKAHEAD = 0.15; // seconds of lead time kept in the scheduler
const WA_MAX_LATENCY = 0.60; // drop backlog if we drift this far ahead
const WA_DECODE_SR = 48000; // Opus always decodes to 48 kHz
if (_useWebAudioPath) console.log('[Audio] WebAudio fallback path (no MSE/WebM-Opus)');
// Browser autoplay policies require an HTMLAudioElement to be created
// and .play()'d synchronously inside a user-gesture event handler.
// We hook the first click/keydown to spin up the element + MediaSource.
// Subsequent activity (e.g. tab regaining focus) re-issues play().
function installAudioGestureUnlock() {
const onGesture = () => {
if (!_audioElement) {
if (_useWebAudioPath) {
_setupWebAudioContext();
} else if (!_audioElement) {
try {
_setupAudioElementAndMediaSource();
console.log('[MSE] <audio> + MediaSource set up by gesture');
@@ -1423,11 +1448,137 @@
}
};
const opts = { passive: true, capture: true };
window.addEventListener('click', onGesture, opts);
window.addEventListener('keydown', onGesture, opts);
// click/keydown cover desktop; touchend/pointerdown are the most
// reliable "user gesture" triggers for AudioContext.resume() on iOS.
window.addEventListener('click', onGesture, opts);
window.addEventListener('keydown', onGesture, opts);
window.addEventListener('touchend', onGesture, opts);
window.addEventListener('pointerdown', onGesture, opts);
// Recover when the app returns to the foreground (iOS may suspend
// the AudioContext while backgrounded).
document.addEventListener('visibilitychange', () => {
if (!document.hidden && _useWebAudioPath) _setupWebAudioContext();
});
}
installAudioGestureUnlock();
// ============================================================
// WebAudio + WASM Opus fallback playback.
// The AudioContext is created/resumed inside installAudioGestureUnlock's
// user-gesture handler above (required by iOS autoplay policy). The
// WASM decoder is created lazily when the first AudioFormat arrives
// (handleAudioFrame), so it knows nothing about the container/codec
// support the MSE path depends on.
// ============================================================
function _setupWebAudioContext() {
if (_audioCtx) {
if (_audioCtx.state === 'suspended') {
_audioCtx.resume()
.then(() => console.log('[Audio] resumed → ' + _audioCtx.state))
.catch(e => console.error('[Audio] resume rejected: ' + (e && e.message)));
}
return;
}
const AC = window.AudioContext || window.webkitAudioContext;
if (!AC) { console.error('[Audio] Web Audio API unavailable'); return; }
try {
_audioCtx = new AC();
console.log('[Audio] AudioContext created, resuming…');
_audioCtx.resume()
.then(() => console.log('[Audio] AudioContext ' + _audioCtx.state))
.catch(e => console.error('[Audio] resume rejected: ' + (e && e.message)));
} catch (e) {
console.error('[Audio] AudioContext failed: ' + (e && e.message));
}
}
function _initWebAudioDecoder(fmt) {
if (_opusDecoder) { try { _opusDecoder.free().catch(() => {}); } catch (e) {} }
_opusDecoder = null;
_decoderReady = false;
_waNextTime = null;
_waDropLogged = false;
_waDecodeFailLogged = false;
const OpusDecoder = window['opus-decoder'] && window['opus-decoder'].OpusDecoder;
if (!OpusDecoder) {
console.error('[Audio] opus-decoder not loaded (static script missing?)');
return;
}
try {
_opusDecoder = new OpusDecoder({ sampleRate: WA_DECODE_SR, channels: (fmt && fmt.channels) || 2 });
_opusDecoder.ready.then(() => {
_decoderReady = true;
console.log('[Audio] WebAudio decoder ready ch=' + fmt.channels);
}).catch(e => {
console.error('[Audio] decoder ready failed: ' + (e && e.message));
});
} catch (e) {
console.error('[Audio] decoder construction failed: ' + (e && e.message));
}
}
function _pushOpusWebAudio(opusBytes) {
if (!_opusDecoder || !_decoderReady) return; // WASM still compiling
if (!_audioCtx) {
if (!_waDropLogged) { _waDropLogged = true; console.warn('[Audio] no AudioContext (tap the page first)'); }
return;
}
if (_audioCtx.state !== 'running') {
if (!_waDropLogged) { _waDropLogged = true; console.warn('[Audio] ctx ' + _audioCtx.state + ' — dropping audio'); }
return;
}
// decodeFrame in this bundle is SYNCHRONOUS: it drives the WASM
// decoder directly and returns {channelData, samplesDecoded,
// sampleRate, errors} — not a Promise. Calling .then() on it throws
// a TypeError before any error handling can run. Decode in try/catch.
let res;
try {
res = _opusDecoder.decodeFrame(new Uint8Array(opusBytes));
} catch (e) {
if (!_waDecodeFailLogged) { _waDecodeFailLogged = true; console.error('[Audio] decodeFrame threw: ' + (e && e.message)); }
return;
}
if (!res || !res.samplesDecoded) {
let errStr = 'no samples';
if (res && res.errors && res.errors.length) {
try { errStr = String(res.errors[0].message || res.errors[0]); } catch (e) { errStr = String(res.errors[0]); }
}
if (!_waDecodeFailLogged) { _waDecodeFailLogged = true; console.error('[Audio] decode fail: ' + errStr); }
return;
}
const ch = res.channelData ? res.channelData.length : 0;
if (ch < 1) {
if (!_waDecodeFailLogged) { _waDecodeFailLogged = true; console.error('[Audio] decode ch=0 samples=' + res.samplesDecoded); }
return;
}
if (!_audioCtx || _audioCtx.state !== 'running') {
if (!_waDropLogged) { _waDropLogged = true; console.warn('[Audio] ctx ' + (_audioCtx && _audioCtx.state) + ' at schedule'); }
return;
}
const samples = res.samplesDecoded;
const sr = res.sampleRate || WA_DECODE_SR;
const buffer = _audioCtx.createBuffer(ch, samples, sr);
for (let c = 0; c < ch; c++) {
if (res.channelData[c]) buffer.copyToChannel(res.channelData[c], c);
}
const src = _audioCtx.createBufferSource();
src.buffer = buffer;
src.connect(_audioCtx.destination);
const now = _audioCtx.currentTime;
if (_waNextTime === null || _waNextTime < now) {
_waNextTime = now + WA_LOOKAHEAD;
}
src.start(_waNextTime);
_waNextTime += samples / sr;
if (_waNextTime - now > WA_MAX_LATENCY) {
// Decoder/latency spike put us too far behind; drop the
// backlog and stay live instead of playing stale audio.
_waNextTime = now + WA_LOOKAHEAD;
}
}
// Pagination and filter state
let currentPage = 1;
let viewMode = 'grid'; // 'grid' or 'list'
@@ -2003,6 +2154,10 @@
function pushOpusPacket(opusBytes) {
if (!audioFormat || audioFormat.compression !== 1) return;
if (_useWebAudioPath) {
_pushOpusWebAudio(opusBytes);
return;
}
if (_sourceBuffer && _initSegmentSent) {
_enqueueAppend(WebMMuxer.buildCluster(opusBytes, _opusTimestampMs));
_opusTimestampMs += OPUS_FRAME_MS;
@@ -2030,6 +2185,12 @@
_initSegmentSent = false;
_opusTimestampMs = 0;
_pendingOpusPackets.length = 0;
// Reset the WebAudio fallback (keep the AudioContext alive so we
// don't forfeit the gesture-acquired play permission on reinit).
if (_opusDecoder) { try { _opusDecoder.free().catch(() => {}); } catch (e) {} }
_opusDecoder = null;
_decoderReady = false;
_waNextTime = null;
}
function handleAudioFrame(data) {
@@ -2073,10 +2234,12 @@
return;
}
stopAllAudio();
if (_mediaSource && _mediaSource.readyState === 'open') {
if (_useWebAudioPath) {
_initWebAudioDecoder(fmt);
} else if (_mediaSource && _mediaSource.readyState === 'open') {
_addSourceBufferAndInit();
}
// else: sourceopen handler will pick up audioFormat when it fires
// else (MSE): sourceopen handler will pick up audioFormat when it fires
console.log('[Audio] Format → ch=' + fmt.channels +
' sr=' + fmt.sampleRate + ' compression=' + fmt.compression);
}
@@ -3079,7 +3242,11 @@
// authoritative — toggleAudio() does not flip state locally; it only
// sends the request and waits for the audio_state broadcast.
function applyAudioState(enabled) {
const prev = audioEnabled;
audioEnabled = !!enabled;
if (prev !== audioEnabled) {
console.log('[Audio] ' + (audioEnabled ? 'enabled' : 'disabled'));
}
const btn = document.getElementById('btn-audio-bar');
if (btn) {
// 0x1F50A speaker / 0x1F507 muted speaker