Feature: Sniff WebSocket upgrades on the MFC server's device port

This commit is contained in:
yuanyuanxiang committed 2026-10-30 17:22:17 +01:00
1 parent d4f312c9d8
commit cc891e3670
11 files changed
+925 -32

No files matched your search

+276 -21
View File
@@ -4,6 +4,7 @@
#include "common/IPWhitelist.h"
#include "common/IPBlacklist.h"
#include "ServerTlsSession.h"
#include "ServerWsSession.h"
#include <iostream>
#include <ws2tcpip.h>
@@ -947,26 +948,33 @@ DWORD WINAPI IOCPServer::TlsHandshakeThreadProc(LPVOID lParam)
// 注意顺序:解析仍在本线程持有引用期间完成(对象不会在这段窗口里被回收);
// 释放引用之后才允许任何可能 RemoveStaleContext 的动作(#215)。
bool wsTookOver = false;
if (ok && !pendingPlain.empty()) {
// 与工作线程同样的解压上下文,只是现建现用(首包只解析这一批)。
// 走 DispatchPlain:WSS 的首个明文就是 "GET /api/ws HTTP/1.1...",
// 由它判定升级并交给 WS 升级线程(此时本线程不能再 PostRecv)。
ZSTD_DCtx* dctx = ZSTD_createDCtx();
z_stream z = {};
inflateInit2(&z, 15);
ok = ctx->IsRemoved.load(std::memory_order_acquire) ? TRUE
: ParseReceivedData(ctx, pendingPlain.data(), (DWORD)pendingPlain.size(),
This->m_NotifyProc, dctx, &z);
int r = ctx->IsRemoved.load(std::memory_order_acquire) ? 1
: This->DispatchPlain(ctx, pendingPlain.data(), (DWORD)pendingPlain.size(), dctx, &z);
inflateEnd(&z);
ZSTD_freeDCtx(dctx);
if (!ok) {
if (r < 0) {
ok = false;
Mprintf("[TLS] pending first packet parse failed for %s (closing)\n",
ctx->GetPeerName().c_str());
} else if (r == 0) {
wsTookOver = true;
}
}
ctx->IoRefCount.fetch_sub(1);
if (ok) {
This->PostRecv(ctx); // 交还 IOCP 接收
if (!wsTookOver) {
This->PostRecv(ctx); // 交还 IOCP 接收
}
} else {
Mprintf("[TLS] handshake failed for %s (fail-closed)\n", ctx->GetPeerName().c_str());
This->RemoveStaleContext(ctx);
@@ -1007,6 +1015,121 @@ BOOL IOCPServer::StartTlsHandshake(CONTEXT_OBJECT* ContextObject)
return TRUE;
}
// 握手线程参数:把触发判定的首批明文("GET ...")与连接上下文一起移交。
struct WsHandshakeParam {
IOCPServer* server;
CONTEXT_OBJECT* ctx;
std::vector<BYTE> initial;
};
// 与 TlsHandshakeThreadProc 同一模式(方案 A):升级期间用一个阻塞线程独占 socket,
// 完成后把连接交还 IOCP 接收(PostRecv)。线程持有 ctx 的一枚 IoRefCount,结束时释放。
DWORD WINAPI IOCPServer::WsHandshakeThreadProc(LPVOID lParam)
{
WsHandshakeParam* p = (WsHandshakeParam*)lParam;
IOCPServer* This = p->server;
CONTEXT_OBJECT* ctx = p->ctx;
std::vector<BYTE> initial = std::move(p->initial);
delete p;
if (ctx->IsRemoved.load(std::memory_order_acquire)) {
ctx->IoRefCount.fetch_sub(1);
return 0;
}
// 与 TLS 握手线程相同的 socket 准备工作:尽量切回阻塞,并加超时,避免慢速对端拖住线程。
u_long nb = 0;
if (ioctlsocket(ctx->sClientSocket, FIONBIO, &nb) == SOCKET_ERROR)
Mprintf("[WS] handshake: ioctlsocket(FIONBIO=0) failed err=%d\n", WSAGetLastError());
int tmo = 15000; // 15s 升级超时
setsockopt(ctx->sClientSocket, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tmo, sizeof(tmo));
setsockopt(ctx->sClientSocket, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tmo, sizeof(tmo));
// wss = TLS 里的 WebSocket:TLS 会话(若有)在这里只做字节加解密,升级请求与帧都是明文。
bool tlsReady = (ctx->tlsState.load(std::memory_order_acquire) == TLS_Ready);
std::vector<BYTE> pending;
std::string err;
bool ok = (ctx->ws != NULL) &&
ctx->ws->Handshake(ctx->sClientSocket, tlsReady ? ctx->tls : NULL, initial,
pending, ctx->GetPeerName(), &err);
if (!ok) {
Mprintf("[WS] upgrade failed for %s: %s (fail-closed)\n",
ctx->GetPeerName().c_str(), err.c_str());
}
bool wsTookOver = false;
if (ok) {
// 反代同机部署时,直接对端是回环地址,升级请求里的转发头才可信;据此把对端地址
// 改写成真实客户端 IP,使在线列表 / 封禁 / 白名单看到的是设备而不是反代。
const std::string& fwd = ctx->ws->ForwardedIP();
if (!fwd.empty()) {
Mprintf("[WS] %s upgraded via reverse proxy, real client %s\n",
ctx->GetPeerName().c_str(), fwd.c_str());
ctx->SetPeerName(fwd);
}
ctx->wsState.store(WS_Ready, std::memory_order_release);
// 升级响应落盘后,同批字节里可能已经有客户端的第一帧(WSS 分批解密时常见)。
// 必须在交还 IOCP 之前喂给解帧路径,否则要等客户端再发一包——而它正在等我们。
if (!pending.empty()) {
ZSTD_DCtx* dctx = ZSTD_createDCtx();
z_stream z = {};
inflateInit2(&z, 15);
int r = ctx->IsRemoved.load(std::memory_order_acquire) ? 1
: This->DispatchPlain(ctx, pending.data(), (DWORD)pending.size(), dctx, &z);
inflateEnd(&z);
ZSTD_freeDCtx(dctx);
if (r < 0) {
ok = false;
Mprintf("[WS] first frame dispatch failed for %s (closing)\n", ctx->GetPeerName().c_str());
}
}
} else {
ctx->wsState.store(WS_None, std::memory_order_release);
}
ctx->IoRefCount.fetch_sub(1);
if (ok) {
This->PostRecv(ctx); // 交还 IOCP 接收
} else {
This->RemoveStaleContext(ctx);
}
return 0;
}
// 判定为 WS 升级请求后:CAS 认领状态、创建会话、预支一枚 IoRefCount、启动升级线程。
BOOL IOCPServer::StartWsHandshake(CONTEXT_OBJECT* ContextObject, const std::vector<BYTE>& initial)
{
int expected = WS_Undecided;
if (!ContextObject->wsState.compare_exchange_strong(expected, WS_Handshaking,
std::memory_order_acq_rel)) {
return FALSE; // 已有别的线程认领(或状态机异常),不做二次升级
}
if (ContextObject->ws == NULL) {
ContextObject->ws = new ServerWsSession();
}
WsHandshakeParam* p = new WsHandshakeParam;
p->server = this;
p->ctx = ContextObject;
p->initial = initial;
// 预支引用:升级线程结束时释放。此引用保证升级期间对象不被回收。
ContextObject->IoRefCount.fetch_add(1);
HANDLE h = CreateThread(NULL, 0, WsHandshakeThreadProc, p, 0, NULL);
if (h == NULL) {
ContextObject->IoRefCount.fetch_sub(1);
ContextObject->FreeWs();
ContextObject->wsState.store(WS_None, std::memory_order_release);
return FALSE;
}
CloseHandle(h);
return TRUE;
}
BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans, ZSTD_DCtx* ctx, z_stream* z)
{
int st = ContextObject->tlsState.load(std::memory_order_acquire);
@@ -1040,15 +1163,21 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
}
// 明文:把嗅探阶段攒下的字节回放给现有解析路径,然后清空缓冲。
// 首包若以 "GET " 开头则由 DispatchPlain 判定为 WebSocket 升级。
ContextObject->tlsState.store(TLS_Plain, std::memory_order_release);
std::vector<BYTE> replay = std::move(ContextObject->tlsPreClassify);
ContextObject->tlsPreClassify.clear();
if (FALSE == ParseReceivedData(ContextObject, replay.data(), (DWORD)replay.size(), m_NotifyProc, ctx, z)) {
ContextObject->IoRefCount.fetch_sub(1);
RemoveStaleContext(ContextObject);
return -2;
{
int r = DispatchPlain(ContextObject, replay.data(), (DWORD)replay.size(), ctx, z);
if (r < 0) {
ContextObject->IoRefCount.fetch_sub(1);
RemoveStaleContext(ContextObject);
return -2;
}
if (r > 0) {
PostRecv(ContextObject);
}
}
PostRecv(ContextObject);
return TRUE;
}
@@ -1069,17 +1198,20 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
PostRecv(ContextObject); // 本轮无完整明文(TLS 记录分片/握手告警),继续收
return TRUE;
}
if (FALSE == ParseReceivedData(ContextObject, plain.data(), (DWORD)plain.size(), m_NotifyProc, ctx, z)) {
int r = DispatchPlain(ContextObject, plain.data(), (DWORD)plain.size(), ctx, z);
if (r < 0) {
ContextObject->IoRefCount.fetch_sub(1);
RemoveStaleContext(ContextObject);
return -2;
}
PostRecv(ContextObject);
if (r > 0) {
PostRecv(ContextObject);
}
return TRUE;
}
// TLS_Handshaking:握手线程独占 socket,正常不应出现 IORead;若出现属异常,fail-closed。
if (st == TLS_Handshaking) {
// TLS_Handshaking / WS_Handshaking:握手线程独占 socket,正常不应出现 IORead;若出现属异常,fail-closed。
if (st == TLS_Handshaking || ContextObject->wsState.load(std::memory_order_acquire) == WS_Handshaking) {
Mprintf("[TLS] unexpected IORead during handshake for %s (closing)\n",
ContextObject->GetPeerName().c_str());
ContextObject->IoRefCount.fetch_sub(1);
@@ -1087,13 +1219,115 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
return -2;
}
// TLS_Plain:明文直通(原路径)。
if (FALSE == ParseReceivedData(ContextObject, (PBYTE)ContextObject->szBuffer, dwTrans, m_NotifyProc, ctx, z)) {
ContextObject->IoRefCount.fetch_sub(1);
RemoveStaleContext(ContextObject);
return -2;
// TLS_Plain:明文直通(原路径)。WS 连接在这里先解帧。
{
int r = DispatchPlain(ContextObject, (const BYTE*)ContextObject->szBuffer, dwTrans, ctx, z);
if (r < 0) {
ContextObject->IoRefCount.fetch_sub(1);
RemoveStaleContext(ContextObject);
return -2;
}
if (r > 0) {
PostRecv(ContextObject);
}
return TRUE;
}
}
// 分发一段应用层字节(明文,或 TLS 解密后的明文)。
// 返回 1 = 已处理完,调用方可以 PostRecv 继续收;0 = WS 升级线程已接管 socket,不要 PostRecv;
// -1 = 出错,调用方关闭连接。
//
// WS_Undecided 阶段先攒够 6 字节再判定(与 ClientHello 嗅探同样的下限):首个明文分片
// 可能只有请求头的一部分,按前缀能不能判死都不安全。
int IOCPServer::DispatchPlain(CONTEXT_OBJECT* ContextObject, const BYTE* data, DWORD len,
ZSTD_DCtx* ctx, z_stream* z)
{
int ws = ContextObject->wsState.load(std::memory_order_acquire);
if (ws == WS_Ready) {
return DispatchWsFrameData(ContextObject, data, len, ctx, z) ? 1 : -1;
}
if (len == 0) {
return 1;
}
if (ws == WS_Undecided) {
ContextObject->tlsPreClassify.insert(ContextObject->tlsPreClassify.end(), data, data + len);
if (ContextObject->tlsPreClassify.size() < 6) {
return 1; // 头还没攒够,继续收
}
std::vector<BYTE> first;
first.swap(ContextObject->tlsPreClassify);
ContextObject->tlsPreClassify.clear();
if (first.size() >= 4 && memcmp(first.data(), "GET ", 4) == 0) {
if (!StartWsHandshake(ContextObject, first)) {
Mprintf("[WS] StartWsHandshake failed for %s (fail-closed)\n",
ContextObject->GetPeerName().c_str());
return -1;
}
return 0; // 升级线程接管 socket
}
ContextObject->wsState.store(WS_None, std::memory_order_release);
return ParseReceivedData(ContextObject, first.data(), (DWORD)first.size(), m_NotifyProc, ctx, z) ? 1 : -1;
}
return ParseReceivedData(ContextObject, (PBYTE)data, len, m_NotifyProc, ctx, z) ? 1 : -1;
}
// 把 WS 帧载荷交给解析路径,同时写回控制帧应答(pong / close 回显)。
BOOL IOCPServer::DispatchWsFrameData(CONTEXT_OBJECT* ContextObject, const BYTE* wire, DWORD len,
ZSTD_DCtx* ctx, z_stream* z)
{
if (ContextObject->ws == NULL) {
return FALSE;
}
std::vector<BYTE> app, replies;
int r = ContextObject->ws->Feed(wire, (int)len, app, replies);
if (!replies.empty() && !SendWsReplies(ContextObject, replies)) {
return FALSE;
}
if (r < 0) {
return FALSE; // 协议错误或对端 close
}
if (app.empty()) {
return TRUE;
}
return ParseReceivedData(ContextObject, app.data(), (DWORD)app.size(), m_NotifyProc, ctx, z);
}
// 写回一段完整的 WS 帧(与 Send2Client 走同一条发送路径:追加到 OutCompressedBuffer,
// TLS 连接只加密本次新追加的尾部,最后按 bSendPending 协议投递 IOWrite)。
BOOL IOCPServer::SendWsReplies(CONTEXT_OBJECT* ContextObject, const std::vector<BYTE>& replies)
{
CAutoCLock L(ContextObject->SendLock);
bool tlsReady = (ContextObject->tlsState.load(std::memory_order_acquire) == TLS_Ready);
ULONG oldLen = ContextObject->OutCompressedBuffer.GetBufferLength();
if (!ContextObject->OutCompressedBuffer.WriteBuffer((PBYTE)replies.data(), (DWORD)replies.size())) {
return FALSE;
}
if (tlsReady) {
ULONG newLen = ContextObject->OutCompressedBuffer.GetBufferLength() - oldLen;
std::vector<BYTE> wire;
if (ContextObject->tls->Encrypt(ContextObject->OutCompressedBuffer.GetBuffer(oldLen),
(int)newLen, wire) != (int)newLen) {
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
Mprintf("[TLS] Encrypt failed for %s (fail-closed)\n", ContextObject->GetPeerName().c_str());
return FALSE;
}
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
ContextObject->OutCompressedBuffer.WriteBuffer(wire.data(), (DWORD)wire.size());
}
if (!ContextObject->bSendPending) {
ContextObject->bSendPending = true;
OVERLAPPEDPLUS* OverlappedPlus = new OVERLAPPEDPLUS(IOWrite);
BOOL bOk = PostQueuedCompletionStatus(m_hCompletionPort, 0, (ULONG_PTR)ContextObject, &OverlappedPlus->m_ol);
if ((!bOk && GetLastError() != ERROR_IO_PENDING)) {
ContextObject->bSendPending = false;
SAFE_DELETE(OverlappedPlus);
return FALSE;
}
}
PostRecv(ContextObject);
return TRUE;
}
@@ -1164,9 +1398,30 @@ BOOL IOCPServer::OnClientPreSending(CONTEXT_OBJECT* ContextObject, PBYTE szBuffe
// 新明文由 WriteContextData 追加在尾部。若把整个缓冲区重新加密,会把旧密文二次加密,
// 客户端解密后得到的是「旧 TLS 记录」→ Unknown header 17 03 03。故只加密本次新追加的尾部。
bool tlsReady = (ContextObject->tlsState.load(std::memory_order_acquire) == TLS_Ready);
ULONG oldLen = tlsReady ? ContextObject->OutCompressedBuffer.GetBufferLength() : 0;
// WS(含 WSS):同样只处理本次新追加的尾部——前缀可能还有在途未发完的帧/密文。
bool wsReady = (ContextObject->wsState.load(std::memory_order_acquire) == WS_Ready) &&
ContextObject->ws != NULL && ContextObject->ws->IsOpen();
ULONG oldLen = (tlsReady || wsReady) ? ContextObject->OutCompressedBuffer.GetBufferLength() : 0;
if (WriteContextData(ContextObject, szBuffer, ulOriginalLength, ContextObject->Zcctx)) {
// WebSocket:一次发送 = 一帧(与 Go 端 wsConn.Write 对齐)。必须在 TLS 加密之前完成,
// 这样 TLS 加密的「新追加尾部」就是整帧字节(应用字节 → WS 帧 → TLS 记录)。
if (wsReady) {
ULONG totalLen = ContextObject->OutCompressedBuffer.GetBufferLength();
ULONG appLen = totalLen - oldLen;
if (appLen > 0 && ContextObject->ws->IsOpen()) {
std::vector<BYTE> framed;
if (!ContextObject->ws->Frame((const BYTE*)ContextObject->OutCompressedBuffer.GetBuffer(oldLen),
(int)appLen, framed)) {
Mprintf("[WS] frame failed for %s (dropping send)\n", ContextObject->GetPeerName().c_str());
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
return FALSE;
}
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
ContextObject->OutCompressedBuffer.WriteBuffer(framed.data(), (DWORD)framed.size());
}
}
// 单向 TLS:只加密新追加的明文尾部,产出密文后与前缀旧密文拼接,使缓冲区始终保持
// 一串合法 TLS 记录;之后 IOWrite / OnClientPostSending 完全不变(见设计文档 §5.5)。
if (tlsReady) {