Feature: Sniff WebSocket upgrades on the MFC server's device port
This commit is contained in:
1 parent
d4f312c9d8
commit
cc891e3670
11 files changed
+925
-32
No files matched your search
@@ -4,6 +4,7 @@
|
||||
#include "common/IPWhitelist.h"
|
||||
#include "common/IPBlacklist.h"
|
||||
#include "ServerTlsSession.h"
|
||||
#include "ServerWsSession.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <ws2tcpip.h>
|
||||
@@ -947,26 +948,33 @@ DWORD WINAPI IOCPServer::TlsHandshakeThreadProc(LPVOID lParam)
|
||||
|
||||
// 注意顺序:解析仍在本线程持有引用期间完成(对象不会在这段窗口里被回收);
|
||||
// 释放引用之后才允许任何可能 RemoveStaleContext 的动作(#215)。
|
||||
bool wsTookOver = false;
|
||||
if (ok && !pendingPlain.empty()) {
|
||||
// 与工作线程同样的解压上下文,只是现建现用(首包只解析这一批)。
|
||||
// 走 DispatchPlain:WSS 的首个明文就是 "GET /api/ws HTTP/1.1...",
|
||||
// 由它判定升级并交给 WS 升级线程(此时本线程不能再 PostRecv)。
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
z_stream z = {};
|
||||
inflateInit2(&z, 15);
|
||||
ok = ctx->IsRemoved.load(std::memory_order_acquire) ? TRUE
|
||||
: ParseReceivedData(ctx, pendingPlain.data(), (DWORD)pendingPlain.size(),
|
||||
This->m_NotifyProc, dctx, &z);
|
||||
int r = ctx->IsRemoved.load(std::memory_order_acquire) ? 1
|
||||
: This->DispatchPlain(ctx, pendingPlain.data(), (DWORD)pendingPlain.size(), dctx, &z);
|
||||
inflateEnd(&z);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
if (!ok) {
|
||||
if (r < 0) {
|
||||
ok = false;
|
||||
Mprintf("[TLS] pending first packet parse failed for %s (closing)\n",
|
||||
ctx->GetPeerName().c_str());
|
||||
} else if (r == 0) {
|
||||
wsTookOver = true;
|
||||
}
|
||||
}
|
||||
|
||||
ctx->IoRefCount.fetch_sub(1);
|
||||
|
||||
if (ok) {
|
||||
This->PostRecv(ctx); // 交还 IOCP 接收
|
||||
if (!wsTookOver) {
|
||||
This->PostRecv(ctx); // 交还 IOCP 接收
|
||||
}
|
||||
} else {
|
||||
Mprintf("[TLS] handshake failed for %s (fail-closed)\n", ctx->GetPeerName().c_str());
|
||||
This->RemoveStaleContext(ctx);
|
||||
@@ -1007,6 +1015,121 @@ BOOL IOCPServer::StartTlsHandshake(CONTEXT_OBJECT* ContextObject)
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
// 握手线程参数:把触发判定的首批明文("GET ...")与连接上下文一起移交。
|
||||
struct WsHandshakeParam {
|
||||
IOCPServer* server;
|
||||
CONTEXT_OBJECT* ctx;
|
||||
std::vector<BYTE> initial;
|
||||
};
|
||||
|
||||
// 与 TlsHandshakeThreadProc 同一模式(方案 A):升级期间用一个阻塞线程独占 socket,
|
||||
// 完成后把连接交还 IOCP 接收(PostRecv)。线程持有 ctx 的一枚 IoRefCount,结束时释放。
|
||||
DWORD WINAPI IOCPServer::WsHandshakeThreadProc(LPVOID lParam)
|
||||
{
|
||||
WsHandshakeParam* p = (WsHandshakeParam*)lParam;
|
||||
IOCPServer* This = p->server;
|
||||
CONTEXT_OBJECT* ctx = p->ctx;
|
||||
std::vector<BYTE> initial = std::move(p->initial);
|
||||
delete p;
|
||||
|
||||
if (ctx->IsRemoved.load(std::memory_order_acquire)) {
|
||||
ctx->IoRefCount.fetch_sub(1);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 与 TLS 握手线程相同的 socket 准备工作:尽量切回阻塞,并加超时,避免慢速对端拖住线程。
|
||||
u_long nb = 0;
|
||||
if (ioctlsocket(ctx->sClientSocket, FIONBIO, &nb) == SOCKET_ERROR)
|
||||
Mprintf("[WS] handshake: ioctlsocket(FIONBIO=0) failed err=%d\n", WSAGetLastError());
|
||||
int tmo = 15000; // 15s 升级超时
|
||||
setsockopt(ctx->sClientSocket, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tmo, sizeof(tmo));
|
||||
setsockopt(ctx->sClientSocket, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tmo, sizeof(tmo));
|
||||
|
||||
// wss = TLS 里的 WebSocket:TLS 会话(若有)在这里只做字节加解密,升级请求与帧都是明文。
|
||||
bool tlsReady = (ctx->tlsState.load(std::memory_order_acquire) == TLS_Ready);
|
||||
std::vector<BYTE> pending;
|
||||
std::string err;
|
||||
bool ok = (ctx->ws != NULL) &&
|
||||
ctx->ws->Handshake(ctx->sClientSocket, tlsReady ? ctx->tls : NULL, initial,
|
||||
pending, ctx->GetPeerName(), &err);
|
||||
if (!ok) {
|
||||
Mprintf("[WS] upgrade failed for %s: %s (fail-closed)\n",
|
||||
ctx->GetPeerName().c_str(), err.c_str());
|
||||
}
|
||||
|
||||
bool wsTookOver = false;
|
||||
if (ok) {
|
||||
// 反代同机部署时,直接对端是回环地址,升级请求里的转发头才可信;据此把对端地址
|
||||
// 改写成真实客户端 IP,使在线列表 / 封禁 / 白名单看到的是设备而不是反代。
|
||||
const std::string& fwd = ctx->ws->ForwardedIP();
|
||||
if (!fwd.empty()) {
|
||||
Mprintf("[WS] %s upgraded via reverse proxy, real client %s\n",
|
||||
ctx->GetPeerName().c_str(), fwd.c_str());
|
||||
ctx->SetPeerName(fwd);
|
||||
}
|
||||
ctx->wsState.store(WS_Ready, std::memory_order_release);
|
||||
|
||||
// 升级响应落盘后,同批字节里可能已经有客户端的第一帧(WSS 分批解密时常见)。
|
||||
// 必须在交还 IOCP 之前喂给解帧路径,否则要等客户端再发一包——而它正在等我们。
|
||||
if (!pending.empty()) {
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
z_stream z = {};
|
||||
inflateInit2(&z, 15);
|
||||
int r = ctx->IsRemoved.load(std::memory_order_acquire) ? 1
|
||||
: This->DispatchPlain(ctx, pending.data(), (DWORD)pending.size(), dctx, &z);
|
||||
inflateEnd(&z);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
if (r < 0) {
|
||||
ok = false;
|
||||
Mprintf("[WS] first frame dispatch failed for %s (closing)\n", ctx->GetPeerName().c_str());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctx->wsState.store(WS_None, std::memory_order_release);
|
||||
}
|
||||
|
||||
ctx->IoRefCount.fetch_sub(1);
|
||||
|
||||
if (ok) {
|
||||
This->PostRecv(ctx); // 交还 IOCP 接收
|
||||
} else {
|
||||
This->RemoveStaleContext(ctx);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 判定为 WS 升级请求后:CAS 认领状态、创建会话、预支一枚 IoRefCount、启动升级线程。
|
||||
BOOL IOCPServer::StartWsHandshake(CONTEXT_OBJECT* ContextObject, const std::vector<BYTE>& initial)
|
||||
{
|
||||
int expected = WS_Undecided;
|
||||
if (!ContextObject->wsState.compare_exchange_strong(expected, WS_Handshaking,
|
||||
std::memory_order_acq_rel)) {
|
||||
return FALSE; // 已有别的线程认领(或状态机异常),不做二次升级
|
||||
}
|
||||
|
||||
if (ContextObject->ws == NULL) {
|
||||
ContextObject->ws = new ServerWsSession();
|
||||
}
|
||||
|
||||
WsHandshakeParam* p = new WsHandshakeParam;
|
||||
p->server = this;
|
||||
p->ctx = ContextObject;
|
||||
p->initial = initial;
|
||||
|
||||
// 预支引用:升级线程结束时释放。此引用保证升级期间对象不被回收。
|
||||
ContextObject->IoRefCount.fetch_add(1);
|
||||
|
||||
HANDLE h = CreateThread(NULL, 0, WsHandshakeThreadProc, p, 0, NULL);
|
||||
if (h == NULL) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
ContextObject->FreeWs();
|
||||
ContextObject->wsState.store(WS_None, std::memory_order_release);
|
||||
return FALSE;
|
||||
}
|
||||
CloseHandle(h);
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans, ZSTD_DCtx* ctx, z_stream* z)
|
||||
{
|
||||
int st = ContextObject->tlsState.load(std::memory_order_acquire);
|
||||
@@ -1040,15 +1163,21 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
|
||||
}
|
||||
|
||||
// 明文:把嗅探阶段攒下的字节回放给现有解析路径,然后清空缓冲。
|
||||
// 首包若以 "GET " 开头则由 DispatchPlain 判定为 WebSocket 升级。
|
||||
ContextObject->tlsState.store(TLS_Plain, std::memory_order_release);
|
||||
std::vector<BYTE> replay = std::move(ContextObject->tlsPreClassify);
|
||||
ContextObject->tlsPreClassify.clear();
|
||||
if (FALSE == ParseReceivedData(ContextObject, replay.data(), (DWORD)replay.size(), m_NotifyProc, ctx, z)) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
RemoveStaleContext(ContextObject);
|
||||
return -2;
|
||||
{
|
||||
int r = DispatchPlain(ContextObject, replay.data(), (DWORD)replay.size(), ctx, z);
|
||||
if (r < 0) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
RemoveStaleContext(ContextObject);
|
||||
return -2;
|
||||
}
|
||||
if (r > 0) {
|
||||
PostRecv(ContextObject);
|
||||
}
|
||||
}
|
||||
PostRecv(ContextObject);
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
@@ -1069,17 +1198,20 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
|
||||
PostRecv(ContextObject); // 本轮无完整明文(TLS 记录分片/握手告警),继续收
|
||||
return TRUE;
|
||||
}
|
||||
if (FALSE == ParseReceivedData(ContextObject, plain.data(), (DWORD)plain.size(), m_NotifyProc, ctx, z)) {
|
||||
int r = DispatchPlain(ContextObject, plain.data(), (DWORD)plain.size(), ctx, z);
|
||||
if (r < 0) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
RemoveStaleContext(ContextObject);
|
||||
return -2;
|
||||
}
|
||||
PostRecv(ContextObject);
|
||||
if (r > 0) {
|
||||
PostRecv(ContextObject);
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
// TLS_Handshaking:握手线程独占 socket,正常不应出现 IORead;若出现属异常,fail-closed。
|
||||
if (st == TLS_Handshaking) {
|
||||
// TLS_Handshaking / WS_Handshaking:握手线程独占 socket,正常不应出现 IORead;若出现属异常,fail-closed。
|
||||
if (st == TLS_Handshaking || ContextObject->wsState.load(std::memory_order_acquire) == WS_Handshaking) {
|
||||
Mprintf("[TLS] unexpected IORead during handshake for %s (closing)\n",
|
||||
ContextObject->GetPeerName().c_str());
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
@@ -1087,13 +1219,115 @@ BOOL IOCPServer::OnClientReceiving(PCONTEXT_OBJECT ContextObject, DWORD dwTrans
|
||||
return -2;
|
||||
}
|
||||
|
||||
// TLS_Plain:明文直通(原路径)。
|
||||
if (FALSE == ParseReceivedData(ContextObject, (PBYTE)ContextObject->szBuffer, dwTrans, m_NotifyProc, ctx, z)) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
RemoveStaleContext(ContextObject);
|
||||
return -2;
|
||||
// TLS_Plain:明文直通(原路径)。WS 连接在这里先解帧。
|
||||
{
|
||||
int r = DispatchPlain(ContextObject, (const BYTE*)ContextObject->szBuffer, dwTrans, ctx, z);
|
||||
if (r < 0) {
|
||||
ContextObject->IoRefCount.fetch_sub(1);
|
||||
RemoveStaleContext(ContextObject);
|
||||
return -2;
|
||||
}
|
||||
if (r > 0) {
|
||||
PostRecv(ContextObject);
|
||||
}
|
||||
return TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
// 分发一段应用层字节(明文,或 TLS 解密后的明文)。
|
||||
// 返回 1 = 已处理完,调用方可以 PostRecv 继续收;0 = WS 升级线程已接管 socket,不要 PostRecv;
|
||||
// -1 = 出错,调用方关闭连接。
|
||||
//
|
||||
// WS_Undecided 阶段先攒够 6 字节再判定(与 ClientHello 嗅探同样的下限):首个明文分片
|
||||
// 可能只有请求头的一部分,按前缀能不能判死都不安全。
|
||||
int IOCPServer::DispatchPlain(CONTEXT_OBJECT* ContextObject, const BYTE* data, DWORD len,
|
||||
ZSTD_DCtx* ctx, z_stream* z)
|
||||
{
|
||||
int ws = ContextObject->wsState.load(std::memory_order_acquire);
|
||||
if (ws == WS_Ready) {
|
||||
return DispatchWsFrameData(ContextObject, data, len, ctx, z) ? 1 : -1;
|
||||
}
|
||||
if (len == 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (ws == WS_Undecided) {
|
||||
ContextObject->tlsPreClassify.insert(ContextObject->tlsPreClassify.end(), data, data + len);
|
||||
if (ContextObject->tlsPreClassify.size() < 6) {
|
||||
return 1; // 头还没攒够,继续收
|
||||
}
|
||||
std::vector<BYTE> first;
|
||||
first.swap(ContextObject->tlsPreClassify);
|
||||
ContextObject->tlsPreClassify.clear();
|
||||
if (first.size() >= 4 && memcmp(first.data(), "GET ", 4) == 0) {
|
||||
if (!StartWsHandshake(ContextObject, first)) {
|
||||
Mprintf("[WS] StartWsHandshake failed for %s (fail-closed)\n",
|
||||
ContextObject->GetPeerName().c_str());
|
||||
return -1;
|
||||
}
|
||||
return 0; // 升级线程接管 socket
|
||||
}
|
||||
ContextObject->wsState.store(WS_None, std::memory_order_release);
|
||||
return ParseReceivedData(ContextObject, first.data(), (DWORD)first.size(), m_NotifyProc, ctx, z) ? 1 : -1;
|
||||
}
|
||||
|
||||
return ParseReceivedData(ContextObject, (PBYTE)data, len, m_NotifyProc, ctx, z) ? 1 : -1;
|
||||
}
|
||||
|
||||
// 把 WS 帧载荷交给解析路径,同时写回控制帧应答(pong / close 回显)。
|
||||
BOOL IOCPServer::DispatchWsFrameData(CONTEXT_OBJECT* ContextObject, const BYTE* wire, DWORD len,
|
||||
ZSTD_DCtx* ctx, z_stream* z)
|
||||
{
|
||||
if (ContextObject->ws == NULL) {
|
||||
return FALSE;
|
||||
}
|
||||
std::vector<BYTE> app, replies;
|
||||
int r = ContextObject->ws->Feed(wire, (int)len, app, replies);
|
||||
if (!replies.empty() && !SendWsReplies(ContextObject, replies)) {
|
||||
return FALSE;
|
||||
}
|
||||
if (r < 0) {
|
||||
return FALSE; // 协议错误或对端 close
|
||||
}
|
||||
if (app.empty()) {
|
||||
return TRUE;
|
||||
}
|
||||
return ParseReceivedData(ContextObject, app.data(), (DWORD)app.size(), m_NotifyProc, ctx, z);
|
||||
}
|
||||
|
||||
// 写回一段完整的 WS 帧(与 Send2Client 走同一条发送路径:追加到 OutCompressedBuffer,
|
||||
// TLS 连接只加密本次新追加的尾部,最后按 bSendPending 协议投递 IOWrite)。
|
||||
BOOL IOCPServer::SendWsReplies(CONTEXT_OBJECT* ContextObject, const std::vector<BYTE>& replies)
|
||||
{
|
||||
CAutoCLock L(ContextObject->SendLock);
|
||||
bool tlsReady = (ContextObject->tlsState.load(std::memory_order_acquire) == TLS_Ready);
|
||||
ULONG oldLen = ContextObject->OutCompressedBuffer.GetBufferLength();
|
||||
if (!ContextObject->OutCompressedBuffer.WriteBuffer((PBYTE)replies.data(), (DWORD)replies.size())) {
|
||||
return FALSE;
|
||||
}
|
||||
if (tlsReady) {
|
||||
ULONG newLen = ContextObject->OutCompressedBuffer.GetBufferLength() - oldLen;
|
||||
std::vector<BYTE> wire;
|
||||
if (ContextObject->tls->Encrypt(ContextObject->OutCompressedBuffer.GetBuffer(oldLen),
|
||||
(int)newLen, wire) != (int)newLen) {
|
||||
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
|
||||
Mprintf("[TLS] Encrypt failed for %s (fail-closed)\n", ContextObject->GetPeerName().c_str());
|
||||
return FALSE;
|
||||
}
|
||||
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
|
||||
ContextObject->OutCompressedBuffer.WriteBuffer(wire.data(), (DWORD)wire.size());
|
||||
}
|
||||
|
||||
if (!ContextObject->bSendPending) {
|
||||
ContextObject->bSendPending = true;
|
||||
OVERLAPPEDPLUS* OverlappedPlus = new OVERLAPPEDPLUS(IOWrite);
|
||||
BOOL bOk = PostQueuedCompletionStatus(m_hCompletionPort, 0, (ULONG_PTR)ContextObject, &OverlappedPlus->m_ol);
|
||||
if ((!bOk && GetLastError() != ERROR_IO_PENDING)) {
|
||||
ContextObject->bSendPending = false;
|
||||
SAFE_DELETE(OverlappedPlus);
|
||||
return FALSE;
|
||||
}
|
||||
}
|
||||
PostRecv(ContextObject);
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
@@ -1164,9 +1398,30 @@ BOOL IOCPServer::OnClientPreSending(CONTEXT_OBJECT* ContextObject, PBYTE szBuffe
|
||||
// 新明文由 WriteContextData 追加在尾部。若把整个缓冲区重新加密,会把旧密文二次加密,
|
||||
// 客户端解密后得到的是「旧 TLS 记录」→ Unknown header 17 03 03。故只加密本次新追加的尾部。
|
||||
bool tlsReady = (ContextObject->tlsState.load(std::memory_order_acquire) == TLS_Ready);
|
||||
ULONG oldLen = tlsReady ? ContextObject->OutCompressedBuffer.GetBufferLength() : 0;
|
||||
// WS(含 WSS):同样只处理本次新追加的尾部——前缀可能还有在途未发完的帧/密文。
|
||||
bool wsReady = (ContextObject->wsState.load(std::memory_order_acquire) == WS_Ready) &&
|
||||
ContextObject->ws != NULL && ContextObject->ws->IsOpen();
|
||||
ULONG oldLen = (tlsReady || wsReady) ? ContextObject->OutCompressedBuffer.GetBufferLength() : 0;
|
||||
|
||||
if (WriteContextData(ContextObject, szBuffer, ulOriginalLength, ContextObject->Zcctx)) {
|
||||
// WebSocket:一次发送 = 一帧(与 Go 端 wsConn.Write 对齐)。必须在 TLS 加密之前完成,
|
||||
// 这样 TLS 加密的「新追加尾部」就是整帧字节(应用字节 → WS 帧 → TLS 记录)。
|
||||
if (wsReady) {
|
||||
ULONG totalLen = ContextObject->OutCompressedBuffer.GetBufferLength();
|
||||
ULONG appLen = totalLen - oldLen;
|
||||
if (appLen > 0 && ContextObject->ws->IsOpen()) {
|
||||
std::vector<BYTE> framed;
|
||||
if (!ContextObject->ws->Frame((const BYTE*)ContextObject->OutCompressedBuffer.GetBuffer(oldLen),
|
||||
(int)appLen, framed)) {
|
||||
Mprintf("[WS] frame failed for %s (dropping send)\n", ContextObject->GetPeerName().c_str());
|
||||
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
|
||||
return FALSE;
|
||||
}
|
||||
ContextObject->OutCompressedBuffer.TruncateBuffer(oldLen);
|
||||
ContextObject->OutCompressedBuffer.WriteBuffer(framed.data(), (DWORD)framed.size());
|
||||
}
|
||||
}
|
||||
|
||||
// 单向 TLS:只加密新追加的明文尾部,产出密文后与前缀旧密文拼接,使缓冲区始终保持
|
||||
// 一串合法 TLS 记录;之后 IOWrite / OnClientPostSending 完全不变(见设计文档 §5.5)。
|
||||
if (tlsReady) {
|
||||
|
||||
Reference in new issue
Block a user