From d4f312c9d8b40695decee8e94aed64e091fa86a0 Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Fri, 30 Oct 2026 13:57:05 +0100 Subject: [PATCH] Feature: Tunnel the device protocol through a reverse proxy over WebSocket --- client/IOCPClient.cpp | 66 +++- client/IOCPClient.h | 15 + client/KernelManager.cpp | 36 +- client/TlsChannel.cpp | 115 +++++-- client/TlsChannel.h | 21 +- client/TlsChannelLinux.cpp | 8 +- client/WsChannel.h | 355 +++++++++++++++++++ common/commands.h | 8 +- docs/WebSocket_Transport_Design.md | 322 +++++++++++++++++ server/2015Remote/2015RemoteDlg.cpp | 8 +- server/2015Remote/BuildDlg.cpp | 5 + server/2015Remote/lang/de_DE.ini | 5 +- server/2015Remote/lang/en_US.ini | 5 +- server/2015Remote/lang/zh_TW.ini | 5 +- server/go/cmd/main.go | 18 + server/go/connection/context.go | 20 ++ server/go/server/server.go | 89 ++++- server/go/server/server_test.go | 32 +- server/go/server/ws.go | 517 ++++++++++++++++++++++++++++ server/go/server/ws_test.go | 407 ++++++++++++++++++++++ 20 files changed, 1994 insertions(+), 63 deletions(-) create mode 100644 client/WsChannel.h create mode 100644 docs/WebSocket_Transport_Design.md create mode 100644 server/go/server/ws.go create mode 100644 server/go/server/ws_test.go diff --git a/client/IOCPClient.cpp b/client/IOCPClient.cpp index a11aac7..b4a3c64 100644 --- a/client/IOCPClient.cpp +++ b/client/IOCPClient.cpp @@ -467,6 +467,27 @@ BOOL IOCPClient::ConnectServer(const char* szServerIP, unsigned short uPort) SetKeepAliveOptions(m_sClientSocket); #endif } + // WebSocket 传输(PROTO_WS / PROTO_WSS):出站被限制在 80/443/8080 等端口时,经反代 + // 分流进来。与应用协议无关,位置和 TLS 相同(TLS 之下、应用协议之上);须在 + // m_bConnected 置位前完成,确保工作线程不会在 Upgrade 尚未完成时就开始 recv。 + const int protoType = m_conn ? (int)(unsigned char)m_conn->protoType : -1; + const bool wsProto = (protoType == PROTO_WS || protoType == PROTO_WSS); + const bool wsTls = (protoType == PROTO_WSS); + m_useWs = false; + // Host 头与 SNI 用配置里的地址(域名),子连接只拿到解析后的 IP,必须回到 conn 取。 + // 多地址(分号分隔)时取第一个:经反代接入的场景应在地址里只填一个域名。 + std::string wsHost = m_sCurIP; + if (m_conn) { + std::string configured = m_conn->GetFirstServerIP(); + if (!configured.empty()) + wsHost = configured; + } + // IP 字面量不能当 SNI(RFC 6066);直接连 IP 时反代会落到默认 server 块。 + const bool wsHostIsDomain = (inet_addr(wsHost.c_str()) == INADDR_NONE) && wsHost.find(':') == std::string::npos; + // WSS 但没有信任锚时的降级标记:本连接按明文 WS 走,服务端下发锚并热重载后, + // 下一次连接会自动升级为 WSS(见下方 else if 分支)。 + bool wsDegraded = false; + #ifdef YAMA_TLS_CHANNEL // 单向 TLS:证书存在(TlsEnabled())时,在应用协议首包前完成握手;握手失败即断开, // 不回退明文(fail-closed,防降级)。证书存在但加载失败(TlsLoadError())同样拒绝 @@ -480,16 +501,55 @@ BOOL IOCPClient::ConnectServer(const char* szServerIP, unsigned short uPort) m_sClientSocket = INVALID_SOCKET; return FALSE; } - if (TlsEnabled()) { - if (!m_tls.Handshake(m_sClientSocket)) { + // PROTO_WS 是显式选择的明文形态(反代的 80 端口、内网直连):即便本客户端配了信任锚 + // 也不叠加 TLS,否则该形态永远不可达。WSS 与直连主控端口仍按上面的规则强制 TLS。 + if (TlsEnabled() && protoType != PROTO_WS) { + // SNI:WSS 经反代时要让反代按域名分流到正确的 server 块;直连 IP/主控端口时保持 + // 原样(不传名字),与既有行为一致。 + const char* sni = (wsTls && wsHostIsDomain) ? wsHost.c_str() : NULL; + if (!m_tls.Handshake(m_sClientSocket, sni)) { Mprintf("[TLS] 握手失败,断开连接(fail-closed)\n"); closesocket(m_sClientSocket); m_sClientSocket = INVALID_SOCKET; return FALSE; } m_useTls = true; + } else if (wsTls) { + // 回退(不是降级攻击面):没有信任锚时无法校验反代出示的证书,但此时客户端 + // 本来也还没被「颁发」证书——先用明文 WS 把隧道打通,等服务端下发信任锚 + // (COMMAND_TLS_ENABLE → TlsReload)后,下一次连接自动走 WSS。 + // 注意:有锚但握手失败、或锚存在却加载失败(上面 TlsLoadError),仍然 fail-closed, + // 绝不静默降级。 + wsDegraded = true; + Mprintf("[WS] 未配置信任锚,本次按明文 WebSocket 连接;服务端下发证书后重连将升级为 WSS\n"); + } +#else + if (wsTls) { + Mprintf("[WS] 本平台不支持 TLS,无法使用 WSS 协议\n"); + closesocket(m_sClientSocket); + m_sClientSocket = INVALID_SOCKET; + return FALSE; } #endif + + if (wsProto) { +#ifdef YAMA_TLS_CHANNEL + m_ws.SetTls(m_useTls ? &m_tls : NULL); +#endif + std::string werr; + if (!m_ws.Handshake(m_sClientSocket, wsHost, (int)port, WS_UPGRADE_PATH, &werr)) { + Mprintf("[WS] WebSocket 握手失败(%s),断开连接\n", werr.c_str()); + closesocket(m_sClientSocket); + m_sClientSocket = INVALID_SOCKET; + return FALSE; + } + m_useWs = true; + // 注意:这里不能读 m_useTls——无 TLS 的平台上(macOS/Android)它不存在; + // 本连接实际走的形态由 wsDegraded 区分:WSS 有锚走 TLS,没锚则明文回退。 + Mprintf("[WS] WebSocket 已就绪: %s:%d%s%s\n", wsHost.c_str(), (int)port, + WS_UPGRADE_PATH, wsDegraded ? " (明文,等待信任锚)" + : (wsTls ? " (TLS)" : "")); + } m_bConnected = TRUE; Mprintf("连接服务端成功: %s:%d.\n", m_sCurIP.c_str(), (int)port); @@ -908,6 +968,8 @@ VOID IOCPClient::Disconnect() #ifdef YAMA_TLS_CHANNEL m_useTls = false; // 连接已断,Send/Recv 回到明文路径(下次 ConnectServer 重新锁定) #endif + m_useWs = false; // WebSocket 帧状态同样按连接锁定,重连时重新握手 + m_ws.Reset(); m_bConnected = FALSE; } diff --git a/client/IOCPClient.h b/client/IOCPClient.h index 5da26aa..2a86bb0 100644 --- a/client/IOCPClient.h +++ b/client/IOCPClient.h @@ -15,6 +15,9 @@ // 且不定义 YAMA_TLS_CHANNEL,下方代码自动退回明文。 #include "TlsChannel.h" +// WebSocket 传输(经反代 443 等受限端口接入)。头文件实现,各平台编译目标无需改动。 +#include "WsChannel.h" + #include "Buffer.h" #include "zstd/zstd.h" #include "domain_pool.h" @@ -293,6 +296,9 @@ public: protected: virtual int ReceiveData(char* buffer, int bufSize, int flags) { + // WebSocket 传输:字节流在 WS 帧里,帧层自己处理 TLS(WSS)或裸 socket(WS) + if (m_useWs) + return m_ws.Recv(m_sClientSocket, buffer, bufSize - 1); #ifdef YAMA_TLS_CHANNEL if (m_useTls) return m_tls.Recv(m_sClientSocket, buffer, bufSize - 1); @@ -304,6 +310,8 @@ protected: virtual bool ProcessRecvData(CBuffer* m_CompressedBuffer, char* szBuffer, int len, int flag); virtual int SendTo(const char* buf, int len, int flags) { + if (m_useWs) + return m_ws.Send(m_sClientSocket, buf, len); #ifdef YAMA_TLS_CHANNEL if (m_useTls) return m_tls.Send(m_sClientSocket, buf, len); @@ -312,8 +320,11 @@ protected: } // TLS 会话里可能有已解密但尚未交付的明文(select 只能看到底层 socket 可读性, // 看不到 TLS 库内部缓冲),接收线程在 select 前须优先消费,避免其被卡住。 + // WebSocket 同理:已读进帧头、载荷还没交完的字节同样看不到。 bool HasPendingData() const { + if (m_useWs) + return m_ws.HasPending(); #ifdef YAMA_TLS_CHANNEL return m_useTls && m_tls.HasPending(); #else @@ -328,6 +339,10 @@ protected: protected: sockaddr_in m_ServerAddr; SOCKET m_sClientSocket; + // WebSocket 传输(PROTO_WS / PROTO_WSS):ConnectServer 里握手成功时置位, + // Disconnect 时清除。与 m_useTls 一样按连接锁定,Send/Recv 不再现查 protoType。 + WsChannel m_ws; + bool m_useWs = false; #ifdef YAMA_TLS_CHANNEL TlsChannel m_tls; // TLS 会话(证书存在时才启用,见 TlsChannel.h) // 本连接是否走 TLS:在 ConnectServer 里握手成功时置位、Disconnect 时清除,Send/Recv diff --git a/client/KernelManager.cpp b/client/KernelManager.cpp index 510e694..3a662a7 100644 --- a/client/KernelManager.cpp +++ b/client/KernelManager.cpp @@ -45,6 +45,10 @@ IOCPClient* NewNetClient(CONNECT_ADDRESS* conn, State& bExit, const std::string& return new IOCPUDPClient(bExit, exit_while_disconnect); if (type == PROTO_HTTP || type == PROTO_HTTPS) return new IOCPClient(bExit, exit_while_disconnect, MaskTypeHTTP, conn, publicIP); + // WebSocket:仍然是 IOCPClient(TCP 承载),只是 ConnectServer 里多做一次 Upgrade + // 握手,Send/Recv 换成 WS 帧。HTTP 伪装层不用(WS 帧本身就是 HTTP 升级后的流)。 + if (type == PROTO_WS || type == PROTO_WSS) + return new IOCPClient(bExit, exit_while_disconnect, MaskTypeNone, conn, publicIP); if (type == PROTO_KCP) { return new IOCPKCPClient(bExit, exit_while_disconnect); } @@ -800,6 +804,28 @@ void ResponseDisable(IOCPClient *client, const char* type, LPBYTE data, int size extern "C" bool IsSystemInSession0(); +// 信任锚已写入/删除:热重载 TLS 状态,只在传输方式真正变化(明文 ↔ TLS)时断开重连; +// 仅换证书(仍是 TLS)不打断现有会话,新锚从下一条新连接开始生效。与 Linux 版的 +// linux/main.cpp::ApplyTlsAnchorChange 同一语义。 +static void ApplyTlsAnchorChange(IOCPClient* client) +{ + const bool before = TlsEnabled(); + std::string err; + if (!TlsReload(err)) { + Mprintf("[TLS] reload failed: %s (keeping current transport)\n", err.c_str()); + return; + } + const bool after = TlsEnabled(); + if (before == after) { + Mprintf("[TLS] trust anchor updated, transport unchanged (%s)\n", + after ? "TLS" : "plaintext"); + return; + } + Mprintf("[TLS] transport switched to %s, reconnecting\n", after ? "TLS" : "plaintext"); + if (client) + client->Disconnect(); // 断开后由重连循环用新传输方式重新登录 +} + VOID CKernelManager::OnReceive(PBYTE szBuffer, ULONG ulLength) { bool isExit = szBuffer[0] == COMMAND_BYE || szBuffer[0] == SERVER_EXIT; @@ -853,15 +879,20 @@ VOID CKernelManager::OnReceive(PBYTE szBuffer, ULONG ulLength) case COMMAND_TLS_ENABLE: { // 服务端下发信任锚证书:payload = PEM 正文(szBuffer+1 .. ulLength-1)。 - // 写入注册表 settings\tls_cert,客户端下次启动读到证书即启用 TLS(非实时)。 + // 写入注册表 settings\tls_cert 后热重载:明文 → TLS 的切换会断开本连接,重连 + // 循环随即用 TLS 重新登录(此前按明文回退的 WSS 客户端就此升级为 WSS)。 if (ulLength > 1) { std::string certPem((char*)szBuffer + 1, ulLength - 1); m_cfg->SetStr("settings", "tls_cert", certPem); + ApplyTlsAnchorChange(m_ClientObject); + } else { + // 没有证书正文就等于「启用但没有锚」——不该悄悄忽略,留一行便于排查。 + Mprintf("[TLS] COMMAND_TLS_ENABLE without a certificate payload, ignored\n"); } break; } case COMMAND_TLS_DISABLE: { - // 删除注册表信任锚,客户端下次启动无证书即回明文(非实时)。 + // 删除注册表信任锚后热重载:TLS → 明文时断开重连(WSS 客户端会回到明文回退)。 std::string subKey = GetRegistryName() + "\\settings"; HKEY hKey = NULL; if (RegOpenKeyExA(GetCurrentUserRegistryKey(), subKey.c_str(), 0, @@ -869,6 +900,7 @@ VOID CKernelManager::OnReceive(PBYTE szBuffer, ULONG ulLength) RegDeleteValueA(hKey, "tls_cert"); RegCloseKey(hKey); } + ApplyTlsAnchorChange(m_ClientObject); break; } diff --git a/client/TlsChannel.cpp b/client/TlsChannel.cpp index 66c5c8a..08784c0 100644 --- a/client/TlsChannel.cpp +++ b/client/TlsChannel.cpp @@ -12,14 +12,18 @@ #include // std::ifstream #include // std::istreambuf_iterator #include -#include // std::call_once +#include // std::mutex #pragma comment(lib, "secur32.lib") #pragma comment(lib, "crypt32.lib") namespace { -std::once_flag g_tlsOnce; +// 进程级 TLS 状态:首连时加载一次,服务端下发/撤销信任锚后可重建(TlsReload)。 +// 所有读写都在 g_tlsMtx 下;证书校验(VerifyServerCert)在校验期间持同一把锁,因此 +// 重建时可以安全地释放旧的 store/engine——不会再有线程正拿着它们用。 +std::mutex g_tlsMtx; +bool g_tlsLoaded = false; // 已加载过至少一次(决定首次失败是否 fail-closed) bool g_tlsEnabled = false; // TLS 会话可用:send/recv 走加密 bool g_tlsError = false; // 证书存在但加载失败:拒绝连接(fail-closed) HCERTSTORE g_tlsStore = NULL; @@ -160,26 +164,23 @@ bool LoadTrustAnchor(const std::string& pem, const std::string& source, HCERTSTO return true; } -void InitTls() +enum TlsAnchor { TLS_ANCHOR_OK, TLS_ANCHOR_NONE, TLS_ANCHOR_INVALID }; + +// 读取当前信任锚并建好 store + 链引擎(成功时经 outStore/outEngine 交出,调用方负责 +// 释放)。不触碰全局状态。source 描述锚的来源,仅用于日志。 +TlsAnchor BuildTlsAnchor(HCERTSTORE* outStore, HCERTCHAINENGINE* outEngine, + std::string& source, std::string& err) { - std::string pem, source; + std::string pem; if (!ReadTrustAnchor(pem, source)) { - // 无证书 → 明文(默认,与老客户端一致) - g_tlsEnabled = false; - g_tlsError = false; - Mprintf("[TLS] disabled: no trust anchor configured, plaintext mode\n"); - return; + err = "no trust anchor configured"; + return TLS_ANCHOR_NONE; } - // 证书已配置:管理员已「颁发」,必须走 TLS;加载失败即拒绝连接(fail-closed), - // 不能静默回退明文。 HCERTSTORE store = NULL; if (!LoadTrustAnchor(pem, source, &store)) { - g_tlsEnabled = false; - g_tlsError = true; - Mprintf("[TLS] FATAL: trust anchor (%s) present but failed to load; refusing connections (fail-closed)\n", - source.c_str()); - return; + err = "trust anchor (" + source + ") present but failed to load"; + return TLS_ANCHOR_INVALID; } // 严格钉扎:链引擎只信任该 store 里的根证书,排除系统根库。否则攻击者拿一张任意 @@ -190,33 +191,87 @@ void InitTls() HCERTCHAINENGINE engine = NULL; if (!CertCreateCertificateChainEngine(&ec, &engine)) { CertCloseStore(store, 0); - g_tlsEnabled = false; - g_tlsError = true; - Mprintf("[TLS] FATAL: CertCreateCertificateChainEngine failed; refusing connections (fail-closed)\n"); - return; + err = "CertCreateCertificateChainEngine failed"; + return TLS_ANCHOR_INVALID; } + *outStore = store; + *outEngine = engine; + return TLS_ANCHOR_OK; +} + +// 按当前信任锚重建 TLS 状态(调用方持 g_tlsMtx)。 +// 首次加载失败 → fail-closed(管理员已「颁发」证书,不能静默回退明文);运行期热重载 +// 失败 → 保持现有状态不变,返回 false(一次坏下发不该打死正在工作的客户端;下次启动 +// 仍会 fail-closed)。 +bool ApplyTlsStateLocked(bool initial, std::string& err) +{ + HCERTSTORE store = NULL; + HCERTCHAINENGINE engine = NULL; + std::string source; + const TlsAnchor anchor = BuildTlsAnchor(&store, &engine, source, err); + + if (anchor == TLS_ANCHOR_INVALID) { + if (initial) { + g_tlsEnabled = false; + g_tlsError = true; + g_tlsLoaded = true; + Mprintf("[TLS] FATAL: %s; refusing connections (fail-closed)\n", err.c_str()); + } else { + Mprintf("[TLS] reload failed: %s (keeping current transport)\n", err.c_str()); + } + return false; + } + + // 互换。旧 store/engine 可以直接释放:校验路径持同一把锁,此刻没有线程在用它们。 + HCERTSTORE oldStore = g_tlsStore; + HCERTCHAINENGINE oldEngine = g_tlsEngine; g_tlsStore = store; g_tlsEngine = engine; - g_tlsEnabled = true; + g_tlsEnabled = (anchor == TLS_ANCHOR_OK); g_tlsError = false; - Mprintf("[TLS] enabled: trust anchor loaded (%s), all connections use TLS\n", source.c_str()); + g_tlsLoaded = true; + if (oldEngine) CertFreeCertificateChainEngine(oldEngine); + if (oldStore) CertCloseStore(oldStore, 0); + + if (g_tlsEnabled) + Mprintf("[TLS] enabled: trust anchor loaded (%s), all connections use TLS\n", source.c_str()); + else + Mprintf("[TLS] disabled: no trust anchor configured, plaintext mode\n"); + return true; +} + +// 首次调用时才真正加载;之后一律用缓存的状态(热重载由 TlsReload 负责)。 +inline void EnsureTlsLoadedLocked() +{ + if (!g_tlsLoaded) { + std::string err; + ApplyTlsStateLocked(true, err); + } } } // namespace bool TlsEnabled() { - std::call_once(g_tlsOnce, InitTls); + std::lock_guard lk(g_tlsMtx); + EnsureTlsLoadedLocked(); return g_tlsEnabled; } bool TlsLoadError() { - std::call_once(g_tlsOnce, InitTls); + std::lock_guard lk(g_tlsMtx); + EnsureTlsLoadedLocked(); return g_tlsError; } +bool TlsReload(std::string& err) +{ + std::lock_guard lk(g_tlsMtx); + return ApplyTlsStateLocked(!g_tlsLoaded, err); +} + // ---- TlsChannel ---- void TlsChannel::CloseHandles() @@ -249,6 +304,8 @@ bool TlsChannel::VerifyServerCert() if (sc != SEC_E_OK || !remote) return false; + // 与 TlsReload 互斥:校验期间持有 g_tlsMtx,重建时释放的旧 store/engine 不会正被使用。 + std::lock_guard lk(g_tlsMtx); if (!g_tlsStore || !g_tlsEngine) { CertFreeCertificateContext(remote); return false; @@ -273,7 +330,7 @@ bool TlsChannel::VerifyServerCert() return trusted; } -bool TlsChannel::Handshake(SOCKET sock) +bool TlsChannel::Handshake(SOCKET sock, const char* sni) { std::lock_guard lk(m_ctxMtx); CloseHandles(); // 重连时清理上一次会话(recursive 锁,此处可重入) @@ -300,8 +357,10 @@ bool TlsChannel::Handshake(SOCKET sock) SecBufferDesc outDesc = { SECBUFFER_VERSION, 1, &outBuf }; ULONG attrs = 0; - // 首次调用产生 ClientHello - sc = InitializeSecurityContextA(&m_cred, NULL, NULL, flags, 0, SECURITY_NATIVE_DREP, + // 首次调用产生 ClientHello。pszTargetName 非空时 Schannel 会把它作为 SNI 发送, + // 供反代按域名分流;证书校验仍是手动钉信任锚(SCH_CRED_MANUAL_CRED_VALIDATION), + // 不因传了名字而启用主机名校验。 + sc = InitializeSecurityContextA(&m_cred, NULL, (SEC_CHAR*)sni, flags, 0, SECURITY_NATIVE_DREP, NULL, 0, &m_ctx, &outDesc, &attrs, &tsExpiry); m_ctxValid = true; // 上下文已分配,之后必须 DeleteSecurityContext if (sc != SEC_I_CONTINUE_NEEDED && sc != SEC_E_OK) { @@ -340,7 +399,7 @@ bool TlsChannel::Handshake(SOCKET sock) outBuf = { 0, SECBUFFER_TOKEN, NULL }; outDesc = { SECBUFFER_VERSION, 1, &outBuf }; - sc = InitializeSecurityContextA(&m_cred, &m_ctx, NULL, flags, 0, SECURITY_NATIVE_DREP, + sc = InitializeSecurityContextA(&m_cred, &m_ctx, (SEC_CHAR*)sni, flags, 0, SECURITY_NATIVE_DREP, &inDesc, 0, NULL, &outDesc, &attrs, &tsExpiry); // 裁剪已消费的输入,仅保留 SECBUFFER_EXTRA 指向的未消费尾部 diff --git a/client/TlsChannel.h b/client/TlsChannel.h index ed22506..0755e54 100644 --- a/client/TlsChannel.h +++ b/client/TlsChannel.h @@ -34,15 +34,24 @@ #include #include +#include #include -// 进程级 TLS 开关:首次调用时加载信任锚(注册表或文件),返回是否启用。线程安全、幂等。 +// 进程级 TLS 开关:首次调用时加载信任锚(注册表或文件),返回是否启用。线程安全。 +// Windows 版同样是「可热切换」的:TlsReload() 之后本函数立刻反映新状态(不是 call_once)。 bool TlsEnabled(); // 信任锚已配置但加载失败(损坏/格式错误)时为 true:表示管理员已「颁发」证书, // 客户端必须拒绝连接(fail-closed),不得静默回退明文。线程安全、幂等。 bool TlsLoadError(); +// 按当前信任锚位置(注册表 settings\tls_cert → settings.ini → exe 同目录 yama-ca.pem) +// 重建 TLS 上下文并更新开关,让随后建立的新连接立刻走新传输方式(已建立的连接不受 +// 影响)。信任锚无效时不做任何改动、返回 false 并给出原因——一次坏下发不该把正在工作 +// 的客户端打死(下次启动仍 fail-closed)。线程安全:与 TlsEnabled/TlsLoadError/握手 +// 的证书校验互斥。 +bool TlsReload(std::string& err); + class TlsChannel { public: @@ -55,7 +64,9 @@ public: // 在已连通的 socket 上完成 TLS 客户端握手并校验服务端证书链。失败返回 false, // 调用方必须关闭 socket(fail-closed)。可重复调用(重连时复用同一对象)。 - bool Handshake(SOCKET sock); + // sni 非空且为域名时作为 SNI 发送,让反代(nginx)按域名分流到正确的 server 块; + // 证书校验仍只钉信任锚、不校主机名,与直接连主控端口时完全一致。 + bool Handshake(SOCKET sock, const char* sni = NULL); // 解密读取:返回 >0 的明文字节数;0 表示对端正常关闭;SOCKET_ERROR(-1) 出错。 int Recv(SOCKET sock, char* buf, int len); @@ -115,7 +126,8 @@ bool TlsLoadError(); // 服务端下发信任锚(COMMAND_TLS_ENABLE):把 PEM 正文写入 // $XDG_CONFIG_HOME/ghost/yama-ca.pem(默认 ~/.config/ghost/yama-ca.pem,无 HOME 时退回 // 程序目录),原子替换、权限 0600。写入前先解析验证,不是合法证书就整包拒绝。 -// 落盘后用 TlsReload() 立即生效(Windows 客户端仍要重启进程)。 +// 落盘后用 TlsReload() 立即生效(Windows 客户端在 COMMAND_TLS_ENABLE 里调用同一个 +// TlsReload(),两平台行为一致)。 bool TlsTrustAnchorSave(const std::string& pem, std::string& err); // 服务端撤销信任锚(COMMAND_TLS_DISABLE):删除上述配置文件(含程序目录同名文件)。 @@ -140,7 +152,8 @@ public: // 在已连通的 socket 上完成 TLS 1.2 客户端握手并按信任锚校验服务端证书链。失败返回 // false,调用方必须关闭 socket(fail-closed)。可重复调用(重连时复用同一对象)。 - bool Handshake(int sock); + // sni 非空且为域名时作为 SNI 发送(反代按域名分流);证书校验仍只钉信任锚。 + bool Handshake(int sock, const char* sni = NULL); // 解密读取:返回 >0 的明文字节数;0 表示对端正常关闭;SOCKET_ERROR(-1) 出错。 int Recv(int sock, char* buf, int len); diff --git a/client/TlsChannelLinux.cpp b/client/TlsChannelLinux.cpp index 9b77070..acc6df0 100644 --- a/client/TlsChannelLinux.cpp +++ b/client/TlsChannelLinux.cpp @@ -40,7 +40,7 @@ namespace { const char* const kAnchorName = "yama-ca.pem"; -// 进程级 TLS 状态。与 Windows 版的 call_once 不同,这里可由 TlsReload() 热切换 +// 进程级 TLS 状态。两平台都可由 TlsReload() 热切换(Windows 版改造前是 call_once) // (服务端 COMMAND_TLS_ENABLE/DISABLE 落盘后立即重建),因此用互斥量而不是 once_flag。 std::mutex g_stateMtx; // 保护下面四项 bool g_inited = false; // 首次加载已完成(含 fail-closed 基态) @@ -415,7 +415,7 @@ void TlsChannel::CloseHandles() m_ready = false; } -bool TlsChannel::Handshake(int sock) +bool TlsChannel::Handshake(int sock, const char* sni) { std::lock_guard lk(m_ctxMtx); CloseHandles(); // 重连时清理上一次会话(recursive 锁,此处可重入) @@ -438,6 +438,10 @@ bool TlsChannel::Handshake(int sock) SSL_free(ssl); return false; } + // SNI:反代(nginx)据此按域名分流到正确的 server 块。证书校验仍是自定义 store 里的 + // 信任锚(SSL_VERIFY_PEER + 不校主机名),与直连主控端口时完全一致。 + if (sni && *sni) + SSL_set_tlsext_host_name(ssl, sni); SSL_set_connect_state(ssl); // 阻塞 socket:WANT_READ / WANT_WRITE 只在记录分片等极端情形出现,重试即可。 diff --git a/client/WsChannel.h b/client/WsChannel.h new file mode 100644 index 0000000..e78ab35 --- /dev/null +++ b/client/WsChannel.h @@ -0,0 +1,355 @@ +// WsChannel.h — WebSocket 传输层(客户端侧,头文件实现,无需改工程文件) +// +// 用途:当出站被限制在 80/443/8080 等少数端口时,客户端经反代(如 nginx 的 443) +// 以 WebSocket 进入服务端。应用协议完全不变:本层只把字节流套进 WS 帧里,位置与 +// TLS 相同(TLS 之下、应用协议之上)。 +// +// 两种形态(见 common/commands.h 的 ProtoType): +// PROTO_WS 裸 WebSocket(反代把明文 upgrade 转发到主控端口) +// PROTO_WSS WebSocket over TLS(反代终止 TLS;客户端仍用既有信任锚校验证书, +// 并通过 SNI 让反代按域名分流到正确的 server 块) +// +// 线程模型:Handshake 在连接线程上、m_bConnected 置位前调用;之后 Recv 只由接收 +// 线程调用、Send 只由发送线程调用。但 Recv 里要为 ping/close 回写控制帧,收、发两个 +// 线程会同时写 socket,因此写方向必须用 m_sendMtx 串行化,且一帧的头与体要一次写出, +// 否则两帧会互相穿插、对端解帧错位。TLS 自身的互斥由 TlsChannel 负责。 +#pragma once + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +// 本头在 IOCPClient.h 里先于 common/commands.h 展开(后者是 Linux 版 SOCKET 的来源), +// 这里补一份同类型定义;重复 typedef 合法。 +typedef int SOCKET; +#endif +#ifndef SOCKET_ERROR +#define SOCKET_ERROR -1 +#endif + +#include "TlsChannel.h" + +class WsChannel +{ +public: + WsChannel() {} + +#ifdef YAMA_TLS_CHANNEL + // 绑定底层加密通道:WSS 指向 IOCPClient 的 TlsChannel,裸 WS 传 NULL。 + // 设置后本类的所有原始收发都经由它(或直接 send/recv)。 + void SetTls(TlsChannel* tls) { m_tls = tls; } +#endif + + // 每次连接前清理上一轮的帧状态。 + void Reset() + { + m_ready = false; + m_remaining = 0; + m_masked = false; + m_maskPos = 0; + } + + bool IsReady() const { return m_ready; } + + // 在已连通的 socket 上完成 HTTP Upgrade 握手。host 用于 Host 头(WSS 的 SNI 由 + // TlsChannel::Handshake 负责);path 是 Upgrade 请求的路径。失败时 err 给出原因。 + bool Handshake(SOCKET sock, const std::string& host, int port, const char* path, std::string* err = NULL) + { + Reset(); + + char hostPort[120] = { 0 }; + if (port == 80 || port == 443) + snprintf(hostPort, sizeof(hostPort), "%s", host.c_str()); + else + snprintf(hostPort, sizeof(hostPort), "%s:%d", host.c_str(), port); + + // Sec-WebSocket-Key 用固定值:它只用于让中间设备确认这是 WS 握手(服务端会原样 + // 做 SHA1/Base64 回显),不承担任何安全职责;真正的身份由隧道内登录和 TLS 承担。 + std::string req; + req.reserve(256); + req += "GET "; + req += path; + req += " HTTP/1.1\r\nHost: "; + req += hostPort; + req += "\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"; + req += "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"; + req += "Sec-WebSocket-Version: 13\r\n\r\n"; + + if (!SendAll(sock, req.data(), (int)req.size())) { + if (err) *err = "send upgrade request failed"; + return false; + } + + // 逐字节读响应头,读到空行即停(多读的字节会被当作 WS 帧,而服务端在我们发首包 + // 前不会主动发数据,所以这里读到空行为止是安全的)。 + std::string head; + bool complete = false; + char c = 0; + while (head.size() < 4096) { + int n = RawRecv(sock, &c, 1); + if (n <= 0) { + if (err) *err = "connection closed during upgrade"; + return false; + } + head += c; + if (head.size() >= 4 && head.compare(head.size() - 4, 4, "\r\n\r\n") == 0) { + complete = true; + break; + } + } + if (!complete) { + if (err) *err = "upgrade response head too large"; + return false; + } + if (head.compare(0, 12, "HTTP/1.1 101") != 0) { + if (err) { + std::string line = head.substr(0, head.find("\r\n")); + *err = "upgrade rejected: " + line; + } + return false; + } + + m_ready = true; + return true; + } + + // 读取应用层字节:>0 为读到的字节数;0 表示对端正常关闭;SOCKET_ERROR 表示出错。 + // 与 recv 语义一致,调用方按同样方式处理。 + int Recv(SOCKET sock, char* buf, int len) + { + if (!m_ready || len <= 0) + return SOCKET_ERROR; + + for (;;) { + if (m_remaining > 0) { + int want = (m_remaining < (long long)len) ? (int)m_remaining : len; + int n = RawRecv(sock, buf, want); + if (n <= 0) + return n; // 0 = 对端关闭;SOCKET_ERROR = 出错(TLS 已记录原因) + if (m_masked) { + for (int i = 0; i < n; i++) { + buf[i] ^= (char)m_mask[m_maskPos]; + m_maskPos = (m_maskPos + 1) & 3; + } + } + m_remaining -= n; + return n; + } + + unsigned char opcode = 0; + long long frameLen = 0; + if (!ReadHeader(sock, opcode, frameLen)) + return SOCKET_ERROR; + + if (opcode >= 0x8) { + // 控制帧:载荷 ≤125 字节,必须在数据帧之间处理 + char ctl[125]; + if (frameLen > (long long)sizeof(ctl)) + return SOCKET_ERROR; + if (frameLen > 0) { + if (!ReadExact(sock, ctl, (int)frameLen)) + return SOCKET_ERROR; + Unmask(ctl, (int)frameLen); + } + if (opcode == 0x8) { // close:回一个 close,然后按正常关闭处理 + (void)WriteFrame(sock, 0x8, ctl, (int)frameLen); + m_ready = false; + return 0; + } + if (opcode == 0x9) { // ping → pong + if (!WriteFrame(sock, 0xA, ctl, (int)frameLen)) + return SOCKET_ERROR; + } + // 0xA (pong) 与其它的直接忽略 + continue; + } + + if (opcode != 0x0 && opcode != 0x1 && opcode != 0x2) + return SOCKET_ERROR; + if (frameLen == 0) + continue; + + m_remaining = frameLen; + m_maskPos = 0; + } + } + + // 发送应用层字节:全部发出返回 len;否则 SOCKET_ERROR。每包一帧(FIN + binary), + // 与主控端 wsConn 的「一次 Write 一帧」对齐,保证包边界与发出时一致。 + int Send(SOCKET sock, const char* buf, int len) + { + if (!m_ready) + return SOCKET_ERROR; + if (len <= 0) + return 0; + return WriteFrame(sock, 0x2, buf, len) ? len : SOCKET_ERROR; + } + + // 是否有已收到但尚未交付的应用层字节。两处来源:帧载荷只解析了一半(m_remaining), + // 或 TLS 会话内部还压着已解密明文(WSS)。接收线程在 select 之前要优先消费, + // 否则 select 看不到这些字节会被 2 秒超时卡住。 + bool HasPending() const + { + if (m_remaining > 0) + return true; +#ifdef YAMA_TLS_CHANNEL + if (m_tls) + return m_tls->HasPending(); +#endif + return false; + } + +private: + // ---- 原始收发:有 TLS 会话时全程走 TLS,否则裸 socket ---- + + int RawRecv(SOCKET sock, char* buf, int len) + { +#ifdef YAMA_TLS_CHANNEL + if (m_tls) + return m_tls->Recv(sock, buf, len); +#endif + return ::recv(sock, buf, len, 0); + } + + int RawSend(SOCKET sock, const char* buf, int len) + { +#ifdef YAMA_TLS_CHANNEL + if (m_tls) + return m_tls->Send(sock, buf, len); +#endif + return ::send(sock, buf, len, 0); + } + + bool SendAll(SOCKET sock, const char* buf, int len) + { + int sent = 0; + while (sent < len) { + int n = RawSend(sock, buf + sent, len - sent); + if (n <= 0) + return false; + sent += n; + } + return true; + } + + bool ReadExact(SOCKET sock, void* buf, int len) + { + char* p = (char*)buf; + int got = 0; + while (got < len) { + int n = RawRecv(sock, p + got, len - got); + if (n <= 0) + return false; + got += n; + } + return true; + } + + // 读帧头(含扩展长度与掩码键)。服务端发来的帧不应带掩码,带了也照样解。 + bool ReadHeader(SOCKET sock, unsigned char& opcode, long long& frameLen) + { + unsigned char h[2] = { 0, 0 }; + if (!ReadExact(sock, h, 2)) + return false; + if (h[0] & 0x70) + return false; // 未协商扩展却置了 RSV 位 + + opcode = h[0] & 0x0F; + m_masked = (h[1] & 0x80) != 0; + frameLen = h[1] & 0x7F; + + if (frameLen == 126) { + unsigned char ext[2]; + if (!ReadExact(sock, ext, 2)) + return false; + frameLen = ((long long)ext[0] << 8) | ext[1]; + } else if (frameLen == 127) { + unsigned char ext[8]; + if (!ReadExact(sock, ext, 8)) + return false; + frameLen = 0; + for (int i = 0; i < 8; i++) + frameLen = (frameLen << 8) | ext[i]; + if (frameLen < 0) + return false; + } + if (m_masked && !ReadExact(sock, m_mask, 4)) + return false; + m_maskPos = 0; + return true; + } + + void Unmask(char* buf, int len) + { + if (!m_masked) + return; + for (int i = 0; i < len; i++) { + buf[i] ^= (char)m_mask[m_maskPos]; + m_maskPos = (m_maskPos + 1) & 3; + } + } + + // 写一帧。客户端发往服务端的帧必须带掩码(RFC 6455 §5.3)。整帧(帧头 + 掩码后的 + // 载荷)在 m_sendMtx 内一次写出:接收线程可能在同一时刻回 pong/close,若两帧各自 + // 分「头」「体」两次发送就会交错,对端从此解帧错位。 + bool WriteFrame(SOCKET sock, unsigned char opcode, const char* payload, int len) + { + unsigned char hdr[14]; + int hn = 2; + hdr[0] = (unsigned char)(0x80 | opcode); + if (len < 126) { + hdr[1] = (unsigned char)(0x80 | len); + } else if (len <= 0xFFFF) { + hdr[1] = 0x80 | 126; + hdr[2] = (unsigned char)((len >> 8) & 0xFF); + hdr[3] = (unsigned char)(len & 0xFF); + hn = 4; + } else { + hdr[1] = 0x80 | 127; + unsigned long long v = (unsigned long long)len; + for (int i = 0; i < 8; i++) + hdr[2 + i] = (unsigned char)((v >> (56 - 8 * i)) & 0xFF); + hn = 10; + } + + unsigned char mask[4]; + for (int i = 0; i < 4; i++) + mask[i] = (unsigned char)(rand() & 0xFF); + memcpy(hdr + hn, mask, 4); + hn += 4; + + std::lock_guard lock(m_sendMtx); + // 复用同一个缓冲区:发送热路径上每帧不再分配一个新 vector。 + m_frame.resize((size_t)hn + (size_t)len); + char* frame = m_frame.data(); + memcpy(frame, hdr, hn); + char* dst = frame + hn; + for (int i = 0; i < len; i++) + dst[i] = (char)(payload[i] ^ (char)mask[i & 3]); + + return SendAll(sock, frame, hn + len); + } + +#ifdef YAMA_TLS_CHANNEL + TlsChannel* m_tls = NULL; // WSS 时指向 IOCPClient 的 TLS 会话 +#endif + bool m_ready = false; + + // 写方向串行化(发送线程与接收线程的 pong/close 都会写帧),以及复用的整帧缓冲。 + std::mutex m_sendMtx; + std::vector m_frame; + + // 当前数据帧的剩余载荷长度与掩码状态(只由接收线程读写,无需加锁) + long long m_remaining = 0; + bool m_masked = false; + unsigned char m_mask[4] = { 0, 0, 0, 0 }; + int m_maskPos = 0; +}; diff --git a/common/commands.h b/common/commands.h index d9d4a09..8c6905d 100644 --- a/common/commands.h +++ b/common/commands.h @@ -825,9 +825,15 @@ enum ProtoType { PROTO_HTTP = 2, // HTTP PROTO_RANDOM = 3, // 随机 PROTO_KCP = 4, // KCP - PROTO_HTTPS = 5, // HTTPS + PROTO_WS = 5, // WebSocket(明文;反代把 upgrade 转发到主控端口) + PROTO_WSS = 6, // WebSocket over TLS(客户端经反代 443 等受限端口接入) + PROTO_HTTPS = 7, // HTTPS(未启用:生成界面从未提供该选项,故 5 可让给 WS;原值 5) }; +// WebSocket 传输的 Upgrade 路径。反代的 location 必须与这里一致(服务端默认接受任意路径, +// 该常量只决定客户端请求哪个路径,见 client/WsChannel.h)。 +#define WS_UPGRADE_PATH "/api/ws" + #define KCP_SESSION_ID 666 enum RunningType { diff --git a/docs/WebSocket_Transport_Design.md b/docs/WebSocket_Transport_Design.md new file mode 100644 index 0000000..4908be8 --- /dev/null +++ b/docs/WebSocket_Transport_Design.md @@ -0,0 +1,322 @@ +# WebSocket 传输设计(受限出站端口下经反代接入) + +> 版本:v1.0(已实现并本地验证) · 日期:2026-10-30 +> 目标:客户端的出站被限制在 80 / 443 / 8080 等少数端口,且这些端口都不归主控所有时, +> 仍能让客户端连上服务端;同时把 nginx 侧改动压到最小(只加一个 server 块或一个 location)。 +> +> 需求原文: +> 1. 「我的客户端连接服务端时端口被限制……只能借助于已有端口分流,比如 nginx 的端口。 +> 但是我的客户端目前是 TCP 流量。调研一下怎么解决」 +> 2. 「我的客户端连接到 xxx.com:6543, 但是 6543 不允许出站」 +> 3. 「6543 我可以修改,但是只能改为 80,443,8080,但是这些端口都已经在使用」 +> 4. 「B 方案更适合目前场景。xxx.com 的 nginx 配置尽量少改,不要大改」 +> +> 已核实前提:目标服务端是 **Go 版(`server/go`)**;客户端当前**都已启用 TLS**。 + +## 1. 方案选型 + +| 方案 | 做法 | 结论 | +|---|---|---| +| A. 占用 8080 | 把主控端口改到 8080 直连 | 否决:8080 已被别的应用占用 | +| **B. WebSocket over nginx 443** | 客户端把应用字节流套进 WS 帧,经 nginx `server_name` + `location /api/ws` 转发到 127.0.0.1:6543 | **采用** | +| C. 租新端口 | 换一台能出站的机器/端口 | 否决:出站白名单是客户端侧策略,换端口仍被拦 | + +B 方案的关键点:WS 升级只是一次普通 HTTP 请求,nginx 现有的 443 已经能承载; +真正的应用协议完全不变,只是「字节流外面多了一层帧」。 + +## 2. 协议号与形态 + +`common/commands.h`: + +| 值 | 枚举 | 形态 | +|---|---|---| +| 5 | `PROTO_WS` | 裸 WebSocket(明文;反代把明文 upgrade 转发到主控端口) | +| 6 | `PROTO_WSS` | WebSocket over TLS(**客户端经反代 443 接入,推荐**) | +| 7 | `PROTO_HTTPS` | 从未在生成界面出现过,故 5 让给 WS(原值 5,见下方风险) | + +`WS_UPGRADE_PATH` = `/api/ws`:客户端请求的 Upgrade 路径,反代的 `location` 必须与之对齐; +服务端默认接受任意路径,`YAMA_WS_PATH` 只用于把路径收窄。 + +传输层位置(自下而上):TCP → TLS(可选)→ **WebSocket 帧** → 既有应用协议(Shine/HELL 头、 +XOR、zstd、HTTP 伪装层)。WS 与 HTTP 伪装的 `MaskType` 互斥:选 WS 时用 `MaskTypeNone`。 + +> 兼容性:`PROTO_HTTPS` 从 5 移到 7。它在 `NewNetClient` 里本来就被直接拒绝、生成界面也从未 +> 提供,因此不存在已发布客户端带 5 号协议的情况。 + +## 3. 客户端实现 + +新增 `client/WsChannel.h`(**头文件实现**,各平台工程文件不必改动): + +- `Handshake(sock, host, port, path, &err)`:发 `GET HTTP/1.1` + `Upgrade: websocket`, + 期望 `HTTP/1.1 101`;`Sec-WebSocket-Key` 用固定值(它只用于让中间设备识别这是 WS 握手, + 安全由隧道内登录与 TLS 承担)。 +- `Send`:一次 `SendTo` 写一帧(FIN + binary,**客户端帧必须掩码**,RFC 6455 §5.3), + 与 Go 端 `wsConn.Write` 的「一次 Write 一帧」对齐,包边界与发出时一致。 +- `Recv`:在帧载荷上做流式读取(`m_remaining` 记录剩余、`m_maskPos` 记录掩码位置), + ping 自动回 pong,close 回 close 并按正常关闭返回 0,跨帧边界的应用层包不会丢。 +- `HasPending()`:帧载荷只解析了一半、或 WSS 下 TLS 会话里还压着已解密明文时返回 true, + 让接收线程在 `select` 之前优先消费(否则只能等 2 秒超时)。 +- 并发写:`Recv` 里要为 ping/close 回写控制帧,与发送线程的数据帧是两条并发的写路径。 + `WriteFrame` 用 `m_sendMtx` 串行化,并把整帧(帧头 + 掩码载荷)放进一个复用缓冲区、 + 一次 `SendAll` 写出——若头体分两次写,两帧会互相穿插,对端从此解帧错位。 + +接线(`IOCPClient.h` / `.cpp`):`ReceiveData` / `SendTo` / `HasPendingData` 三处在 +`m_useTls` 之前判 `m_useWs`;`ConnectServer` 在 socket 选项之后、`m_bConnected = TRUE` 之前 +完成 `TLS 握手 → WS 握手`,`Disconnect` 清 `m_useWs` 并 `m_ws.Reset()`。 + +### 3.1 信任模型:不换锚,靠 SNI 分流 + +客户端证书校验仍是**钉死信任锚**(Windows:`CertCreateCertificateChainEngine` + `hExclusiveRoot` +手动建链;Linux:`SSL_VERIFY_PEER` + 自定义 `X509_STORE`;均**不校主机名**)。因此 +「经 nginx 接入」不能随便换成网站自己的证书——那会校验失败。 + +于是给 nginx 一个**独立 `server_name`**(如 `ws.xxx.com`)并挂主控自己的证书,客户端靠 +**SNI** 让 nginx 选中这个 server 块: + +- Windows Schannel:`InitializeSecurityContextA(..., pszTargetName = sni, ...)`(TlsChannel.cpp) +- Linux OpenSSL:`SSL_set_tlsext_host_name(ssl, sni)`(TlsChannelLinux.cpp) +- 只在 `PROTO_WSS` 且地址是域名时传;直连主控端口时保持原样(不发明 SNI),行为与改造前一致。 + +Host 头同理:用**配置里的地址(域名)**,而不是解析后的 IP——子连接只拿到 IP,必须回到 +`CONNECT_ADDRESS::GetFirstServerIP()` 取。地址里填分号分隔的多个地址时取第一个,因此经反代 +接入只应填一个域名。 + +### 3.2 回退与 fail-closed + +传输方式只由信任锚决定(进程内缓存,服务端下发后可热重载,见 §3.3): + +| 情形 | 行为 | +|---|---| +| 有锚 + `PROTO_WSS` / 直连主控端口 | TLS 握手;**握手或证书校验失败即断开**(fail-closed) | +| **无锚 + `PROTO_WSS`** | **回退为明文 WebSocket**:客户端还没被「颁发」证书,先打通隧道;服务端下发锚后自动升级(见下) | +| 有锚但加载失败(损坏/为空) | 拒绝连接(fail-closed)——管理员已「颁发」证书,不能静默回退明文 | +| 无锚 + `PROTO_WS` | 明文(本来如此) | +| 有锚 + `PROTO_WS` | **仍是明文**(显式选择的形态,见下) | +| 平台无 TLS(macOS / Android 未定义 `YAMA_TLS_CHANNEL`)+ `PROTO_WSS` | 直接断开并打日志:没有热切换路径,回退只会让人误以为在加密 | +| WS 握手失败 | 断开,`Mprintf` 里带上服务端返回的状态行 | + +回退的判据是「本机**根本没有**信任锚」这一可观测状态,而不是握手中途失败——TLS 握手 +失败、证书链校验失败、WS 握手失败一律断开。因此中间人最多能让「尚未下发锚」的客户端 +留在明文(此时本来也没有可校验的锚),无法把已经持锚的客户端拉回明文。 + +升级路径(服务端菜单「TLS 传输 → 启用」,`COMMAND_TLS_ENABLE` + PEM 正文):客户端把锚 +落盘(Windows 注册表 `settings\tls_cert`;Linux `~/.config/ghost/yama-ca.pem`),调用 +`TlsReload()` 重建 TLS 上下文,**只在传输方式真正变化(明文 ↔ TLS)时**断开重连,之后 +的新连接自动走 WSS;仅换一张证书(仍是 TLS)不打断现有会话。两平台都是立即生效, +不再需要重启进程。撤销(`COMMAND_TLS_DISABLE`)同理反向切换。 + +> 部署含义:「先明文、之后升级为 WSS」要求同一个 `地址:端口` 既能吃明文 WS 也能吃 TLS +> ——Go 服务端的设备端口正是这样(`wrapTLS` → `wrapWebSocket` 两级嗅探,见 §4)。由 +> nginx 终止 TLS 的部署做不到:443 只收 TLS,明文回退会直接失败,这种拓扑必须先把证书 +> 配到 nginx 上再出包。(对应的两个 nginx 监听端口也无法互替,地址是出包时写死的。) + +`PROTO_WS` 的定位不变:它是**显式选择的明文形态**(§5.2 的 80 端口明文反代、内网直连), +即便客户端配了信任锚也不叠加 TLS,否则该路径永远不可达——选择 WS 是出包时的决定, +不存在远端可触发的降级。 + +### 3.3 信任锚热重载 + +`TlsEnabled()` / `TlsLoadError()` 首次调用时加载锚,之后读写的是受 `TlsReload` 保护的 +进程级状态(Windows 原来用 `std::call_once` 缓存到进程结束,无法热切换;现改为互斥量, +与 Linux 版语义一致): + +- 重建失败(锚存在但无效)→ **保持现有状态不变**,只记一行日志:一次坏下发不该打死正在 + 工作的客户端;下次启动仍会 fail-closed。 +- 重建成功 → 互换 store / 链引擎并释放旧的。Windows 的证书校验(`VerifyServerCert`) + 在整个校验过程中持同一把锁,因此旧句柄不会在别的线程手里被释放。 +- 回环验证:Linux 由 `linux/main.cpp::ApplyTlsAnchorChange`、Windows 由 + `client/KernelManager.cpp::ApplyTlsAnchorChange` 调用,两者都只在 `TlsEnabled()` + 前后值发生变化时 `Disconnect()`,让重连循环用新传输方式重新登录。 + +## 4. 服务端实现(Go) + +`server/go/server/ws.go`,在 `handleConnection` 里分层嗅探: + +``` +raw conn --wrapTLS(嗅 0x16 0x03)--> [TLS] --wrapWebSocket(嗅 "GET ")--> [WS 帧] +``` + +- 非 WS 连接:`bufferedConn` 原样返回,既有客户端零影响;嗅探用 `Peek`,不消耗字节。 +- WS 连接:解析请求头、校验 `Upgrade`/`Connection`/`Sec-WebSocket-Key`,`YAMA_WS_PATH` + 非空时再校验路径;通过后回 101 并返回 `wsConn`。 +- `wsConn`:`Read` 在帧载荷上流式返回,`Write` 一次一帧(`net.Buffers` writev 拼头与体), + 处理客户端掩码、控制帧(ping/pong/close),`Close` 发 close 帧并设 1 秒写超时。 +- 反代信任:仅当对端 IP 落在 `TrustedProxies`(默认回环)内才采信 `X-Forwarded-For` / + `X-Real-IP`,取到的真实 IP 用于 `Context.GetPeerIP()`、`hub.Device.PeerIP` 与 ConnAuth + 子连接的源 IP 绑定。 +- 真实 IP 取 `X-Forwarded-For` 的**最后一段**(nginx 的 `$proxy_add_x_forwarded_for` 把 + 它实际看到的地址追加在末尾,客户端自己塞在开头的段一律无视;与 `web/ws.go` 的 `clientIP` + 同一规则),没有 XFF 时才退回 `X-Real-IP`。 +- 每 IP 限流分两段:**直连对端**在 `acceptLoop` 里、起 goroutine 和任何嗅探读之前就计数 + (否则一个只建连接不发字节的对端能白占 goroutine 和缓冲直到 `ReadTimeout`);**反代后面 + 的设备**在 `handleConnection` 里按上报的真实 IP 计数(否则所有设备共用一个回环额度)。 + +环境变量: + +| 变量 | 说明 | +|---|---| +| `YAMA_WS_PATH` | 只接受该路径的 upgrade(空 = 任意路径) | +| `YAMA_TRUSTED_PROXIES` | 允许上报真实 IP 的代理地址,逗号分隔的 IP/CIDR,默认 `127.0.0.0/8,::1/128` | + +## 5. nginx 最小改动 + +前提:客户端地址填 `ws.xxx.com`、端口 **443**、协议选 **WebSocket+TLS**。 +DNS 加一条 `ws.xxx.com` → 同一台机器即可(共用 443,靠 SNI + Host 分流,**不动**现有站点块)。 + +```nginx +# 新增一个 server 块;证书用主控自己的那份(就是客户端钉的锚,与 6543 的 YAMA_TLS_CERT 同一文件) +server { + listen 443 ssl; # nginx ≥1.25.1 可另加一行 http2 on; + server_name ws.xxx.com; + + ssl_certificate /etc/nginx/certs/yama.crt; + ssl_certificate_key /etc/nginx/certs/yama.key; + + location /api/ws { + proxy_pass http://127.0.0.1:6543; # 主控设备端口 + proxy_http_version 1.1; + + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; # 让服务端拿到设备真实 IP + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_buffering off; # 长连接必须关缓冲 + proxy_read_timeout 86400s; # 空闲心跳间隔远小于此,防止被掐 + proxy_send_timeout 86400s; + } +} +``` + +若该域名现有的 443 证书**正好就是客户端钉的那份**,可以连 server 块都不加, +只把上面 `location /api/ws { ... }` 贴进现有 443 块里。 + +### 5.1 站点与主控用两张不同证书时 + +实际部署里 `xxx.com` 的 443 证书(网站那张)和 6543 用的证书(客户端钉的锚)**是两张**。 +同一端口放两张证书没问题,nginx 按 **SNI** 选 server 块: + +| 客户端填的地址 | 客户端发的 SNI | nginx 选中 | 出示的证书 | 结果 | +|---|---|---|---|---| +| `xxx.com` | `xxx.com` | 网站块 | 网站的证书 | **握手失败**(不是钉的锚) | +| `ws.xxx.com` | `ws.xxx.com` | 主控块 | 6543 那张 | 通过 | + +所以设备端必须用**另一个名字**(`ws.xxx.com`,或任何解析到同机、nginx 里能单独成块的名字): + +- DNS:加一条 `ws.xxx.com` A 记录;若已有泛解析 `*.xxx.com` 则无需改动。 +- `ssl_certificate` 指向**与 Go 服务端 `YAMA_TLS_CERT` 完全相同的那个文件**(Go 端若配的是 + fullchain,这里也用 fullchain)。证书内容稍有不同(换过一张、少了中间证书)就会校验失败。 +- 这张证书**不需要覆盖 `ws.xxx.com`**——客户端只验链到钉死的锚,不校主机名(见 §3.1)。 +- 不要给主控块加 `default_server`,让网站块继续当默认块,其它访客行为不变。 + +自查(在服务器上执行,输出应是 6543 那张证书): + +```bash +openssl s_client -connect 127.0.0.1:443 -servername ws.xxx.com /dev/null \ + | openssl x509 -noout -subject -fingerprint -sha256 +``` + +若看到的是网站的证书,说明 SNI 没落到主控块(名字拼错、块没 reload、或客户端地址填的是 `xxx.com`)。 + +### 5.2 拿不到 `ws.xxx.com` 这种名字时:地址填 IP,用 443 的默认块 + +结论:**DNS 不是必需的**。SNI 只是 ClientHello 里的一个字符串,nginx 拿它跟 `server_name` +比,不做解析;真正需要 DNS 的只有客户端解析地址这一步。所以换一条路走: + +**客户端地址填 IP**(`1.2.3.4`,就是 `xxx.com` 解析出的那个),协议 `WebSocket+TLS`,端口 443。 +地址是 IP 字面量时,客户端按 RFC 6066 **不发 SNI**(见 §3.1 的条件),nginx 于是落到 443 的 +`default_server` 块——把主控块设成默认块即可。 + +```nginx +# 主控块:无 SNI(或 SNI 不匹配)时命中,出示客户端钉的那张证书 +server { + listen 443 ssl default_server; + server_name _; # 兜底块,不占任何真实域名 + + ssl_certificate /root/cert/yama.crt; # 与 6543 的 YAMA_TLS_CERT 同一文件 + ssl_certificate_key /root/cert/yama.key; + + location /api/ws { + proxy_pass http://127.0.0.1:6543; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_buffering off; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + } +} +``` + +现有网站块**一个字都不用改**:浏览器访问 `https://xxx.com` 时会发 SNI `xxx.com`, +按名字命中网站块,照旧拿网站的证书。 + +四个注意点: + +- **同一 `地址:端口` 只能有一个 `default_server`**。若网站块原本写了 + `listen 443 ssl default_server;`,把那里的 `default_server` 去掉(`nginx -t` 会直接报 + duplicate default server)。 +- 若默认块上设了 `ssl_reject_handshake on;`(有的加固模板会这么写),把它删掉,否则无 SNI 的连接被直接拒。 +- IPv6 的默认块是独立的:若还有 `listen [::]:443`,默认块要么同样加 `[::]:443 default_server`, + 要么保持 IPv4 即可(本客户端只连 AF_INET)。 +- 代价是地址里写死 IP:换机器/换 IP 要重出客户端。站点域名不变、只是 `xxx.com` 的 A 记录 + 换 IP 时,旧客户端会连到旧机器。 + +> 还有两条退路,代价更大,一般不用: +> - 走 **80 端口明文**:把 `location /api/ws` 贴进现有 80 块,客户端选 `PROTO_WS`。没有证书问题, +> 但整条隧道是明文(只剩既有 XOR/Shine 混淆)。 +> - 给客户端加一个「SNI/Host 单独覆盖」的配置项,让地址仍是域名、SNI 用一个不存在的名字。 +> 需要改代码和重新出客户端,只有在必须保留域名地址时才值得做。 + +要点: + +- `proxy_http_version 1.1` + 两条 `Upgrade` 头缺一不可,否则 nginx 会把升级请求当普通请求代理。 +- 关 `proxy_buffering`,否则 nginx 攒包,画面与控制会明显延迟。 +- `X-Real-IP` / `X-Forwarded-For` 只有在服务端的 `YAMA_TRUSTED_PROXIES` 覆盖 `127.0.0.1` + 时才被采信(默认已覆盖)。 + +## 6. 部署清单 + +1. 服务端:Go 版加环境变量 `YAMA_WS_PATH=/api/ws`(可选)、`YAMA_TRUSTED_PROXIES`(默认够用)。 +2. nginx:按 §5 加 server 块或 location,`nginx -t && nginx -s reload`。 +3. DNS:`ws.xxx.com` 指向该机器。 +4. 生成客户端:协议选 `WebSocket+TLS`,地址填 `ws.xxx.com`,端口 443。 +5. 日志确认: + - 服务端 `WebSocket transport on device port enabled: path=/api/ws`、 + `WebSocket device from <真实 IP>`; + - 客户端 `[WS] WebSocket 已就绪: ws.xxx.com:443/api/ws (TLS)`。 + +## 7. 验证方法(本次已做) + +- `server/go`:`go test ./...` 全绿,`ws_test.go` 覆盖升级、隧道、ping/pong、1 MiB 大帧、 + 坏握手拒绝、转发 IP(含「XFF 末尾优先 / 开头伪造被无视」)、信任代理解析,以及 + 「无换行的超长请求行在 8 KiB 上限处被拒」;`server_test.go` 覆盖直连对端在嗅探前即计入 + 每 IP 上限、回环反代被豁免。 +- 客户端:`client/WsChannel.h` 用一份**独立的 Python 最小 WS 服务端**(std lib 实现,非同一套 + 代码)交叉验证:1/5/125/126/130/4096/70000 字节全部往返一致、`Host: localhost:18080` + 正确、客户端帧带掩码、服务端主动 ping 被回 pong。 +- 并发写:另一份 Python 校验端一边每 5 帧发一次 ping、一边逐帧校验 3000×300 字节的载荷, + 客户端同时从发送线程写数据帧、从接收线程回 pong。修复前在 ~40 帧处即出现解帧错位 + (`BAD_PAYLOAD`/发送失败),修复后 3000 帧全部逐字节正确。 +- 互操作:把该测试客户端直接指向真实 Go 服务端(`-port 16543`,`YAMA_WS_PATH=/api/ws`), + 服务端日志出现 `WebSocket device from 127.0.0.1`,握手通过。 +- 信任锚热重载(Windows):独立 harness(`tlsreload_test.cpp` + 真实 `TlsChannel.cpp`,锚 + 走 exe 同目录 `yama-ca.pem`)覆盖 6 个场景全部通过:无锚→明文、有效锚→TLS、坏锚首次 + 加载→fail-closed、无锚→下发锚后重载生效(明文→TLS)、坏锚重载→保持 TLS 不中断、撤销 + 锚→回到明文。Linux 侧的热重载此前已随 `TlsTrustAnchorSave/Remove` 实现。 +- 构建:`ghost` x64/x86 Debug、`ServerDll.dll`、MFC 服务端 Debug x64、Linux + `IOCPClient.cpp` / `TlsChannelLinux.cpp` 语法检查均通过。 + +## 8. 限制与非目标 + +- **MFC 服务端(`server/2015Remote`)未实现设备端口的 WS 嗅探**:本功能目前只对接 Go 版服务端。 + MFC 侧要支持需在它的连接建立处做同样的一层 sniff。生成界面照常提供这两个协议,但生成完成的 + 提示里已注明「服务端须为 Go 版本」,避免出包后才发现连不上。 +- 裸 `PROTO_WS` 是明文,只适合内网或回环(nginx 已在同一台机器上终止 TLS 的情形用 WSS 即可)。 +- macOS / Android 客户端没有 TLS 实现,只能选 `PROTO_WS`(明文),走不了 443。 +- 经反代接入时地址只应填**一个域名**(多地址取第一个做 Host/SNI)。 +- 不改应用协议、不改 MCP、不改 Web 控制台。 diff --git a/server/2015Remote/2015RemoteDlg.cpp b/server/2015Remote/2015RemoteDlg.cpp index d135e14..297b799 100644 --- a/server/2015Remote/2015RemoteDlg.cpp +++ b/server/2015Remote/2015RemoteDlg.cpp @@ -4799,10 +4799,10 @@ void CMy2015RemoteDlg::OnExperimentTlsEnable() return; } // 二次确认:避免误点就切换被控端 TLS。 - // 生效时机因客户端而异:Windows 写注册表、下次启动生效;Linux 落盘后立即热切换并 - // 断开重连(见 client/TlsChannelLinux.cpp 的 TlsReload),正在进行的远程会话会中断。 + // 两平台都是立即生效:客户端收到命令后落盘信任锚(Windows 注册表 / Linux 配置文件)、 + // 热重载 TLS 状态(各平台 TlsChannel 的 TlsReload)并断开重连,正在进行的远程会话会中断。 if (IDYES != MessageBoxL(_T("确定对选中主机启用 TLS 传输吗?\n" - "(Linux 客户端立即生效并断线重连;Windows 客户端下次启动生效)"), + "(客户端立即生效并断线重连,正在进行的远程会话会中断)"), _T("提示"), MB_ICONQUESTION | MB_YESNO)) return; // 读取服务端证书(信任锚),随启用指令一并下发;客户端写入注册表 settings\tls_cert。 @@ -4835,7 +4835,7 @@ void CMy2015RemoteDlg::OnExperimentTlsDisable() return; } if (IDYES != MessageBoxL(_T("确定取消选中主机的 TLS 传输吗?\n" - "(Linux 客户端立即生效并断线重连;Windows 客户端下次启动生效)"), + "(客户端立即生效并断线重连,正在进行的远程会话会中断)"), _T("提示"), MB_ICONQUESTION | MB_YESNO)) return; BYTE bToken = COMMAND_TLS_DISABLE; //向被控端发送停用 TLS 传输命令 SendSelectedCommand(&bToken, sizeof(BYTE)); diff --git a/server/2015Remote/BuildDlg.cpp b/server/2015Remote/BuildDlg.cpp index 1860a85..cfe09fe 100644 --- a/server/2015Remote/BuildDlg.cpp +++ b/server/2015Remote/BuildDlg.cpp @@ -609,6 +609,8 @@ void CBuildDlg::OnBnClickedOk() File.Close(); CString tip = index == IndexTestRun_DLL ? "\r\n" + _TR("提示: 请生成\"ServerDll.dll\",以便程序正常运行。") : _T(""); tip += g_ConnectAddress.protoType==PROTO_KCP ? "\n" + _TR("提示: 使用KCP协议生成服务,必须设置主控UDP协议参数为1。") : _T(""); + tip += (g_ConnectAddress.protoType==PROTO_WS || g_ConnectAddress.protoType==PROTO_WSS) ? + "\n" + _TR("提示: WebSocket协议需经反向代理(如nginx 443)转发到主控端口,地址填域名;服务端须为Go版本。") : _T(""); std::string upx; int sel = m_ComboCompress.GetCurSel(); if(sel == CLIENT_COMPRESS_UPX)upx = ReleaseUPX(); @@ -840,6 +842,9 @@ BOOL CBuildDlg::OnInitDialog() m_ComboProto.InsertStringL(PROTO_HTTP, "HTTP"); m_ComboProto.InsertStringL(PROTO_RANDOM, "随机"); m_ComboProto.InsertStringL(PROTO_KCP, "KCP"); + // 序号即 protoType,中间不能有空洞(下方 GetCurSel 直接当协议值用) + m_ComboProto.InsertStringL(PROTO_WS, "WebSocket"); + m_ComboProto.InsertStringL(PROTO_WSS, "WS+TLS"); m_ComboProto.SetCurSel(PROTO_TCP); m_ComboEncrypt.InsertStringL(PROTOCOL_SHINE, "Shine"); diff --git a/server/2015Remote/lang/de_DE.ini b/server/2015Remote/lang/de_DE.ini index 36800e5..1c86def 100644 --- a/server/2015Remote/lang/de_DE.ini +++ b/server/2015Remote/lang/de_DE.ini @@ -279,6 +279,7 @@ Web ͨÓÃ=Allgemein ¸߼¶ (·DZØÌîÏî)=Erweitert (Optional) Ìáʾ: ¶à¸öÉÏÏߵØַÓ÷ֺŷָô£¬99¸ö×ַûÒÔÄڡ£½ö¹©ѧϰºÍ×ÔÓã¬ÑϽûÓÃÓڷǷ¨ĿµÄʹÓá£=Tipp: Mehrere Adressen durch Semikolons getrennt, max. 99 Zeichen. +Ìáʾ: WebSocketЭÒéÐ辭·´Ïò´úÀí(Èçnginx 443)ת·¢µ½Ö÷¿ض˿ڣ¬µØַÌîÓòÃû£»·þÎñ¶ËÐëΪGo°汾¡£=Tipp: WebSocket benoetigt einen Reverse-Proxy (z. B. nginx auf 443), der auf den Steuerungsport weiterleitet; als Adresse die Domain eintragen; der Server muss die Go-Version sein. ; ============================================ ; Service Manager @@ -2110,8 +2111,8 @@ TLS ֤Êé/˽Կ¼ÓÔØʧ°ܣ¬¼ÓÃܹ¦ÄÜδÉúЧ¡£Çë¼ì²éÎļþ·¾¶Óë¸ñʽ£¨֤Êé PEM¡¢˽Կ PKCS#8£©¡£=Zertifikat/privater Schluessel konnte nicht geladen werden; Verschluesselung ist inaktiv. Bitte Pfad und Format pruefen (PEM-Zertifikat, PKCS#8-Schluessel). TLS´«Êä=TLS-Transport ȡÏûTLS=TLS deaktivieren -ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=TLS-Transport fuer die ausgewaehlten Hosts aktivieren?\n(Linux-Clients wechseln sofort und verbinden sich neu; Windows-Clients erst nach dem Neustart) -ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=TLS-Transport fuer die ausgewaehlten Hosts deaktivieren?\n(Linux-Clients wechseln sofort und verbinden sich neu; Windows-Clients erst nach dem Neustart) +ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=TLS-Transport fuer die ausgewaehlten Hosts aktivieren?\n(Clients wirken sofort und verbinden sich neu; laufende Sitzungen werden unterbrochen) +ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=TLS-Transport fuer die ausgewaehlten Hosts deaktivieren?\n(Clients wirken sofort und verbinden sich neu; laufende Sitzungen werden unterbrochen) ·þÎñ¶ËδÆôÓà TLS ֤Ê飬Î޷¨Çл»¿ͻ§¶Ë TLS ´«Êä=Server-TLS-Zertifikat ist nicht aktiviert; TLS-Transport der Clients kann nicht umgeschaltet werden ¶Áȡ·þÎñ¶Ë TLS ֤Êéʧ°ܣ¬Î޷¨Ï·¢=Server-TLS-Zertifikat konnte nicht gelesen werden; Verteilung nicht moeglich °×Ãûµ¥=Whitelist diff --git a/server/2015Remote/lang/en_US.ini b/server/2015Remote/lang/en_US.ini index 4003063..39e303c 100644 --- a/server/2015Remote/lang/en_US.ini +++ b/server/2015Remote/lang/en_US.ini @@ -282,6 +282,7 @@ Web ͨÓÃ=General ¸߼¶ (·DZØÌîÏî)=Advanced (Optional) Ìáʾ: ¶à¸öÉÏÏߵØַÓ÷ֺŷָô£¬99¸ö×ַûÒÔÄڡ£½ö¹©ѧϰºÍ×ÔÓã¬ÑϽûÓÃÓڷǷ¨ĿµÄʹÓá£=Tip: Multiple addresses separated by semicolons, max 99 chars. +Ìáʾ: WebSocketЭÒéÐ辭·´Ïò´úÀí(Èçnginx 443)ת·¢µ½Ö÷¿ض˿ڣ¬µØַÌîÓòÃû£»·þÎñ¶ËÐëΪGo°汾¡£=Tip: WebSocket requires a reverse proxy (e.g. nginx on 443) forwarding to the control port; enter the domain as the address, and run the Go server. ; ============================================ ; Service Manager @@ -2129,8 +2130,8 @@ TLS ֤Êé/˽Կ¼ÓÔØʧ°ܣ¬¼ÓÃܹ¦ÄÜδÉúЧ¡£Çë¼ì²éÎļþ·¾¶Óë¸ñʽ£¨֤Êé PEM¡¢˽Կ PKCS#8£©¡£=Failed to load certificate/private key; encryption is not active. Check the file path and format (PEM certificate, PKCS#8 private key). TLS´«Êä=TLS Transport ȡÏûTLS=Disable TLS -ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=Enable TLS transport for the selected host?\n(Linux clients switch immediately and reconnect; Windows clients apply it on next start) -ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=Disable TLS transport for the selected host?\n(Linux clients switch immediately and reconnect; Windows clients apply it on next start) +ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=Enable TLS transport for the selected host?\n(Clients apply it immediately and reconnect; any active remote session is interrupted) +ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=Disable TLS transport for the selected host?\n(Clients apply it immediately and reconnect; any active remote session is interrupted) ·þÎñ¶ËδÆôÓà TLS ֤Ê飬Î޷¨Çл»¿ͻ§¶Ë TLS ´«Êä=Server TLS certificate is not enabled; cannot toggle client TLS transport ¶Áȡ·þÎñ¶Ë TLS ֤Êéʧ°ܣ¬Î޷¨Ï·¢=Failed to read the server TLS certificate; cannot distribute °×Ãûµ¥=Whitelist diff --git a/server/2015Remote/lang/zh_TW.ini b/server/2015Remote/lang/zh_TW.ini index dedbf81..ba85080 100644 --- a/server/2015Remote/lang/zh_TW.ini +++ b/server/2015Remote/lang/zh_TW.ini @@ -282,6 +282,7 @@ Web ͨÓÃ=ͨÓà ¸߼¶ (·DZØÌîÏî)=ßMëA (·DZØÌîí—) Ìáʾ: ¶à¸öÉÏÏߵØַÓ÷ֺŷָô£¬99¸ö×ַûÒÔÄڡ£½ö¹©ѧϰºÍ×ÔÓã¬ÑϽûÓÃÓڷǷ¨ĿµÄʹÓá£=Ìáʾ: ¶à‚€ÉϾ€λַÓ÷Ö̖·ָô£¬99‚€×ÖԪÒԃȡ£ƒH¹©ŒWÁ•ºÍ×ÔÓ㬇À½ûÓÃ춷Ƿ¨ĿµÄʹÓᣠ+Ìáʾ: WebSocketЭÒéÐ辭·´Ïò´úÀí(Èçnginx 443)ת·¢µ½Ö÷¿ض˿ڣ¬µØַÌîÓòÃû£»·þÎñ¶ËÐëΪGo°汾¡£=Ìáʾ: WebSocket …f¶¨Ð轛·´Ïò´úÀí(Èç nginx 443)ÞD°lµ½Ö÷¿ØßB½Ӳº£¬λַՈÌî¾WÓ򣻷þ„նË횞é Go °汾¡£ ; ============================================ ; Service Manager @@ -2120,8 +2121,8 @@ TLS ֤Êé/˽Կ¼ÓÔØʧ°ܣ¬¼ÓÃܹ¦ÄÜδÉúЧ¡£Çë¼ì²éÎļþ·¾¶Óë¸ñʽ£¨֤Êé PEM¡¢˽Կ PKCS#8£©¡£=‘{×C/˽è€ÝdÈëʧ”¡£¬¼ÓÃܹ¦ÄÜδÉúЧ¡£Ո™z²é™n°¸·�½Åc¸ñʽ£¨‘{×C PEM¡¢˽è€ PKCS#8£©¡£ TLS´«Êä=TLS ‚÷ݔ ȡÏûTLS=ͣÓà TLS -ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=´_¶¨Œ¦ßxȡÖ÷™C†¢Óà TLS ‚÷ݔ†᣿\n£¨Linux ÓÑô¶ËÁ¢¼´ÉúЧ�K”ྀÖØßB£»Windows ÓÑô¶ËÏ´Ά¢„ÓÉúЧ£© -ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨Linux ¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£»Windows ¿ͻ§¶ËÏ´ÎÆô¶¯ÉúЧ£©=´_¶¨ȡÏûßxȡÖ÷™CµÄ TLS ‚÷ݔ†᣿\n£¨Linux ÓÑô¶ËÁ¢¼´ÉúЧ�K”ྀÖØßB£»Windows ÓÑô¶ËÏ´Ά¢„ÓÉúЧ£© +ȷ¶¨¶ÔѡÖÐÖ÷»úÆôÓà TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=´_¶¨Œ¦ßxȡÖ÷™C†¢Óà TLS ‚÷ݔ†᣿\n£¨ÓÑô¶ËÁ¢¼´ÉúЧ�K”ྀÖØßB£¬ÕýÔÚßMÐеÄßh¶˹¤×÷ëA¶ΕþÖДࣩ +ȷ¶¨ȡÏûѡÖÐÖ÷»úµÄ TLS ´«ÊäÂð£¿\n£¨¿ͻ§¶ËÁ¢¼´ÉúЧ²¢¶ÏÏßÖØÁ¬£¬ÕýÔڽøÐеÄԶ³̻Ự»áÖжϣ©=´_¶¨ȡÏûßxȡÖ÷™CµÄ TLS ‚÷ݔ†᣿\n£¨ÓÑô¶ËÁ¢¼´ÉúЧ�K”ྀÖØßB£¬ÕýÔÚßMÐеÄßh¶˹¤×÷ëA¶ΕþÖДࣩ ·þÎñ¶ËδÆôÓà TLS ֤Ê飬Î޷¨Çл»¿ͻ§¶Ë TLS ´«Êä=ËŷþÆ÷δ†¢Óà TLS ‘{×C£¬Ÿo·¨ÇГQÓÑô¶Ë TLS ‚÷ݔ ¶Áȡ·þÎñ¶Ë TLS ֤Êéʧ°ܣ¬Î޷¨Ï·¢=×xȡËŷþÆ÷ TLS ‘{×Cʧ”¡£¬Ÿo·¨Ï°l °×Ãûµ¥=°×Ãû†Î diff --git a/server/go/cmd/main.go b/server/go/cmd/main.go index ac23f72..15c3805 100644 --- a/server/go/cmd/main.go +++ b/server/go/cmd/main.go @@ -1153,6 +1153,22 @@ func main() { log.Info("ConnAuth HMAC verification enabled for device sub-connections (YAMA_CONN_AUTH_KEY)") } + // Optional WebSocket transport on the device port, for devices that can only + // reach the server through a reverse proxy on a shared port (nginx on 443). + // YAMA_WS_PATH restricts upgrades to one request path (empty = any path). + // YAMA_TRUSTED_PROXIES lists the IPs/CIDRs allowed to report a device's real + // address via X-Forwarded-For / X-Real-IP (default: loopback only). + wsPath := strings.TrimSpace(os.Getenv("YAMA_WS_PATH")) + trustedProxies := splitCSV(os.Getenv("YAMA_TRUSTED_PROXIES")) + if wsPath != "" { + log.Info("WebSocket transport on device port enabled: path=%s", wsPath) + } else { + log.Info("WebSocket transport on device port enabled: any path") + } + if len(trustedProxies) > 0 { + log.Info("Trusting forwarded client IPs from: %s", strings.Join(trustedProxies, ", ")) + } + // Create servers for each port var servers []*server.Server for _, port := range ports { @@ -1163,6 +1179,8 @@ func main() { config.PerIPConnRateLimit = envInt("YAMA_PERIP_CONN_RATE", config.PerIPConnRateLimit) config.TLSCertFile = tlsCert config.TLSKeyFile = tlsKey + config.WSPath = wsPath + config.TrustedProxies = trustedProxies srv := server.New(config) srv.SetLogger(log.WithPrefix(fmt.Sprintf("Server:%d", port))) diff --git a/server/go/connection/context.go b/server/go/connection/context.go index 1c14c4c..6d43a4f 100644 --- a/server/go/connection/context.go +++ b/server/go/connection/context.go @@ -61,6 +61,10 @@ type Context struct { closed atomic.Bool sendLock sync.Mutex server *Manager + + // peerIPOverride replaces the address derived from RemoteAddr when a + // trusted reverse proxy reports the device's real address. + peerIPOverride string } // FlagType represents the protocol flag type @@ -156,12 +160,28 @@ func (c *Context) IsClosed() bool { // GetPeerIP returns the peer IP address func (c *Context) GetPeerIP() string { + c.mu.RLock() + override := c.peerIPOverride + c.mu.RUnlock() + if override != "" { + return override + } if host, _, err := net.SplitHostPort(c.RemoteAddr); err == nil { return host } return c.RemoteAddr } +// SetPeerIP overrides the address reported by GetPeerIP. The server uses it +// when a trusted reverse proxy reports a device's real address, so per-IP +// accounting, logging and the ConnAuth sub-connection binding all see the +// device rather than the proxy. +func (c *Context) SetPeerIP(ip string) { + c.mu.Lock() + c.peerIPOverride = ip + c.mu.Unlock() +} + // AliveTime returns how long the connection has been alive func (c *Context) AliveTime() time.Duration { return time.Since(c.OnlineTime) diff --git a/server/go/server/server.go b/server/go/server/server.go index 6b51696..45ca1b3 100644 --- a/server/go/server/server.go +++ b/server/go/server/server.go @@ -42,6 +42,17 @@ type Config struct { PerIPConnRateLimit int // max new connections per PerIPConnRateWindow per IP (0 = unlimited) PerIPConnRateWindow time.Duration // window for PerIPConnRateLimit + // WebSocket transport. Devices that can only reach the server through a + // reverse proxy on a shared port (e.g. nginx on 443) upgrade to WebSocket; + // the device port then carries the unchanged app protocol inside binary + // frames. WSPath restricts the upgrade to one request path ("" = any). + WSPath string + + // TrustedProxies lists the IPs/CIDRs whose X-Forwarded-For / X-Real-IP + // headers are believed. Only connections from these peers are allowed to + // report a different device address; empty defaults to loopback. + TrustedProxies []string + // TLSCertFile / TLSKeyFile enable optional one-way TLS on the device port. // When both are set the server loads the cert and auto-detects TLS clients // by their ClientHello; plaintext clients keep working unchanged. Both empty @@ -87,6 +98,10 @@ type Server struct { // wrapTLS consults it to sniff and wrap incoming connections. tlsCfg *tls.Config + // trustedProxies are the peers allowed to report a device's real address + // via X-Forwarded-For / X-Real-IP (the reverse proxy in front of us). + trustedProxies []*net.IPNet + running atomic.Bool wg sync.WaitGroup ctx context.Context @@ -109,6 +124,7 @@ func New(config Config) *Server { cancel: cancel, logger: logger.New(logger.DefaultConfig()).WithPrefix("Server"), } + s.trustedProxies = parseTrustedProxies(config.TrustedProxies) return s } @@ -356,38 +372,85 @@ func (s *Server) acceptLoop() { continue } - // Check connection limit before spawning goroutine + // Check connection limit before spawning goroutine. if s.manager.Count() >= s.config.MaxConnections { s.logger.Warn("Max connections reached, rejecting new connection from %s", conn.RemoteAddr()) _ = conn.Close() continue } - // Per-IP connection cap + accept rate. Rejecting here (before any - // buffer allocation or goroutine) is cheap, and a rejected device - // simply retries on its reconnect loop. - if !s.ipLimit.allow(peerIP(conn.RemoteAddr())) { - s.logger.Warn("Per-IP connection limit reached, rejecting %s", conn.RemoteAddr()) + peer := peerIP(conn.RemoteAddr()) + counted, allowed := s.limitDirectPeer(peer) + if !allowed { + s.logger.Warn("Per-IP connection limit reached, rejecting %s", peer) _ = conn.Close() continue } // Handle each connection in its own goroutine - go s.handleConnection(conn) + go s.handleConnection(conn, counted) } } -// handleConnection handles a single connection -func (s *Server) handleConnection(rawConn net.Conn) { +// limitDirectPeer applies the per-IP cap and accept rate to a freshly accepted +// peer, before the goroutine is spawned and before any sniffing read: a peer +// that opens sockets and never sends a full ClientHello / upgrade would +// otherwise hold a goroutine and a reader for up to ReadTimeout each, with no +// limit at all. Peers that are trusted proxies are exempt and report +// counted=false — every device behind a proxy arrives from the proxy's +// address, so they are limited later by the address their proxy reports. +func (s *Server) limitDirectPeer(peer string) (counted, allowed bool) { + if s.trustedProxy(peer) { + return false, true + } + if !s.ipLimit.allow(peer) { + return false, false + } + return true, true +} + +// handleConnection handles a single connection. peerCapped reports whether +// acceptLoop already counted this peer against the per-IP limiter (direct +// peers only), in which case the count is released here when the connection +// ends. +func (s *Server) handleConnection(rawConn net.Conn, peerCapped bool) { + peer := peerIP(rawConn.RemoteAddr()) + if peerCapped { + defer s.ipLimit.release(peer) + } + // Sniff and (when TLS is enabled) wrap TLS peers; plaintext passes through. conn := s.wrapTLS(rawConn) + // Sniff for a WebSocket upgrade. Proxied devices (nginx on a shared port) + // arrive as a plaintext HTTP upgrade after the proxy terminated TLS. + conn, realIP := s.wrapWebSocket(conn, peer) + if conn == nil { + return + } + + if !peerCapped { + // Proxied device: limit by the forwarded address so one device behind + // the proxy cannot consume the whole budget for everyone else. + limitIP := peer + if realIP != "" { + limitIP = realIP + } + if !s.ipLimit.allow(limitIP) { + s.logger.Warn("Per-IP connection limit reached, rejecting %s (peer %s)", limitIP, peer) + _ = conn.Close() + return + } + defer s.ipLimit.release(limitIP) + } + // Create context ctx := connection.NewContext(conn, nil) - - // Always release the per-IP slot on every exit path, including the - // manager.Add failure below (a rare accept/add race). - defer s.ipLimit.release(ctx.GetPeerIP()) + if realIP != "" { + // Report the device's real address everywhere GetPeerIP is consulted + // (per-IP accounting, logging, ConnAuth sub-connection binding). + ctx.SetPeerIP(realIP) + } // Add to manager if err := s.manager.Add(ctx); err != nil { diff --git a/server/go/server/server_test.go b/server/go/server/server_test.go index 7a88855..bd8b2cb 100644 --- a/server/go/server/server_test.go +++ b/server/go/server/server_test.go @@ -93,6 +93,35 @@ func TestPeerIP(t *testing.T) { } } +// The per-IP cap must apply to direct peers before any sniffing read, so a +// peer that connects and stays silent cannot hold unbounded goroutines and +// readers. Peers that are trusted proxies are exempt (and counted later by the +// forwarded address), because all devices behind a proxy share its address. +func TestLimitDirectPeerBeforeSniff(t *testing.T) { + cfg := DefaultConfig() + cfg.PerIPMaxConnections = 1 + cfg.PerIPConnRateLimit = 0 // concurrent cap only + s := New(cfg) + + if counted, ok := s.limitDirectPeer("203.0.113.5"); !ok || !counted { + t.Fatalf("first direct peer: counted=%v ok=%v, want true/true", counted, ok) + } + // A second silent socket from the same address must be refused outright. + if counted, ok := s.limitDirectPeer("203.0.113.5"); ok || counted { + t.Fatalf("second direct peer: counted=%v ok=%v, want false/false", counted, ok) + } + // Loopback is a trusted proxy by default: exempt from the peer cap here. + if counted, ok := s.limitDirectPeer("127.0.0.1"); !ok || counted { + t.Fatalf("trusted proxy: counted=%v ok=%v, want false/true", counted, ok) + } + + // Releasing the slot frees capacity again. + s.ipLimit.release("203.0.113.5") + if _, ok := s.limitDirectPeer("203.0.113.5"); !ok { + t.Fatal("after release the address should be allowed again") + } +} + // A peer that sends bytes which never assemble into a complete packet (here an // HTTP-mask envelope with no terminating CRLF CRLF) must be disconnected once // the per-connection buffer cap is exceeded, rather than allowed to grow it @@ -110,7 +139,8 @@ func TestHandleConnectionBufferCap(t *testing.T) { done := make(chan struct{}) go func() { - srv.handleConnection(serverConn) + // false: acceptLoop was bypassed, so the peer was not pre-counted. + srv.handleConnection(serverConn, false) close(done) }() diff --git a/server/go/server/ws.go b/server/go/server/ws.go new file mode 100644 index 0000000..96ebbf3 --- /dev/null +++ b/server/go/server/ws.go @@ -0,0 +1,517 @@ +package server + +import ( + "bufio" + "bytes" + "crypto/sha1" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "strings" + "sync" + "time" +) + +// WebSocket transport for devices that can only reach the server through a +// reverse proxy (nginx) on a shared port such as 443. +// +// The app protocol is unchanged: the WS layer is a transparent byte-stream +// tunnel. A device connects to the proxy with a normal WebSocket upgrade, +// nginx terminates TLS and proxies the (now plaintext) upgrade to the device +// port, and this file completes the handshake and carries the existing parser +// traffic inside binary frames. +// +// Upgrade request detection happens after the TLS sniff in handleConnection, +// so a proxied wss connection (nginx terminates TLS) and a direct wss +// connection (YAMA_TLS_CERT set, client speaks wss end to end) both work. + +const ( + wsMagicGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + wsOpContinuation = 0x0 + wsOpText = 0x1 + wsOpBinary = 0x2 + wsOpClose = 0x8 + wsOpPing = 0x9 + wsOpPong = 0xA + + // wsMaxHandshakeBytes caps the upgrade request head we are willing to + // buffer before rejecting the peer. + wsMaxHandshakeBytes = 8 * 1024 + + // wsHandshakeTimeout bounds reading the request head once "GET " was seen. + wsHandshakeTimeout = 10 * time.Second +) + +var ( + errWSProtocol = errors.New("websocket: protocol error") + errWSHandshake = errors.New("websocket: bad handshake") + + // wsUpgradePrefix identifies an HTTP request whose first token is GET. The + // app protocol never starts a connection with these bytes (its flags are + // "Shine"/"HELL"/"POST" variants), so this is unambiguous. + wsUpgradePrefix = []byte("GET ") +) + +// wrapWebSocket sniffs the already TLS-sniffed connection for an HTTP +// WebSocket upgrade. When the peer is not upgrading, conn is returned +// unchanged. When the upgrade completes, the returned net.Conn frames all +// reads/writes, and realIP carries the client address reported by a trusted +// reverse proxy (empty otherwise). A failed upgrade returns (nil, ""). +func (s *Server) wrapWebSocket(conn net.Conn, peer string) (net.Conn, string) { + br := bufio.NewReader(conn) + if s.config.ReadTimeout > 0 { + _ = conn.SetReadDeadline(time.Now().Add(s.config.ReadTimeout)) + } + hdr, err := br.Peek(len(wsUpgradePrefix)) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + // Peer never sent enough to identify itself; let the normal read loop + // surface EOF/timeout exactly as it would without the sniff. + return &bufferedConn{Conn: conn, r: br}, "" + } + if !bytes.Equal(hdr, wsUpgradePrefix) { + return &bufferedConn{Conn: conn, r: br}, "" + } + + req, err := readWSRequest(br, conn) + if err != nil { + s.logger.Debug("wrapWebSocket: reject %s: %v", conn.RemoteAddr(), err) + writeWSReject(conn, err) + _ = conn.Close() + return nil, "" + } + + if s.config.WSPath != "" && req.path != s.config.WSPath { + s.logger.Debug("wrapWebSocket: reject %s: path %q != %q", conn.RemoteAddr(), req.path, s.config.WSPath) + writeWSReject(conn, errWSProtocol) + _ = conn.Close() + return nil, "" + } + + accept := wsAcceptKey(req.key) + _ = conn.SetWriteDeadline(time.Now().Add(wsHandshakeTimeout)) + resp := "HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + accept + "\r\n\r\n" + _, werr := io.WriteString(conn, resp) + _ = conn.SetWriteDeadline(time.Time{}) + if werr != nil { + _ = conn.Close() + return nil, "" + } + + realIP := "" + if s.trustedProxy(peer) { + realIP = reqForwardedIP(req.headers) + } + if realIP != "" { + s.logger.Info("WebSocket device from %s (forwarded by %s)", realIP, peer) + } else { + s.logger.Info("WebSocket device from %s", peer) + } + return newWSConn(conn, br), realIP +} + +// wsRequest is the parsed subset of an upgrade request the server needs. +type wsRequest struct { + path string + key string + headers map[string]string // lower-cased names; repeated headers joined by ", " +} + +// readWSRequest reads and validates the HTTP upgrade head. The buffered reader +// is left positioned at the first byte after the head so any frames the peer +// pipelined behind the request are preserved. +func readWSRequest(br *bufio.Reader, conn net.Conn) (*wsRequest, error) { + _ = conn.SetReadDeadline(time.Now().Add(wsHandshakeTimeout)) + defer func() { _ = conn.SetReadDeadline(time.Time{}) }() + + total := 0 + line, err := readWSLine(br, &total) + if err != nil { + return nil, err + } + parts := strings.Fields(line) + if len(parts) != 3 || parts[0] != "GET" || !strings.HasPrefix(parts[2], "HTTP/1.") { + return nil, errWSHandshake + } + req := &wsRequest{headers: make(map[string]string)} + req.path = parts[1] + if i := strings.IndexByte(req.path, '?'); i >= 0 { + req.path = req.path[:i] + } + + for { + line, err = readWSLine(br, &total) + if err != nil { + return nil, err + } + if line == "" { + break + } + name, value, ok := strings.Cut(line, ":") + if !ok { + return nil, errWSHandshake + } + name = strings.ToLower(strings.TrimSpace(name)) + value = strings.TrimSpace(value) + if prev, dup := req.headers[name]; dup { + req.headers[name] = prev + ", " + value + } else { + req.headers[name] = value + } + } + + if !headerHasToken(req.headers["upgrade"], "websocket") || + !headerHasToken(req.headers["connection"], "upgrade") { + return nil, errWSHandshake + } + req.key = req.headers["sec-websocket-key"] + if req.key == "" { + return nil, errWSHandshake + } + return req, nil +} + +// readWSLine reads one CRLF-terminated line, rejecting the peer as soon as the +// accumulated head exceeds wsMaxHandshakeBytes. The input is peer-controlled: +// a plain ReadString would buffer a newline-free line without limit. +func readWSLine(br *bufio.Reader, total *int) (string, error) { + var line []byte + for { + chunk, err := br.ReadSlice('\n') + line = append(line, chunk...) + *total += len(chunk) + if *total > wsMaxHandshakeBytes { + return "", errWSProtocol + } + if err == bufio.ErrBufferFull { + continue + } + if err != nil { + return "", err + } + return strings.TrimRight(string(line), "\r\n"), nil + } +} + +// headerHasToken reports whether a comma-separated header value contains token +// (case-insensitive), as required for Connection: keep-alive, Upgrade. +func headerHasToken(value, token string) bool { + for _, part := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(part), token) { + return true + } + } + return false +} + +// reqForwardedIP extracts the device's real address from a proxy's forwarded +// headers. The LAST X-Forwarded-For entry wins, not the first: nginx's +// $proxy_add_x_forwarded_for appends the address it accepted the connection +// from, so whatever a client prepends is ignored — the same rule web/ws.go's +// clientIP applies. Trusting the first entry would let a device rotate its key +// through the per-IP budget, the login throttle and the ConnAuth peer binding. +// X-Real-IP is only a fallback for proxies configured to set that and not XFF. +// Only IP literals are accepted, so a header cannot inject an arbitrary peer key. +func reqForwardedIP(headers map[string]string) string { + if xff := headers["x-forwarded-for"]; xff != "" { + last := strings.TrimSpace(xff[strings.LastIndexByte(xff, ',')+1:]) + if ip := headerIP(last); ip != "" { + return ip + } + } + return headerIP(headers["x-real-ip"]) +} + +func headerIP(s string) string { + s = strings.TrimSpace(s) + if i := strings.LastIndexByte(s, ':'); i > 0 && strings.Count(s, ":") == 1 { + // strip an IPv4:port pair, but leave bare IPv6 literals alone + if net.ParseIP(s[:i]) != nil { + s = s[:i] + } + } + if ip := net.ParseIP(s); ip != nil { + return ip.String() + } + return "" +} + +// wsAcceptKey computes Sec-WebSocket-Accept: base64(SHA1(key + magic GUID)). +func wsAcceptKey(key string) string { + h := sha1.New() + io.WriteString(h, key) + io.WriteString(h, wsMagicGUID) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func writeWSReject(conn net.Conn, reason error) { + status := "400 Bad Request" + body := "bad request" + if errors.Is(reason, errWSProtocol) { + body = "protocol error" + } + _ = conn.SetWriteDeadline(time.Now().Add(wsHandshakeTimeout)) + fmt.Fprintf(conn, "HTTP/1.1 %s\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", + status, len(body), body) +} + +// trustedProxy reports whether peer is allowed to set the device's real IP via +// forwarded headers. Without this check any direct client could spoof its +// address and, with it, the per-IP limits and the ConnAuth peer binding. +func (s *Server) trustedProxy(peer string) bool { + ip := net.ParseIP(peer) + if ip == nil { + return false + } + for _, n := range s.trustedProxies { + if n.Contains(ip) { + return true + } + } + return false +} + +// parseTrustedProxies converts configured IPs/CIDRs into networks. An empty or +// unparsable list falls back to loopback, the only sensible default for a +// proxy running on the same host as the server. +func parseTrustedProxies(list []string) []*net.IPNet { + var nets []*net.IPNet + for _, item := range list { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if !strings.Contains(item, "/") { + if ip := net.ParseIP(item); ip != nil { + bits := 32 + if ip.To4() == nil { + bits = 128 + } + item = fmt.Sprintf("%s/%d", ip.String(), bits) + } + } + if _, n, err := net.ParseCIDR(item); err == nil { + nets = append(nets, n) + } + } + if len(nets) == 0 { + _, v4, _ := net.ParseCIDR("127.0.0.0/8") + _, v6, _ := net.ParseCIDR("::1/128") + nets = append(nets, v4, v6) + } + return nets +} + +// wsConn presents a WebSocket connection as a net.Conn byte stream: each Write +// becomes one binary frame and Read returns the concatenated payload of data +// frames, transparently answering pings and closes. Frame payloads are streamed +// rather than buffered whole, so a multi-megabyte upload allocates nothing here. +type wsConn struct { + conn net.Conn + r *bufio.Reader + + writeMu sync.Mutex + closeMu sync.Mutex + closed bool + + // state of the data frame currently being read + remaining int64 + masked bool + mask [4]byte + maskPos byte +} + +func newWSConn(conn net.Conn, r *bufio.Reader) *wsConn { + return &wsConn{conn: conn, r: r} +} + +// Read implements net.Conn. +func (c *wsConn) Read(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + for { + if c.remaining > 0 { + n := len(p) + if int64(n) > c.remaining { + n = int(c.remaining) + } + nr, err := io.ReadFull(c.r, p[:n]) + if nr > 0 { + if c.masked { + for i := 0; i < nr; i++ { + p[i] ^= c.mask[c.maskPos] + c.maskPos = (c.maskPos + 1) & 3 + } + } + c.remaining -= int64(nr) + } + if err != nil { + return nr, err + } + return nr, nil + } + + opcode, fin, length, masked, err := c.readFrameHeader() + if err != nil { + return 0, err + } + if opcode >= 0x8 { + if !fin || length > 125 { + return 0, errWSProtocol + } + payload, err := c.readControlPayload(int(length), masked) + if err != nil { + return 0, err + } + switch opcode { + case wsOpClose: + _ = c.writeFrame(wsOpClose, payload[:min(len(payload), 125)]) + return 0, io.EOF + case wsOpPing: + if err := c.writeFrame(wsOpPong, payload); err != nil { + return 0, err + } + case wsOpPong: + // ignore + default: + return 0, errWSProtocol + } + continue + } + + switch opcode { + case wsOpBinary, wsOpText, wsOpContinuation: + default: + return 0, errWSProtocol + } + if length == 0 { + continue + } + c.remaining = length + c.masked = masked + c.maskPos = 0 + } +} + +// readFrameHeader reads and validates a frame header. Frames from the client +// are normally masked; unmasked data frames are tolerated because the only +// speakers on this port are YAMA's own clients. +func (c *wsConn) readFrameHeader() (opcode byte, fin bool, length int64, masked bool, err error) { + var hdr [2]byte + if _, err = io.ReadFull(c.r, hdr[:]); err != nil { + return + } + if hdr[0]&0x70 != 0 { + return 0, false, 0, false, errWSProtocol // RSV bits set without a negotiated extension + } + fin = hdr[0]&0x80 != 0 + opcode = hdr[0] & 0x0f + masked = hdr[1]&0x80 != 0 + length = int64(hdr[1] & 0x7f) + + switch length { + case 126: + var ext [2]byte + if _, err = io.ReadFull(c.r, ext[:]); err != nil { + return + } + length = int64(binary.BigEndian.Uint16(ext[:])) + case 127: + var ext [8]byte + if _, err = io.ReadFull(c.r, ext[:]); err != nil { + return + } + u := binary.BigEndian.Uint64(ext[:]) + if u > 1<<62 { + return 0, false, 0, false, errWSProtocol + } + length = int64(u) + } + + if masked { + if _, err = io.ReadFull(c.r, c.mask[:]); err != nil { + return + } + } + return opcode, fin, length, masked, nil +} + +func (c *wsConn) readControlPayload(n int, masked bool) ([]byte, error) { + payload := make([]byte, n) + if _, err := io.ReadFull(c.r, payload); err != nil { + return nil, err + } + if masked { + for i := range payload { + payload[i] ^= c.mask[i&3] + } + } + return payload, nil +} + +// Write implements net.Conn: one binary frame per call. The app protocol's +// sends are already whole packets (Context.Send), so this keeps writes atomic +// and lets the peer deliver each packet as it was produced. +func (c *wsConn) Write(p []byte) (int, error) { + if err := c.writeFrame(wsOpBinary, p); err != nil { + return 0, err + } + return len(p), nil +} + +// writeFrame emits one FIN frame with the given opcode. +func (c *wsConn) writeFrame(opcode byte, payload []byte) error { + var hdr [10]byte + hdr[0] = 0x80 | opcode + n := 2 + switch { + case len(payload) < 126: + hdr[1] = byte(len(payload)) + case len(payload) <= 0xffff: + hdr[1] = 126 + binary.BigEndian.PutUint16(hdr[2:], uint16(len(payload))) + n = 4 + default: + hdr[1] = 127 + binary.BigEndian.PutUint64(hdr[2:], uint64(len(payload))) + n = 10 + } + + c.writeMu.Lock() + defer c.writeMu.Unlock() + bufs := net.Buffers{hdr[:n], payload} + if _, err := bufs.WriteTo(c.conn); err != nil { + return err + } + return nil +} + +// Close sends a best-effort close frame and closes the transport. +func (c *wsConn) Close() error { + c.closeMu.Lock() + if c.closed { + c.closeMu.Unlock() + return nil + } + c.closed = true + c.closeMu.Unlock() + + _ = c.conn.SetWriteDeadline(time.Now().Add(time.Second)) + _ = c.writeFrame(wsOpClose, nil) + return c.conn.Close() +} + +func (c *wsConn) LocalAddr() net.Addr { return c.conn.LocalAddr() } +func (c *wsConn) RemoteAddr() net.Addr { return c.conn.RemoteAddr() } + +func (c *wsConn) SetDeadline(t time.Time) error { return c.conn.SetDeadline(t) } +func (c *wsConn) SetReadDeadline(t time.Time) error { return c.conn.SetReadDeadline(t) } +func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) } diff --git a/server/go/server/ws_test.go b/server/go/server/ws_test.go new file mode 100644 index 0000000..8e48f40 --- /dev/null +++ b/server/go/server/ws_test.go @@ -0,0 +1,407 @@ +package server + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + "net" + "strings" + "testing" + "time" +) + +// wsTestPair dials a loopback listener, runs the server side of wrapWebSocket +// on the accepted connection, performs the upgrade handshake, and returns both +// ends. The loopback peer makes the test connection a trusted proxy, so the +// forwarded-address path is exercised too. +func wsTestPair(t *testing.T, s *Server, path string, extra map[string]string) (*wsConn, net.Conn, *bufio.Reader, string, string) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + + type result struct { + conn net.Conn + ip string + } + resCh := make(chan result, 1) + go func() { + srv, err := ln.Accept() + if err != nil { + resCh <- result{} + return + } + conn, ip := s.wrapWebSocket(srv, peerIP(srv.RemoteAddr())) + resCh <- result{conn, ip} + }() + + cli, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("dial: %v", err) + } + t.Cleanup(func() { cli.Close() }) + _ = cli.SetDeadline(time.Now().Add(10 * time.Second)) + + // Write the upgrade request concurrently: the server blocks in its sniff + // until the first bytes arrive. + var b strings.Builder + b.WriteString("GET " + path + " HTTP/1.1\r\n") + b.WriteString("Host: yama.example.com\r\n") + b.WriteString("Upgrade: websocket\r\n") + b.WriteString("Connection: Upgrade\r\n") + b.WriteString("Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n") + b.WriteString("Sec-WebSocket-Version: 13\r\n") + for k, v := range extra { + b.WriteString(k + ": " + v + "\r\n") + } + b.WriteString("\r\n") + go func() { _, _ = io.WriteString(cli, b.String()) }() + + br := bufio.NewReader(cli) + head := &strings.Builder{} + for { + line, err := br.ReadString('\n') + if err != nil { + t.Fatalf("read response: %v", err) + } + head.WriteString(line) + if line == "\r\n" || line == "\n" { + break + } + } + + res := <-resCh + if res.conn == nil { + t.Fatalf("wrapWebSocket rejected the connection:\n%s", head.String()) + } + t.Cleanup(func() { res.conn.Close() }) + wc, ok := res.conn.(*wsConn) + if !ok { + t.Fatalf("wrapWebSocket returned %T, want *wsConn", res.conn) + } + _ = wc.SetDeadline(time.Now().Add(10 * time.Second)) + return wc, cli, br, head.String(), res.ip +} + +// clientFrame builds a masked client frame (the only form a server should see). +func clientFrame(opcode byte, payload []byte) []byte { + var hdr []byte + hdr = append(hdr, 0x80|opcode) // FIN + opcode + mask := [4]byte{0x37, 0xfa, 0x21, 0x3d} + n := len(payload) + switch { + case n < 126: + hdr = append(hdr, byte(n)|0x80) + case n <= 0xffff: + hdr = append(hdr, 126|0x80, byte(n>>8), byte(n)) + default: + hdr = append(hdr, 127|0x80) + var ext [8]byte + binary.BigEndian.PutUint64(ext[:], uint64(n)) + hdr = append(hdr, ext[:]...) + } + hdr = append(hdr, mask[:]...) + body := make([]byte, n) + for i := range payload { + body[i] = payload[i] ^ mask[i&3] + } + return append(hdr, body...) +} + +// serverFrame reads one unmasked frame produced by the server. +func serverFrame(t *testing.T, br *bufio.Reader) (byte, []byte) { + t.Helper() + var hdr [2]byte + if _, err := io.ReadFull(br, hdr[:]); err != nil { + t.Fatalf("read frame header: %v", err) + } + n := int(hdr[1] & 0x7f) + switch n { + case 126: + var ext [2]byte + if _, err := io.ReadFull(br, ext[:]); err != nil { + t.Fatalf("read ext len: %v", err) + } + n = int(binary.BigEndian.Uint16(ext[:])) + case 127: + var ext [8]byte + if _, err := io.ReadFull(br, ext[:]); err != nil { + t.Fatalf("read ext len: %v", err) + } + n = int(binary.BigEndian.Uint64(ext[:])) + } + if hdr[1]&0x80 != 0 { + t.Fatalf("server frame is masked") + } + payload := make([]byte, n) + if _, err := io.ReadFull(br, payload); err != nil { + t.Fatalf("read frame payload: %v", err) + } + return hdr[0] & 0x0f, payload +} + +func TestWebSocketUpgradeAndTunnel(t *testing.T) { + s := New(DefaultConfig()) + // A device that prepends its own XFF entry must not be believed: nginx + // appends the address it accepted the connection from on the right. + wc, cli, br, head, realIP := wsTestPair(t, s, "/api/socket", map[string]string{ + "X-Forwarded-For": "203.0.113.9, 198.51.100.20", + }) + if !strings.HasPrefix(head, "HTTP/1.1 101 ") { + t.Fatalf("unexpected response head:\n%s", head) + } + // RFC 6455 sample key -> known accept value. + if !strings.Contains(head, "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=") { + t.Fatalf("missing/incorrect accept key:\n%s", head) + } + if realIP != "198.51.100.20" { + t.Fatalf("forwarded real IP = %q, want 198.51.100.20 (last XFF entry)", realIP) + } + + // Client -> server: app bytes arrive as a plain stream. + payload := []byte("Shine\x00\x00\x00\x03abc") + if _, err := cli.Write(clientFrame(wsOpBinary, payload)); err != nil { + t.Fatalf("write frame: %v", err) + } + got := make([]byte, len(payload)) + if _, err := io.ReadFull(wc, got); err != nil { + t.Fatalf("read tunneled bytes: %v", err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("tunneled bytes = %q, want %q", got, payload) + } + + // Server -> client: each Write becomes one binary frame. + if _, err := wc.Write([]byte("pong-data")); err != nil { + t.Fatalf("write: %v", err) + } + opcode, body := serverFrame(t, br) + if opcode != wsOpBinary || string(body) != "pong-data" { + t.Fatalf("server frame = opcode %d payload %q", opcode, body) + } +} + +func TestWebSocketPingPongAndClose(t *testing.T) { + s := New(DefaultConfig()) + wc, cli, br, _, _ := wsTestPair(t, s, "/x", nil) + + // A ping is answered with a pong carrying the same payload, and the app + // stream is unaffected. + if _, err := cli.Write(clientFrame(wsOpPing, []byte("hi"))); err != nil { + t.Fatalf("write ping: %v", err) + } + readDone := make(chan error, 1) + go func() { + buf := make([]byte, 8) + _, err := wc.Read(buf) + readDone <- err + }() + opcode, body := serverFrame(t, br) + if opcode != wsOpPong || string(body) != "hi" { + t.Fatalf("pong = opcode %d payload %q", opcode, body) + } + + // A close frame ends the pending Read with EOF and is echoed. + if _, err := cli.Write(clientFrame(wsOpClose, []byte{0x03, 0xe8})); err != nil { + t.Fatalf("write close: %v", err) + } + if err := <-readDone; err != io.EOF { + t.Fatalf("Read after close = %v, want io.EOF", err) + } + opcode, _ = serverFrame(t, br) + if opcode != wsOpClose { + t.Fatalf("close echo opcode = %d", opcode) + } +} + +func TestWebSocketLargeFrameStreams(t *testing.T) { + s := New(DefaultConfig()) + wc, cli, _, _, _ := wsTestPair(t, s, "/x", nil) + + payload := bytes.Repeat([]byte("0123456789ABCDEF"), 64*1024) // 1 MiB + frame := clientFrame(wsOpBinary, payload) + go func() { _, _ = cli.Write(frame) }() + + got := make([]byte, len(payload)) + if _, err := io.ReadFull(wc, got); err != nil { + t.Fatalf("read large frame: %v", err) + } + if !bytes.Equal(got, payload) { + t.Fatalf("large frame corrupted") + } +} + +func TestWebSocketRejectsBadHandshake(t *testing.T) { + tests := []struct { + name string + req string + path string + allow bool + }{ + { + name: "not an upgrade", + req: "GET / HTTP/1.1\r\nHost: h\r\n\r\n", + }, + { + name: "missing key", + req: "GET / HTTP/1.1\r\nHost: h\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n", + }, + { + name: "wrong path", + req: "GET /other HTTP/1.1\r\nHost: h\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n", + path: "/api/socket", + }, + { + name: "path match with query string", + req: "GET /api/socket?v=1 HTTP/1.1\r\nHost: h\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n", + path: "/api/socket", + allow: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := DefaultConfig() + cfg.WSPath = tt.path + s := New(cfg) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + + resCh := make(chan net.Conn, 1) + go func() { + srv, err := ln.Accept() + if err != nil { + resCh <- nil + return + } + conn, _ := s.wrapWebSocket(srv, peerIP(srv.RemoteAddr())) + resCh <- conn + }() + + cli, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer cli.Close() + _ = cli.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err := io.WriteString(cli, tt.req); err != nil { + t.Fatalf("write: %v", err) + } + + conn := <-resCh + if tt.allow { + if conn == nil { + t.Fatalf("connection rejected, want accepted") + } + return + } + if conn != nil { + t.Fatalf("connection accepted, want rejected") + } + br := bufio.NewReader(cli) + line, err := br.ReadString('\n') + if err != nil { + t.Fatalf("read reject response: %v", err) + } + if !strings.HasPrefix(line, "HTTP/1.1 400 ") { + t.Fatalf("reject status = %q", strings.TrimSpace(line)) + } + }) + } +} + +func TestWSForwardedIP(t *testing.T) { + tests := []struct { + name string + headers map[string]string + want string + }{ + {"x-real-ip", map[string]string{"x-real-ip": "198.51.100.7"}, "198.51.100.7"}, + {"xff single", map[string]string{"x-forwarded-for": "198.51.100.8"}, "198.51.100.8"}, + // The rightmost entry is the one the trusted proxy appended; a device + // that prepends a made-up address must not be able to choose its key. + {"xff chain takes rightmost", map[string]string{"x-forwarded-for": "198.51.100.9, 10.0.0.1, 10.0.0.2"}, "10.0.0.2"}, + {"spoofed xff head ignored", map[string]string{"x-forwarded-for": "1.2.3.4, 198.51.100.10"}, "198.51.100.10"}, + {"xff wins over x-real-ip", map[string]string{"x-real-ip": "198.51.100.7", "x-forwarded-for": "198.51.100.9"}, "198.51.100.9"}, + {"garbage rightmost falls back", map[string]string{"x-forwarded-for": "198.51.100.11, not-an-ip"}, ""}, + {"x-real-ip fallback", map[string]string{"x-real-ip": "198.51.100.7", "x-forwarded-for": "not-an-ip"}, "198.51.100.7"}, + {"v4 with port", map[string]string{"x-real-ip": "198.51.100.7:5555"}, "198.51.100.7"}, + {"ipv6 literal", map[string]string{"x-real-ip": "2001:db8::1"}, "2001:db8::1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := reqForwardedIP(tt.headers); got != tt.want { + t.Errorf("reqForwardedIP(%v) = %q, want %q", tt.headers, got, tt.want) + } + }) + } +} + +// A header line that never ends must be rejected as soon as the handshake +// budget is exhausted, not buffered until the process runs out of memory. +func TestReadWSLineBounded(t *testing.T) { + br := bufio.NewReader(bytes.NewReader(bytes.Repeat([]byte{'A'}, 1<<20))) + total := 0 + _, err := readWSLine(br, &total) + if !errors.Is(err, errWSProtocol) { + t.Fatalf("err = %v, want errWSProtocol", err) + } + if total > wsMaxHandshakeBytes+br.Size() { + t.Fatalf("read %d bytes, want at most %d", total, wsMaxHandshakeBytes+br.Size()) + } + + // A normal line still parses. + br = bufio.NewReader(strings.NewReader("GET /api/ws HTTP/1.1\r\n")) + total = 0 + line, err := readWSLine(br, &total) + if err != nil || line != "GET /api/ws HTTP/1.1" { + t.Fatalf("line = %q, err = %v", line, err) + } +} + +func TestTrustedProxies(t *testing.T) { + s := New(DefaultConfig()) + if !s.trustedProxy("127.0.0.1") || !s.trustedProxy("::1") { + t.Fatal("loopback must be trusted by default") + } + if s.trustedProxy("203.0.113.9") { + t.Fatal("public peer must not be trusted by default") + } + + cfg := DefaultConfig() + cfg.TrustedProxies = []string{"10.0.0.0/8", "203.0.113.9"} + s = New(cfg) + if !s.trustedProxy("10.1.2.3") || !s.trustedProxy("203.0.113.9") { + t.Fatal("configured proxies must be trusted") + } + if s.trustedProxy("127.0.0.1") { + t.Fatal("an explicit list replaces the loopback defaults") + } +} + +func TestHeaderHasToken(t *testing.T) { + tests := []struct { + value string + token string + want bool + }{ + {"Upgrade", "upgrade", true}, + {"keep-alive, Upgrade", "upgrade", true}, + {"upgrade, keep-alive", "upgrade", true}, + {"keep-alive", "upgrade", false}, + {"", "upgrade", false}, + } + for _, tt := range tests { + if got := headerHasToken(tt.value, tt.token); got != tt.want { + t.Errorf("headerHasToken(%q, %q) = %v, want %v", tt.value, tt.token, got, tt.want) + } + } +}