Three of the findings only bite in deployment. First, the client's TLS and WebSocket handshakes run synchronously on the reconnect thread with no socket timeout, so a peer that accepts TCP and then goes quiet (black-holed proxy upstream, middlebox that only receives) blocks recv() forever and stalls the reconnect loop until the process is restarted. Set a 15 s SO_RCVTIMEO and SO_SNDTIMEO for the handshake window - the same value both server handshake threads use - and clear them once the connection is up so the worker thread's select plus blocking recv keeps working as before. Second, per-IP rate limiting counted the getpeername address before the real device was known. Behind a same-host reverse proxy that address is loopback, which IPWhitelist always trusts, so WS traffic was neither counted nor banned; behind a remote proxy every device shares one bucket, and tripping the threshold disconnected all of them. Count and ban by the forwarded device IP once the upgrade resolves it, mirroring the Go server's limitDirectPeer and handleConnection split. Direct connections keep their accept-time count and are not counted twice. Third, the protocol switch disarmed its five-minute auto-revert as soon as the new entry connected, which only proves TCP/TLS/WS handshaking finished. If login is never accepted there (wrong backend, reverse proxy pointing at some other WS service), the revert could never fire and the client would keep retrying a dead entry. Disarm on the first CMD_MASTERSETTING instead, which the server only sends after adding the host to its online list, and re-check it on every heartbeat so a late one still counts. The smaller items came along: free the handshake parameter when CreateThread fails in both handshake starters, move the WS_Handshaking fail-closed guard ahead of the TLS branch so it also covers WSS (where tlsState is already TLS_Ready), generate the frame mask from a per-channel PRNG inside the send lock instead of an unseeded global rand(), make the transport flags and the WS session's ready flag atomic, drop rather than send unframed application bytes when a WS session has closed, and abort a protocol switch whose PEM fails to load instead of moving to an entry that cannot connect. The Go server now bounds the upgrade sniff even when ReadTimeout is zero, and the design doc records the two-stage limiter, the login-based disarm and the review items that stay as known limits. The protocol switch's input prompt is cut down to "reverse-proxy entry (host:port)" - the label static is only 81 dialog units wide, and the old sentence ran to roughly three times that. The three language files follow: en_US and zh_TW gain the shortened prompt and drop the long key it replaced, the German file gains the protocol-switch block it never had, and its WebSocket tip key is corrected - it had picked up a Go-server clause the source string never carried, so the tip was never translated at all. Co-Authored-By: deepseek-flash
524 lines
15 KiB
Go
524 lines
15 KiB
Go
package server
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"crypto/sha1"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// WebSocket transport for devices that can only reach the server through a
|
|
// reverse proxy (nginx) on a shared port such as 443.
|
|
//
|
|
// The app protocol is unchanged: the WS layer is a transparent byte-stream
|
|
// tunnel. A device connects to the proxy with a normal WebSocket upgrade,
|
|
// nginx terminates TLS and proxies the (now plaintext) upgrade to the device
|
|
// port, and this file completes the handshake and carries the existing parser
|
|
// traffic inside binary frames.
|
|
//
|
|
// Upgrade request detection happens after the TLS sniff in handleConnection,
|
|
// so a proxied wss connection (nginx terminates TLS) and a direct wss
|
|
// connection (YAMA_TLS_CERT set, client speaks wss end to end) both work.
|
|
|
|
const (
|
|
wsMagicGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
|
|
|
|
wsOpContinuation = 0x0
|
|
wsOpText = 0x1
|
|
wsOpBinary = 0x2
|
|
wsOpClose = 0x8
|
|
wsOpPing = 0x9
|
|
wsOpPong = 0xA
|
|
|
|
// wsMaxHandshakeBytes caps the upgrade request head we are willing to
|
|
// buffer before rejecting the peer.
|
|
wsMaxHandshakeBytes = 8 * 1024
|
|
|
|
// wsHandshakeTimeout bounds reading the request head once "GET " was seen.
|
|
wsHandshakeTimeout = 10 * time.Second
|
|
)
|
|
|
|
var (
|
|
errWSProtocol = errors.New("websocket: protocol error")
|
|
errWSHandshake = errors.New("websocket: bad handshake")
|
|
|
|
// wsUpgradePrefix identifies an HTTP request whose first token is GET. The
|
|
// app protocol never starts a connection with these bytes (its flags are
|
|
// "Shine"/"HELL"/"POST" variants), so this is unambiguous.
|
|
wsUpgradePrefix = []byte("GET ")
|
|
)
|
|
|
|
// wrapWebSocket sniffs the already TLS-sniffed connection for an HTTP
|
|
// WebSocket upgrade. When the peer is not upgrading, conn is returned
|
|
// unchanged. When the upgrade completes, the returned net.Conn frames all
|
|
// reads/writes, and realIP carries the client address reported by a trusted
|
|
// reverse proxy (empty otherwise). A failed upgrade returns (nil, "").
|
|
func (s *Server) wrapWebSocket(conn net.Conn, peer string) (net.Conn, string) {
|
|
br := bufio.NewReader(conn)
|
|
// Bound the sniff even when the server has no ReadTimeout: a peer that opens a
|
|
// socket and never sends the prefix would otherwise hold the goroutine (and its
|
|
// per-IP slot) for as long as it likes. When ReadTimeout is configured and
|
|
// shorter, keep the tighter bound.
|
|
sniffTimeout := wsHandshakeTimeout
|
|
if s.config.ReadTimeout > 0 && s.config.ReadTimeout < sniffTimeout {
|
|
sniffTimeout = s.config.ReadTimeout
|
|
}
|
|
_ = conn.SetReadDeadline(time.Now().Add(sniffTimeout))
|
|
hdr, err := br.Peek(len(wsUpgradePrefix))
|
|
_ = conn.SetReadDeadline(time.Time{})
|
|
if err != nil {
|
|
// Peer never sent enough to identify itself; let the normal read loop
|
|
// surface EOF/timeout exactly as it would without the sniff.
|
|
return &bufferedConn{Conn: conn, r: br}, ""
|
|
}
|
|
if !bytes.Equal(hdr, wsUpgradePrefix) {
|
|
return &bufferedConn{Conn: conn, r: br}, ""
|
|
}
|
|
|
|
req, err := readWSRequest(br, conn)
|
|
if err != nil {
|
|
s.logger.Debug("wrapWebSocket: reject %s: %v", conn.RemoteAddr(), err)
|
|
writeWSReject(conn, err)
|
|
_ = conn.Close()
|
|
return nil, ""
|
|
}
|
|
|
|
if s.config.WSPath != "" && req.path != s.config.WSPath {
|
|
s.logger.Debug("wrapWebSocket: reject %s: path %q != %q", conn.RemoteAddr(), req.path, s.config.WSPath)
|
|
writeWSReject(conn, errWSProtocol)
|
|
_ = conn.Close()
|
|
return nil, ""
|
|
}
|
|
|
|
accept := wsAcceptKey(req.key)
|
|
_ = conn.SetWriteDeadline(time.Now().Add(wsHandshakeTimeout))
|
|
resp := "HTTP/1.1 101 Switching Protocols\r\n" +
|
|
"Upgrade: websocket\r\n" +
|
|
"Connection: Upgrade\r\n" +
|
|
"Sec-WebSocket-Accept: " + accept + "\r\n\r\n"
|
|
_, werr := io.WriteString(conn, resp)
|
|
_ = conn.SetWriteDeadline(time.Time{})
|
|
if werr != nil {
|
|
_ = conn.Close()
|
|
return nil, ""
|
|
}
|
|
|
|
realIP := ""
|
|
if s.trustedProxy(peer) {
|
|
realIP = reqForwardedIP(req.headers)
|
|
}
|
|
if realIP != "" {
|
|
s.logger.Info("WebSocket device from %s (forwarded by %s)", realIP, peer)
|
|
} else {
|
|
s.logger.Info("WebSocket device from %s", peer)
|
|
}
|
|
return newWSConn(conn, br), realIP
|
|
}
|
|
|
|
// wsRequest is the parsed subset of an upgrade request the server needs.
|
|
type wsRequest struct {
|
|
path string
|
|
key string
|
|
headers map[string]string // lower-cased names; repeated headers joined by ", "
|
|
}
|
|
|
|
// readWSRequest reads and validates the HTTP upgrade head. The buffered reader
|
|
// is left positioned at the first byte after the head so any frames the peer
|
|
// pipelined behind the request are preserved.
|
|
func readWSRequest(br *bufio.Reader, conn net.Conn) (*wsRequest, error) {
|
|
_ = conn.SetReadDeadline(time.Now().Add(wsHandshakeTimeout))
|
|
defer func() { _ = conn.SetReadDeadline(time.Time{}) }()
|
|
|
|
total := 0
|
|
line, err := readWSLine(br, &total)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
parts := strings.Fields(line)
|
|
if len(parts) != 3 || parts[0] != "GET" || !strings.HasPrefix(parts[2], "HTTP/1.") {
|
|
return nil, errWSHandshake
|
|
}
|
|
req := &wsRequest{headers: make(map[string]string)}
|
|
req.path = parts[1]
|
|
if i := strings.IndexByte(req.path, '?'); i >= 0 {
|
|
req.path = req.path[:i]
|
|
}
|
|
|
|
for {
|
|
line, err = readWSLine(br, &total)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if line == "" {
|
|
break
|
|
}
|
|
name, value, ok := strings.Cut(line, ":")
|
|
if !ok {
|
|
return nil, errWSHandshake
|
|
}
|
|
name = strings.ToLower(strings.TrimSpace(name))
|
|
value = strings.TrimSpace(value)
|
|
if prev, dup := req.headers[name]; dup {
|
|
req.headers[name] = prev + ", " + value
|
|
} else {
|
|
req.headers[name] = value
|
|
}
|
|
}
|
|
|
|
if !headerHasToken(req.headers["upgrade"], "websocket") ||
|
|
!headerHasToken(req.headers["connection"], "upgrade") {
|
|
return nil, errWSHandshake
|
|
}
|
|
req.key = req.headers["sec-websocket-key"]
|
|
if req.key == "" {
|
|
return nil, errWSHandshake
|
|
}
|
|
return req, nil
|
|
}
|
|
|
|
// readWSLine reads one CRLF-terminated line, rejecting the peer as soon as the
|
|
// accumulated head exceeds wsMaxHandshakeBytes. The input is peer-controlled:
|
|
// a plain ReadString would buffer a newline-free line without limit.
|
|
func readWSLine(br *bufio.Reader, total *int) (string, error) {
|
|
var line []byte
|
|
for {
|
|
chunk, err := br.ReadSlice('\n')
|
|
line = append(line, chunk...)
|
|
*total += len(chunk)
|
|
if *total > wsMaxHandshakeBytes {
|
|
return "", errWSProtocol
|
|
}
|
|
if err == bufio.ErrBufferFull {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimRight(string(line), "\r\n"), nil
|
|
}
|
|
}
|
|
|
|
// headerHasToken reports whether a comma-separated header value contains token
|
|
// (case-insensitive), as required for Connection: keep-alive, Upgrade.
|
|
func headerHasToken(value, token string) bool {
|
|
for _, part := range strings.Split(value, ",") {
|
|
if strings.EqualFold(strings.TrimSpace(part), token) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// reqForwardedIP extracts the device's real address from a proxy's forwarded
|
|
// headers. The LAST X-Forwarded-For entry wins, not the first: nginx's
|
|
// $proxy_add_x_forwarded_for appends the address it accepted the connection
|
|
// from, so whatever a client prepends is ignored — the same rule web/ws.go's
|
|
// clientIP applies. Trusting the first entry would let a device rotate its key
|
|
// through the per-IP budget, the login throttle and the ConnAuth peer binding.
|
|
// X-Real-IP is only a fallback for proxies configured to set that and not XFF.
|
|
// Only IP literals are accepted, so a header cannot inject an arbitrary peer key.
|
|
func reqForwardedIP(headers map[string]string) string {
|
|
if xff := headers["x-forwarded-for"]; xff != "" {
|
|
last := strings.TrimSpace(xff[strings.LastIndexByte(xff, ',')+1:])
|
|
if ip := headerIP(last); ip != "" {
|
|
return ip
|
|
}
|
|
}
|
|
return headerIP(headers["x-real-ip"])
|
|
}
|
|
|
|
func headerIP(s string) string {
|
|
s = strings.TrimSpace(s)
|
|
if i := strings.LastIndexByte(s, ':'); i > 0 && strings.Count(s, ":") == 1 {
|
|
// strip an IPv4:port pair, but leave bare IPv6 literals alone
|
|
if net.ParseIP(s[:i]) != nil {
|
|
s = s[:i]
|
|
}
|
|
}
|
|
if ip := net.ParseIP(s); ip != nil {
|
|
return ip.String()
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// wsAcceptKey computes Sec-WebSocket-Accept: base64(SHA1(key + magic GUID)).
|
|
func wsAcceptKey(key string) string {
|
|
h := sha1.New()
|
|
io.WriteString(h, key)
|
|
io.WriteString(h, wsMagicGUID)
|
|
return base64.StdEncoding.EncodeToString(h.Sum(nil))
|
|
}
|
|
|
|
func writeWSReject(conn net.Conn, reason error) {
|
|
status := "400 Bad Request"
|
|
body := "bad request"
|
|
if errors.Is(reason, errWSProtocol) {
|
|
body = "protocol error"
|
|
}
|
|
_ = conn.SetWriteDeadline(time.Now().Add(wsHandshakeTimeout))
|
|
fmt.Fprintf(conn, "HTTP/1.1 %s\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s",
|
|
status, len(body), body)
|
|
}
|
|
|
|
// trustedProxy reports whether peer is allowed to set the device's real IP via
|
|
// forwarded headers. Without this check any direct client could spoof its
|
|
// address and, with it, the per-IP limits and the ConnAuth peer binding.
|
|
func (s *Server) trustedProxy(peer string) bool {
|
|
ip := net.ParseIP(peer)
|
|
if ip == nil {
|
|
return false
|
|
}
|
|
for _, n := range s.trustedProxies {
|
|
if n.Contains(ip) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// parseTrustedProxies converts configured IPs/CIDRs into networks. An empty or
|
|
// unparsable list falls back to loopback, the only sensible default for a
|
|
// proxy running on the same host as the server.
|
|
func parseTrustedProxies(list []string) []*net.IPNet {
|
|
var nets []*net.IPNet
|
|
for _, item := range list {
|
|
item = strings.TrimSpace(item)
|
|
if item == "" {
|
|
continue
|
|
}
|
|
if !strings.Contains(item, "/") {
|
|
if ip := net.ParseIP(item); ip != nil {
|
|
bits := 32
|
|
if ip.To4() == nil {
|
|
bits = 128
|
|
}
|
|
item = fmt.Sprintf("%s/%d", ip.String(), bits)
|
|
}
|
|
}
|
|
if _, n, err := net.ParseCIDR(item); err == nil {
|
|
nets = append(nets, n)
|
|
}
|
|
}
|
|
if len(nets) == 0 {
|
|
_, v4, _ := net.ParseCIDR("127.0.0.0/8")
|
|
_, v6, _ := net.ParseCIDR("::1/128")
|
|
nets = append(nets, v4, v6)
|
|
}
|
|
return nets
|
|
}
|
|
|
|
// wsConn presents a WebSocket connection as a net.Conn byte stream: each Write
|
|
// becomes one binary frame and Read returns the concatenated payload of data
|
|
// frames, transparently answering pings and closes. Frame payloads are streamed
|
|
// rather than buffered whole, so a multi-megabyte upload allocates nothing here.
|
|
type wsConn struct {
|
|
conn net.Conn
|
|
r *bufio.Reader
|
|
|
|
writeMu sync.Mutex
|
|
closeMu sync.Mutex
|
|
closed bool
|
|
|
|
// state of the data frame currently being read
|
|
remaining int64
|
|
masked bool
|
|
mask [4]byte
|
|
maskPos byte
|
|
}
|
|
|
|
func newWSConn(conn net.Conn, r *bufio.Reader) *wsConn {
|
|
return &wsConn{conn: conn, r: r}
|
|
}
|
|
|
|
// Read implements net.Conn.
|
|
func (c *wsConn) Read(p []byte) (int, error) {
|
|
if len(p) == 0 {
|
|
return 0, nil
|
|
}
|
|
for {
|
|
if c.remaining > 0 {
|
|
n := len(p)
|
|
if int64(n) > c.remaining {
|
|
n = int(c.remaining)
|
|
}
|
|
nr, err := io.ReadFull(c.r, p[:n])
|
|
if nr > 0 {
|
|
if c.masked {
|
|
for i := 0; i < nr; i++ {
|
|
p[i] ^= c.mask[c.maskPos]
|
|
c.maskPos = (c.maskPos + 1) & 3
|
|
}
|
|
}
|
|
c.remaining -= int64(nr)
|
|
}
|
|
if err != nil {
|
|
return nr, err
|
|
}
|
|
return nr, nil
|
|
}
|
|
|
|
opcode, fin, length, masked, err := c.readFrameHeader()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if opcode >= 0x8 {
|
|
if !fin || length > 125 {
|
|
return 0, errWSProtocol
|
|
}
|
|
payload, err := c.readControlPayload(int(length), masked)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
switch opcode {
|
|
case wsOpClose:
|
|
_ = c.writeFrame(wsOpClose, payload[:min(len(payload), 125)])
|
|
return 0, io.EOF
|
|
case wsOpPing:
|
|
if err := c.writeFrame(wsOpPong, payload); err != nil {
|
|
return 0, err
|
|
}
|
|
case wsOpPong:
|
|
// ignore
|
|
default:
|
|
return 0, errWSProtocol
|
|
}
|
|
continue
|
|
}
|
|
|
|
switch opcode {
|
|
case wsOpBinary, wsOpText, wsOpContinuation:
|
|
default:
|
|
return 0, errWSProtocol
|
|
}
|
|
if length == 0 {
|
|
continue
|
|
}
|
|
c.remaining = length
|
|
c.masked = masked
|
|
c.maskPos = 0
|
|
}
|
|
}
|
|
|
|
// readFrameHeader reads and validates a frame header. Frames from the client
|
|
// are normally masked; unmasked data frames are tolerated because the only
|
|
// speakers on this port are YAMA's own clients.
|
|
func (c *wsConn) readFrameHeader() (opcode byte, fin bool, length int64, masked bool, err error) {
|
|
var hdr [2]byte
|
|
if _, err = io.ReadFull(c.r, hdr[:]); err != nil {
|
|
return
|
|
}
|
|
if hdr[0]&0x70 != 0 {
|
|
return 0, false, 0, false, errWSProtocol // RSV bits set without a negotiated extension
|
|
}
|
|
fin = hdr[0]&0x80 != 0
|
|
opcode = hdr[0] & 0x0f
|
|
masked = hdr[1]&0x80 != 0
|
|
length = int64(hdr[1] & 0x7f)
|
|
|
|
switch length {
|
|
case 126:
|
|
var ext [2]byte
|
|
if _, err = io.ReadFull(c.r, ext[:]); err != nil {
|
|
return
|
|
}
|
|
length = int64(binary.BigEndian.Uint16(ext[:]))
|
|
case 127:
|
|
var ext [8]byte
|
|
if _, err = io.ReadFull(c.r, ext[:]); err != nil {
|
|
return
|
|
}
|
|
u := binary.BigEndian.Uint64(ext[:])
|
|
if u > 1<<62 {
|
|
return 0, false, 0, false, errWSProtocol
|
|
}
|
|
length = int64(u)
|
|
}
|
|
|
|
if masked {
|
|
if _, err = io.ReadFull(c.r, c.mask[:]); err != nil {
|
|
return
|
|
}
|
|
}
|
|
return opcode, fin, length, masked, nil
|
|
}
|
|
|
|
func (c *wsConn) readControlPayload(n int, masked bool) ([]byte, error) {
|
|
payload := make([]byte, n)
|
|
if _, err := io.ReadFull(c.r, payload); err != nil {
|
|
return nil, err
|
|
}
|
|
if masked {
|
|
for i := range payload {
|
|
payload[i] ^= c.mask[i&3]
|
|
}
|
|
}
|
|
return payload, nil
|
|
}
|
|
|
|
// Write implements net.Conn: one binary frame per call. The app protocol's
|
|
// sends are already whole packets (Context.Send), so this keeps writes atomic
|
|
// and lets the peer deliver each packet as it was produced.
|
|
func (c *wsConn) Write(p []byte) (int, error) {
|
|
if err := c.writeFrame(wsOpBinary, p); err != nil {
|
|
return 0, err
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
// writeFrame emits one FIN frame with the given opcode.
|
|
func (c *wsConn) writeFrame(opcode byte, payload []byte) error {
|
|
var hdr [10]byte
|
|
hdr[0] = 0x80 | opcode
|
|
n := 2
|
|
switch {
|
|
case len(payload) < 126:
|
|
hdr[1] = byte(len(payload))
|
|
case len(payload) <= 0xffff:
|
|
hdr[1] = 126
|
|
binary.BigEndian.PutUint16(hdr[2:], uint16(len(payload)))
|
|
n = 4
|
|
default:
|
|
hdr[1] = 127
|
|
binary.BigEndian.PutUint64(hdr[2:], uint64(len(payload)))
|
|
n = 10
|
|
}
|
|
|
|
c.writeMu.Lock()
|
|
defer c.writeMu.Unlock()
|
|
bufs := net.Buffers{hdr[:n], payload}
|
|
if _, err := bufs.WriteTo(c.conn); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Close sends a best-effort close frame and closes the transport.
|
|
func (c *wsConn) Close() error {
|
|
c.closeMu.Lock()
|
|
if c.closed {
|
|
c.closeMu.Unlock()
|
|
return nil
|
|
}
|
|
c.closed = true
|
|
c.closeMu.Unlock()
|
|
|
|
_ = c.conn.SetWriteDeadline(time.Now().Add(time.Second))
|
|
_ = c.writeFrame(wsOpClose, nil)
|
|
return c.conn.Close()
|
|
}
|
|
|
|
func (c *wsConn) LocalAddr() net.Addr { return c.conn.LocalAddr() }
|
|
func (c *wsConn) RemoteAddr() net.Addr { return c.conn.RemoteAddr() }
|
|
|
|
func (c *wsConn) SetDeadline(t time.Time) error { return c.conn.SetDeadline(t) }
|
|
func (c *wsConn) SetReadDeadline(t time.Time) error { return c.conn.SetReadDeadline(t) }
|
|
func (c *wsConn) SetWriteDeadline(t time.Time) error { return c.conn.SetWriteDeadline(t) }
|