From 5227d4d78ec57b554c22979d74e4139657123547 Mon Sep 17 00:00:00 2001 From: yuanyuanxiang <962914132@qq.com> Date: Thu, 12 Nov 2026 22:07:07 +0100 Subject: [PATCH] Feature(Go): get_screenshot over the main-connection preview RPC COMMAND_SCREEN_PREVIEW_REQ (247) / TOKEN_SCREEN_PREVIEW_RSP (248) is a request/reply pair on the device's main connection: the client captures, scales and encodes the frame itself, so no sub-connection is opened and no screen-session slot is taken. get_screenshot can therefore run while a human is watching the same machine. The server keeps one pending request per device in a small table, matches replies by reqId and drops foreign or late ones, and bounds a round trip with an 8s timer. A missing CLIENT_CAP_SCREEN_PREVIEW bit is refused with -32005 before anything is sent; region/window/monitor parameters are refused with the same code because the v2 CLIENT_CAP_SCREEN_REGION extension is not implemented in this build. Status 4 maps to -32602 because the caller can fix the target, while capture/encode failures stay -32000. Windows clients return JPEG directly. Linux clients have no JPEG encoder and return top-down BGRX, which the server converts (R/B swap, opaque alpha) before wrapping the frame as an MCP image item. The image item is new to the content envelope; text tools keep text-only items. Verified live against x64\Debug\ghost.exe (JPEG FF D8 FF, decoded size matches metadata, max_width clamped to 64) and against linux/ghost under WSL (BGRX path, valid JPEG, black frame from the known Xwayland capture limitation). gofmt/vet clean, unit and -race tests green. Co-Authored-By: deepseek-flash --- server/go/cmd/main.go | 80 +++--- server/go/mcp/screen.go | 424 ++++++++++++++++++++++++++++++ server/go/mcp/screen_test.go | 458 +++++++++++++++++++++++++++++++++ server/go/mcp/server.go | 3 + server/go/mcp/tools_local.go | 9 +- server/go/protocol/commands.go | 10 +- 6 files changed, 947 insertions(+), 37 deletions(-) create mode 100644 server/go/mcp/screen.go create mode 100644 server/go/mcp/screen_test.go diff --git a/server/go/cmd/main.go b/server/go/cmd/main.go index 32ad615..da15598 100644 --- a/server/go/cmd/main.go +++ b/server/go/cmd/main.go @@ -39,6 +39,7 @@ type MyHandler struct { signer licensing.Signer // CMD_MASTERSETTING signer (local / remote / noop) loginLimit *wsauth.RateLimiter // per-IP TOKEN_AUTH throttle connAuthKey []byte // nil = ConnAuth HMAC verification disabled (legacy always-OK) + mcp *mcp.Server // routes main-connection RPC replies (screen preview, file lists) } // OnConnect is called when a client connects @@ -165,6 +166,14 @@ func (h *MyHandler) OnReceive(ctx *connection.Context, data []byte) { // overlay learns to render arbitrary BGRA images. Drop silently for // now; the standard IDC_* index (data[10] of every frame header) is // what we actually use right now. + case protocol.TokenScreenPreviewRsp: + // get_screenshot is a main-connection RPC: the reply carries only a + // reqId, so MCP matches it against the pending request for this device + // and drops a late or foreign one. No other component reads this token. + if h.mcp != nil && h.mcp.OnPacket(ctx, data) { + return + } + h.log.Info("Unhandled command %d from client %d", cmd, ctx.ID) case protocol.CmdSockDllLoader: // SOCKET_DLLLOADER (210): the shellcode / memory-DLL loader asks for the // full DLL payload before it can load the client. Serving it is the MFC @@ -1204,6 +1213,43 @@ func main() { log.Info("Trusting forwarded client IPs from: %s", strings.Join(trustedProxies, ", ")) } + // MCP endpoint: a static bearer token independent of the web login. An + // empty YAMA_MCP_TOKEN leaves /mcp unregistered (404) — there is no + // auto-generated token, because a secret that only appears in a log is not + // a secret. Version/commit come from the link-time buildinfo injection. + // + // Built before the TCP servers so each handler can hand main-connection + // RPC replies (screen preview, file lists) to it. Starting its reaper a + // few statements early is harmless; the HTTP route is mounted later. + mcpReadonly := envBool("YAMA_MCP_READONLY") + var mcpUnsubscribe func() + bindAddr := "" + if *httpPort != 0 { + bindAddr = "0.0.0.0" // web server listens on :port (all interfaces) + } + mcpSrv := mcp.New(mcp.Config{ + Token: strings.TrimSpace(os.Getenv("YAMA_MCP_TOKEN")), + Readonly: mcpReadonly, + TermIdle: time.Duration(envInt("YAMA_MCP_TERM_IDLE", 180)) * time.Second, + MaxConcurrency: envInt("YAMA_MCP_MAX_CONCURRENCY", 16), + ReadMax: int64(envInt("YAMA_MCP_READ_MAX", 2<<20)), + Version: buildinfo.Version, + StartedAt: time.Now(), + Log: log.WithPrefix("MCP"), + Hub: deviceHub, + History: deviceHistory, + AuditPath: auditFile, + Port: *httpPort, + Bind: bindAddr, + }) + if mcpSrv.Enabled() { + // Terminal tools need the hub's shell events (ready/data/closed). + // Unsubscribed again at shutdown, before the TCP servers stop. + mcpUnsubscribe = deviceHub.Subscribe(mcpSrv) + log.Info("MCP endpoint enabled on /mcp (readonly=%v, commit=%s)", + mcpReadonly, buildinfo.Commit) + } + // Create servers for each port var servers []*server.Server for _, port := range ports { @@ -1229,6 +1275,7 @@ func main() { signer: signer, loginLimit: authLoginLimit, connAuthKey: connAuthKey, + mcp: mcpSrv, } srv.SetHandler(handler) @@ -1303,39 +1350,6 @@ func main() { SSO: ssoDetector, } - // MCP endpoint: a static bearer token independent of the web login. An - // empty YAMA_MCP_TOKEN leaves /mcp unregistered (404) — there is no - // auto-generated token, because a secret that only appears in a log is not - // a secret. Version/commit come from the link-time buildinfo injection. - mcpReadonly := envBool("YAMA_MCP_READONLY") - var mcpUnsubscribe func() - bindAddr := "" - if *httpPort != 0 { - bindAddr = "0.0.0.0" // web server listens on :port (all interfaces) - } - mcpSrv := mcp.New(mcp.Config{ - Token: strings.TrimSpace(os.Getenv("YAMA_MCP_TOKEN")), - Readonly: mcpReadonly, - TermIdle: time.Duration(envInt("YAMA_MCP_TERM_IDLE", 180)) * time.Second, - MaxConcurrency: envInt("YAMA_MCP_MAX_CONCURRENCY", 16), - ReadMax: int64(envInt("YAMA_MCP_READ_MAX", 2<<20)), - Version: buildinfo.Version, - StartedAt: time.Now(), - Log: log.WithPrefix("MCP"), - Hub: deviceHub, - History: deviceHistory, - AuditPath: auditFile, - Port: *httpPort, - Bind: bindAddr, - }) - if mcpSrv.Enabled() { - // Terminal tools need the hub's shell events (ready/data/closed). - // Unsubscribed again at shutdown, before the TCP servers stop. - mcpUnsubscribe = deviceHub.Subscribe(mcpSrv) - log.Info("MCP endpoint enabled on /mcp (readonly=%v, commit=%s)", - mcpReadonly, buildinfo.Commit) - } - // Start HTTP server for web UI. Hub gives it read-only access to the // device registry; the authenticator owns user accounts and session tokens. httpSrv := web.New(*httpPort, log.WithPrefix("Web"), deviceHub, webAuth). diff --git a/server/go/mcp/screen.go b/server/go/mcp/screen.go new file mode 100644 index 0000000..d37f896 --- /dev/null +++ b/server/go/mcp/screen.go @@ -0,0 +1,424 @@ +package mcp + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "image" + "image/jpeg" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/yuanyuanxiang/SimpleRemoter/server/go/connection" + "github.com/yuanyuanxiang/SimpleRemoter/server/go/hub" + "github.com/yuanyuanxiang/SimpleRemoter/server/go/protocol" +) + +// get_screenshot uses the main-connection preview RPC +// (COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP): the client captures, +// scales and encodes the frame itself, so the request opens no sub-connection +// and competes with nobody for the per-device screen slot (design §7.1). +const ( + screenPreviewDefaultWidth = 1024 + screenPreviewMinWidth = 64 + screenPreviewMaxWidth = 1920 + screenPreviewJPEGQuality = 80 + + // screenPreviewRspHeaderLen is sizeof(ScreenPreviewRspHeader) with + // #pragma pack(1) (common/commands.h); the image bytes follow directly. + screenPreviewRspHeaderLen = 16 + screenPreviewOffReqID = 1 + screenPreviewOffStatus = 3 + screenPreviewOffFormat = 4 + screenPreviewOffWidth = 5 + screenPreviewOffHeight = 7 + screenPreviewOffBytes = 9 +) + +// ScreenPreviewStatus / ScreenPreviewFormat values (common/commands.h). Only +// two formats exist on the wire: Windows clients send JPEG, Linux clients send +// BGRX because they carry no JPEG encoder — the server encodes it here. +const ( + screenPreviewStatusOK byte = 0 + screenPreviewStatusCaptureFailed byte = 1 + screenPreviewStatusEncodeFailed byte = 2 + screenPreviewStatusNotSupported byte = 3 + screenPreviewStatusBadTarget byte = 4 + + screenPreviewFmtJPEG byte = 0 + screenPreviewFmtBGRX byte = 3 +) + +// screenPreviewTimeout bounds one round trip. It is a variable rather than a +// constant only so tests can shorten it; production code never changes it. +// There is deliberately no cancellation on timeout: the request rides the +// device's main connection, where MFC likewise refuses CancelIO (§7.4) — the +// pending entry is dropped and a late reply fails the reqId match. +var screenPreviewTimeout = 8 * time.Second + +// screenshotArgs is the published inputSchema as Go. Pointer fields separate +// "absent" from an explicit value: mode=screen must still reject the +// region/window/monitor parameters instead of silently ignoring them. +type screenshotArgs struct { + ID string `json:"id"` + MaxWidth *int `json:"max_width"` + Mode string `json:"mode"` + HWND *json.RawMessage `json:"hwnd"` + X *int `json:"x"` + Y *int `json:"y"` + Width *int `json:"width"` + Height *int `json:"height"` + Monitor *int `json:"monitor"` +} + +// screenshotImage / screenshotCapture are the structuredContent fields the +// shared outputSchema defines. Fields the v1 RPC cannot report are omitted +// rather than faked (design §3.2). +type screenshotImage struct { + MimeType string `json:"mimeType"` + Width int `json:"width"` + Height int `json:"height"` + Bytes int `json:"bytes"` +} + +type screenshotCapture struct { + Mode string `json:"mode"` +} + +type screenshotOutput struct { + Image screenshotImage `json:"image"` + Capture screenshotCapture `json:"capture"` +} + +// previewReply is one decoded TOKEN_SCREEN_PREVIEW_RSP. +type previewReply struct { + status byte + format byte + width uint16 + height uint16 + payload []byte +} + +// previewRequest is one in-flight screenshot. ch is buffered so delivery never +// blocks the TCP read loop. +type previewRequest struct { + reqID uint16 + ch chan previewReply +} + +// previewTable matches a client reply to the call waiting for it. The +// per-device call lock already serialises MCP screenshots, so at most one +// entry exists per device; responses are still keyed by device because the +// reply itself carries no device id — only the connection does. +type previewTable struct { + mu sync.Mutex + byDevice map[string]*previewRequest + nextID atomic.Uint32 +} + +func newPreviewTable() *previewTable { + return &previewTable{byDevice: make(map[string]*previewRequest)} +} + +// nextReqID mirrors MFC's NextPreviewReqId: increment and skip 0, which would +// otherwise match an "unset" reqId on the client side. +func (t *previewTable) nextReqID() uint16 { + for { + if v := uint16(t.nextID.Add(1)); v != 0 { + return v + } + } +} + +// begin reserves the device's single preview slot. nil means another request +// is already pending (only possible if the call lock was bypassed). +func (t *previewTable) begin(deviceID string) *previewRequest { + t.mu.Lock() + defer t.mu.Unlock() + if _, ok := t.byDevice[deviceID]; ok { + return nil + } + r := &previewRequest{reqID: t.nextReqID(), ch: make(chan previewReply, 1)} + t.byDevice[deviceID] = r + return r +} + +// cancel drops a pending slot unless it was already delivered. The pointer +// check keeps a slow caller from deleting a newer request for the same device. +func (t *previewTable) cancel(deviceID string, r *previewRequest) { + t.mu.Lock() + defer t.mu.Unlock() + if t.byDevice[deviceID] == r { + delete(t.byDevice, deviceID) + } +} + +// deliver hands a matching reply to its waiter. A reply whose reqId does not +// match — an expired request or one from another caller — is dropped, exactly +// like MFC's TakePreviewResponse. +func (t *previewTable) deliver(deviceID string, reqID uint16, reply previewReply) bool { + t.mu.Lock() + r, ok := t.byDevice[deviceID] + if !ok || r.reqID != reqID { + t.mu.Unlock() + return false + } + delete(t.byDevice, deviceID) + t.mu.Unlock() + r.ch <- reply // buffered, and the entry is gone: nobody else can send + return true +} + +func (s *Server) registerScreenTools() { + s.handlers["get_screenshot"] = s.callGetScreenshot +} + +// OnPacket feeds one packet from a device connection to the MCP layer. It +// reports whether MCP consumed the packet; anything MCP does not own returns +// false and falls through to cmd's normal command dispatch. +func (s *Server) OnPacket(ctx *connection.Context, data []byte) bool { + if s == nil || len(data) == 0 || data[0] != protocol.TokenScreenPreviewRsp { + return false + } + deviceID := ctx.GetInfo().ClientID + rsp, err := parseScreenPreviewRsp(data) + if err != nil { + s.log.Warn("malformed screen preview response from conn %d (device=%q): %v", + ctx.ID, deviceID, err) + return true + } + if deviceID != "" { + s.previews.deliver(deviceID, rsp.reqID, rsp.reply) + } + return true +} + +// parseScreenPreviewRsp validates the 16-byte header and copies the image +// bytes out of the TCP read buffer. A declared length that overruns the packet +// marks the whole response malformed: the client would otherwise hand us an +// under-filled image. +func parseScreenPreviewRsp(data []byte) (struct { + reqID uint16 + reply previewReply +}, error) { + var rsp struct { + reqID uint16 + reply previewReply + } + if len(data) < screenPreviewRspHeaderLen { + return rsp, fmt.Errorf("packet is %d bytes, want at least %d", + len(data), screenPreviewRspHeaderLen) + } + if data[0] != protocol.TokenScreenPreviewRsp { + return rsp, fmt.Errorf("token %d, want %d", data[0], protocol.TokenScreenPreviewRsp) + } + rsp.reqID = binary.LittleEndian.Uint16(data[screenPreviewOffReqID:]) + rsp.reply.status = data[screenPreviewOffStatus] + rsp.reply.format = data[screenPreviewOffFormat] + rsp.reply.width = binary.LittleEndian.Uint16(data[screenPreviewOffWidth:]) + rsp.reply.height = binary.LittleEndian.Uint16(data[screenPreviewOffHeight:]) + n := binary.LittleEndian.Uint32(data[screenPreviewOffBytes:]) + if avail := len(data) - screenPreviewRspHeaderLen; n > uint32(avail) { + return rsp, fmt.Errorf("declares %d image bytes but only %d follow", n, avail) + } + rsp.reply.payload = append([]byte(nil), data[screenPreviewRspHeaderLen:screenPreviewRspHeaderLen+int(n)]...) + return rsp, nil +} + +// buildScreenPreviewReq renders the 8-byte v1 request (whole virtual desktop). +// v2 (region/window/monitor) is gated behind CLIENT_CAP_SCREEN_REGION and is +// not implemented by this server build. +func buildScreenPreviewReq(reqID, maxWidth uint16, quality byte) []byte { + buf := make([]byte, 8) + buf[0] = protocol.CommandScreenPreviewReq + binary.LittleEndian.PutUint16(buf[1:3], reqID) + binary.LittleEndian.PutUint16(buf[3:5], maxWidth) + buf[5] = quality + return buf +} + +// callGetScreenshot captures one frame through the main-connection RPC. +func (s *Server) callGetScreenshot(ctx context.Context, raw json.RawMessage) (any, *rpcError) { + if s.cfg.Hub == nil { + return nil, errf(CodeInternal, "server device registry is not configured") + } + var args screenshotArgs + if rerr := decodeArgs(raw, &args); rerr != nil { + return nil, rerr + } + if args.ID == "" { + return nil, errf(CodeInvalidParams, "Missing required parameter: id") + } + if !isDecimalID(args.ID) { + return nil, errf(CodeInvalidParams, "Invalid id: expected a decimal host id string") + } + if s.cfg.Hub.MainConn(args.ID) == nil { + return nil, errf(CodeOffline, "Host not found or offline: %s", args.ID) + } + if !protocol.SupportsCap(s.cfg.Hub.Capability(args.ID), protocol.ClientCapScreenPreview) { + return nil, errf(CodeUnsupported, + "Host does not support screen preview: the client does not declare CLIENT_CAP_SCREEN_PREVIEW (upgrade the client)") + } + + mode := strings.TrimSpace(args.Mode) + if mode == "" { + mode = "screen" + } + if mode != "screen" { + switch mode { + case "region", "window", "monitor": + return nil, errf(CodeUnsupported, + "mode=%s needs the CLIENT_CAP_SCREEN_REGION extension, which this server build does not implement; use mode=screen", mode) + default: + return nil, errf(CodeInvalidParams, + "Invalid mode: %s (expect screen|region|window|monitor)", mode) + } + } + if name := firstRegionParam(&args); name != "" { + return nil, errf(CodeUnsupported, + "parameter %q needs the CLIENT_CAP_SCREEN_REGION extension, which this server build does not implement; use mode=screen with only id and max_width", name) + } + + maxWidth := screenPreviewDefaultWidth + if args.MaxWidth != nil && *args.MaxWidth > 0 { + maxWidth = min(max(*args.MaxWidth, screenPreviewMinWidth), screenPreviewMaxWidth) + } + + req := s.previews.begin(args.ID) + if req == nil { + return nil, errf(CodeBusy, + "Device busy: another screenshot request is already pending for host %s", args.ID) + } + defer s.previews.cancel(args.ID, req) + + if err := s.cfg.Hub.SendToDevice(args.ID, + buildScreenPreviewReq(req.reqID, uint16(maxWidth), screenPreviewJPEGQuality)); err != nil { + if errors.Is(err, hub.ErrDeviceOffline) { + return nil, errf(CodeOffline, "Host not found or offline: %s", args.ID) + } + return nil, errf(CodeSendFailed, "Failed to send screenshot request to host: %v", err) + } + + timer := time.NewTimer(screenPreviewTimeout) + defer timer.Stop() + select { + case reply := <-req.ch: + return screenshotResult(reply, mode) + case <-timer.C: + return nil, errf(CodeTimeout, + "Timeout waiting for the screenshot (the host did not answer in %s; a client without screen-preview support cannot be asked)", screenPreviewTimeout) + case <-ctx.Done(): + return nil, errf(CodeTimeout, "Request cancelled while waiting for the screenshot") + } +} + +// firstRegionParam names the first v2 capture parameter present in the call. +// Checked in schema order so the error message is deterministic. +func firstRegionParam(args *screenshotArgs) string { + switch { + case args.HWND != nil: + return "hwnd" + case args.X != nil: + return "x" + case args.Y != nil: + return "y" + case args.Width != nil: + return "width" + case args.Height != nil: + return "height" + case args.Monitor != nil: + return "monitor" + } + return "" +} + +// screenshotResult maps the device status/format onto the MCP result. Status 4 +// is a caller-visible parameter problem (target invalid), so it keeps -32602; +// capture/encode failures are -32000 because retrying the same call cannot fix +// them. +func screenshotResult(reply previewReply, mode string) (any, *rpcError) { + switch reply.status { + case screenPreviewStatusOK: + case screenPreviewStatusCaptureFailed: + return nil, errf(CodeInternal, "Screenshot capture failed on the host (status 1)") + case screenPreviewStatusEncodeFailed: + return nil, errf(CodeInternal, "Screenshot encoding failed on the host (status 2)") + case screenPreviewStatusNotSupported: + return nil, errf(CodeUnsupported, "Host does not support screen capture (status 3)") + case screenPreviewStatusBadTarget: + return nil, errf(CodeInvalidParams, "Screenshot target is invalid (status 4)") + default: + return nil, errf(CodeInternal, "Screenshot failed with unknown status %d", reply.status) + } + + w, h := int(reply.width), int(reply.height) + switch reply.format { + case screenPreviewFmtJPEG: + if len(reply.payload) == 0 { + return nil, errf(CodeInternal, "Screenshot response carries no image data") + } + return imageResult(reply.payload, w, h, mode), nil + case screenPreviewFmtBGRX: + jpg, err := bgrxToJPEG(reply.payload, w, h) + if err != nil { + return nil, errf(CodeInternal, "Malformed BGRX screenshot from host: %v", err) + } + // The reported width/height describe the pixels, which do not change in + // the JPEG encode; bytes describes what is actually returned. + return imageResult(jpg, w, h, mode), nil + default: + return nil, errf(CodeInternal, "Unsupported screenshot format %d", reply.format) + } +} + +// bgrxToJPEG converts a top-down 4-byte-per-pixel B,G,R,X frame (Linux +// clients) into a JPEG. There is no row padding, and X is forced opaque. +func bgrxToJPEG(src []byte, w, h int) ([]byte, error) { + if w <= 0 || h <= 0 { + return nil, fmt.Errorf("invalid dimensions %dx%d", w, h) + } + if len(src) != w*h*4 { + return nil, fmt.Errorf("got %d bytes, want %d for %dx%d BGRX", len(src), w*h*4, w, h) + } + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for i := 0; i < w*h; i++ { + img.Pix[i*4+0] = src[i*4+2] // R ← B + img.Pix[i*4+1] = src[i*4+1] // G + img.Pix[i*4+2] = src[i*4+0] // B ← R + img.Pix[i*4+3] = 0xFF + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: screenPreviewJPEGQuality}); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// imageResult wraps the encoded JPEG in the MCP envelope: one image content +// item plus the metadata the outputSchema declares. +func imageResult(jpg []byte, w, h int, mode string) toolResult { + return toolResult{ + Content: []contentItem{{ + Type: "image", + Data: base64.StdEncoding.EncodeToString(jpg), + MimeType: "image/jpeg", + }}, + StructuredContent: screenshotOutput{ + Image: screenshotImage{ + MimeType: "image/jpeg", + Width: w, + Height: h, + Bytes: len(jpg), + }, + // v1 sends no ScreenCaptureMeta, so the source rect and scale are + // unknown; report the mode only rather than inventing values. + Capture: screenshotCapture{Mode: mode}, + }, + } +} diff --git a/server/go/mcp/screen_test.go b/server/go/mcp/screen_test.go new file mode 100644 index 0000000..ca9e23a --- /dev/null +++ b/server/go/mcp/screen_test.go @@ -0,0 +1,458 @@ +package mcp + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "image" + "image/color" + "image/jpeg" + "testing" + "time" + + "github.com/yuanyuanxiang/SimpleRemoter/server/go/connection" + "github.com/yuanyuanxiang/SimpleRemoter/server/go/hub" + "github.com/yuanyuanxiang/SimpleRemoter/server/go/protocol" +) + +// screenRig is a live MCP server plus a hub whose main connection records the +// outbound screenshot requests. Tests answer them through OnPacket, i.e. +// exactly what cmd/main.go does from the TCP read loop. +type screenRig struct { + t *testing.T + s *Server + h *hub.Hub + main *connection.Context + writes chan []byte +} + +func newScreenRig(t *testing.T, capability string) *screenRig { + t.Helper() + h := hub.New() + main := &connection.Context{RemoteAddr: "10.0.0.1:4000"} + main.Info.ClientID = "1001" + main.Info.ClientType = "EXE" + rig := &screenRig{ + t: t, h: h, main: main, + writes: make(chan []byte, 8), + } + h.SetSender(func(ctx *connection.Context, data []byte) error { + if ctx == main { + select { + case rig.writes <- append([]byte(nil), data...): + default: + } + } + return nil + }) + h.Register(&hub.Device{ID: "1001", PeerIP: "10.0.0.1", Capability: capability}, main) + rig.s = New(Config{Token: testToken, Hub: h}) + t.Cleanup(rig.s.Close) + return rig +} + +type screenCall struct { + res toolResult + err *rpcError +} + +func (r *screenRig) call(args string) <-chan screenCall { + ch := make(chan screenCall, 1) + go func() { + res, rerr := r.s.callGetScreenshot(context.Background(), []byte(args)) + if rerr != nil { + ch <- screenCall{err: rerr} + return + } + tr, ok := res.(toolResult) + if !ok { + ch <- screenCall{err: errf(CodeInternal, "unexpected result type %T", res)} + return + } + ch <- screenCall{res: tr} + }() + return ch +} + +func (r *screenRig) wait(ch <-chan screenCall) screenCall { + r.t.Helper() + select { + case c := <-ch: + return c + case <-time.After(5 * time.Second): + r.t.Fatal("timed out waiting for the tool call") + return screenCall{} + } +} + +// nextRequest returns the reqId and maxWidth of the next outbound request. +func (r *screenRig) nextRequest() (reqID, maxWidth uint16) { + r.t.Helper() + select { + case b := <-r.writes: + if len(b) != 8 || b[0] != protocol.CommandScreenPreviewReq { + r.t.Fatalf("unexpected screenshot request wire %v", b) + } + if b[5] != screenPreviewJPEGQuality || b[6] != 0 || b[7] != 0 { + r.t.Fatalf("request tail = %v, want quality %d and zero reserved", b[5:], screenPreviewJPEGQuality) + } + return binary.LittleEndian.Uint16(b[1:3]), binary.LittleEndian.Uint16(b[3:5]) + case <-time.After(5 * time.Second): + r.t.Fatal("no screenshot request was sent") + return 0, 0 + } +} + +func previewPacket(reqID uint16, status, format byte, w, h uint16, payload []byte) []byte { + buf := make([]byte, screenPreviewRspHeaderLen+len(payload)) + buf[0] = protocol.TokenScreenPreviewRsp + binary.LittleEndian.PutUint16(buf[screenPreviewOffReqID:], reqID) + buf[screenPreviewOffStatus] = status + buf[screenPreviewOffFormat] = format + binary.LittleEndian.PutUint16(buf[screenPreviewOffWidth:], w) + binary.LittleEndian.PutUint16(buf[screenPreviewOffHeight:], h) + binary.LittleEndian.PutUint32(buf[screenPreviewOffBytes:], uint32(len(payload))) + copy(buf[screenPreviewRspHeaderLen:], payload) + return buf +} + +func solidJPEG(t *testing.T, w, h int, c color.RGBA) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for i := 0; i < w*h; i++ { + img.Pix[i*4+0] = c.R + img.Pix[i*4+1] = c.G + img.Pix[i*4+2] = c.B + img.Pix[i*4+3] = 0xFF + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + t.Fatalf("encode fixture: %v", err) + } + return buf.Bytes() +} + +// solidBGRX builds a top-down B,G,R,X frame of one colour. +func solidBGRX(w, h int, b, g, r byte) []byte { + out := make([]byte, w*h*4) + for i := 0; i < w*h; i++ { + out[i*4+0] = b + out[i*4+1] = g + out[i*4+2] = r + out[i*4+3] = 0xFF + } + return out +} + +func TestScreenshotReturnsJPEG(t *testing.T) { + rig := newScreenRig(t, "0004") + payload := solidJPEG(t, 4, 4, color.RGBA{255, 0, 0, 255}) + ch := rig.call(`{"id":"1001"}`) + reqID, maxWidth := rig.nextRequest() + if maxWidth != screenPreviewDefaultWidth { + t.Fatalf("maxWidth = %d, want %d", maxWidth, screenPreviewDefaultWidth) + } + if !rig.s.OnPacket(rig.main, previewPacket(reqID, screenPreviewStatusOK, screenPreviewFmtJPEG, 4, 4, payload)) { + t.Fatal("OnPacket did not consume the preview response") + } + c := rig.wait(ch) + if c.err != nil { + t.Fatalf("call error: %+v", c.err) + } + if len(c.res.Content) != 1 || c.res.Content[0].Type != "image" { + t.Fatalf("content = %+v, want one image item", c.res.Content) + } + if c.res.Content[0].MimeType != "image/jpeg" { + t.Fatalf("mimeType = %q", c.res.Content[0].MimeType) + } + raw, err := base64.StdEncoding.DecodeString(c.res.Content[0].Data) + if err != nil { + t.Fatalf("base64: %v", err) + } + if !bytes.HasPrefix(raw, []byte{0xFF, 0xD8, 0xFF}) { + t.Fatalf("payload does not start with the JPEG magic: %x", raw[:3]) + } + out, ok := c.res.StructuredContent.(screenshotOutput) + if !ok { + t.Fatalf("structured content %T", c.res.StructuredContent) + } + if out.Image.Width != 4 || out.Image.Height != 4 || out.Image.Bytes != len(raw) { + t.Fatalf("image metadata = %+v (raw %d bytes)", out.Image, len(raw)) + } + if out.Image.MimeType != "image/jpeg" || out.Capture.Mode != "screen" { + t.Fatalf("structured content = %+v", out) + } +} + +func TestScreenshotConvertsBGRX(t *testing.T) { + rig := newScreenRig(t, "0004") + // Pure red in B,G,R,X order. + frame := solidBGRX(8, 8, 0, 0, 255) + ch := rig.call(`{"id":"1001"}`) + reqID, _ := rig.nextRequest() + rig.s.OnPacket(rig.main, previewPacket(reqID, screenPreviewStatusOK, screenPreviewFmtBGRX, 8, 8, frame)) + c := rig.wait(ch) + if c.err != nil { + t.Fatalf("call error: %+v", c.err) + } + raw, err := base64.StdEncoding.DecodeString(c.res.Content[0].Data) + if err != nil { + t.Fatalf("base64: %v", err) + } + if !bytes.HasPrefix(raw, []byte{0xFF, 0xD8, 0xFF}) { + t.Fatalf("payload does not start with the JPEG magic: %x", raw[:3]) + } + img, err := jpeg.Decode(bytes.NewReader(raw)) + if err != nil { + t.Fatalf("decode returned JPEG: %v", err) + } + if b := img.Bounds(); b.Dx() != 8 || b.Dy() != 8 { + t.Fatalf("image is %v, want 8x8", b) + } + r16, g16, b16, _ := img.At(4, 4).RGBA() + // JPEG is lossy; a solid frame should still be clearly red. + if r16>>8 < 200 || g16>>8 > 60 || b16>>8 > 60 { + t.Fatalf("pixel = (%d,%d,%d), want red", r16>>8, g16>>8, b16>>8) + } +} + +func TestScreenshotStatusMapping(t *testing.T) { + cases := []struct { + name string + status byte + code int + }{ + {"capture failed", screenPreviewStatusCaptureFailed, CodeInternal}, + {"encode failed", screenPreviewStatusEncodeFailed, CodeInternal}, + {"not supported", screenPreviewStatusNotSupported, CodeUnsupported}, + {"bad target", screenPreviewStatusBadTarget, CodeInvalidParams}, + {"unknown", 9, CodeInternal}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rig := newScreenRig(t, "0004") + ch := rig.call(`{"id":"1001"}`) + reqID, _ := rig.nextRequest() + rig.s.OnPacket(rig.main, previewPacket(reqID, tc.status, screenPreviewFmtJPEG, 0, 0, nil)) + c := rig.wait(ch) + if c.err == nil || c.err.Code != tc.code { + t.Fatalf("error = %+v, want code %d", c.err, tc.code) + } + }) + } +} + +func TestScreenshotDropsMismatchedReqID(t *testing.T) { + rig := newScreenRig(t, "0004") + payload := solidJPEG(t, 2, 2, color.RGBA{0, 255, 0, 255}) + ch := rig.call(`{"id":"1001"}`) + reqID, _ := rig.nextRequest() + + // A response for another request must be dropped, not handed to this one. + rig.s.OnPacket(rig.main, previewPacket(reqID+1, screenPreviewStatusOK, screenPreviewFmtJPEG, 2, 2, payload)) + select { + case c := <-ch: + t.Fatalf("call finished on a foreign reqId: %+v", c) + case <-time.After(100 * time.Millisecond): + } + + rig.s.OnPacket(rig.main, previewPacket(reqID, screenPreviewStatusOK, screenPreviewFmtJPEG, 2, 2, payload)) + if c := rig.wait(ch); c.err != nil { + t.Fatalf("matching response still failed: %+v", c.err) + } +} + +func TestScreenshotTimeoutDropsPending(t *testing.T) { + old := screenPreviewTimeout + screenPreviewTimeout = 50 * time.Millisecond + t.Cleanup(func() { screenPreviewTimeout = old }) + + rig := newScreenRig(t, "0004") + ch := rig.call(`{"id":"1001"}`) + reqID, _ := rig.nextRequest() + c := rig.wait(ch) + if c.err == nil || c.err.Code != CodeTimeout { + t.Fatalf("error = %+v, want -32001", c.err) + } + + rig.s.previews.mu.Lock() + pending := len(rig.s.previews.byDevice) + rig.s.previews.mu.Unlock() + if pending != 0 { + t.Fatalf("%d preview entries left after the timeout", pending) + } + // A late reply is consumed and discarded; it must not resurrect the entry. + rig.s.OnPacket(rig.main, previewPacket(reqID, screenPreviewStatusOK, screenPreviewFmtJPEG, 1, 1, []byte{0xFF, 0xD8, 0xFF, 0xD9})) + rig.s.previews.mu.Lock() + pending = len(rig.s.previews.byDevice) + rig.s.previews.mu.Unlock() + if pending != 0 { + t.Fatalf("%d preview entries after a late reply", pending) + } +} + +func TestScreenshotRequiresCapability(t *testing.T) { + rig := newScreenRig(t, "") + c := rig.wait(rig.call(`{"id":"1001"}`)) + if c.err == nil || c.err.Code != CodeUnsupported { + t.Fatalf("error = %+v, want -32005 for a client without CLIENT_CAP_SCREEN_PREVIEW", c.err) + } + if len(rig.writes) != 0 { + t.Fatal("a request was sent to a host that cannot answer it") + } +} + +func TestScreenshotClampsMaxWidth(t *testing.T) { + cases := []struct { + name string + args string + want uint16 + }{ + {"absent", `{"id":"1001"}`, screenPreviewDefaultWidth}, + {"zero", `{"id":"1001","max_width":0}`, screenPreviewDefaultWidth}, + {"negative", `{"id":"1001","max_width":-5}`, screenPreviewDefaultWidth}, + {"below floor", `{"id":"1001","max_width":10}`, screenPreviewMinWidth}, + {"in range", `{"id":"1001","max_width":800}`, 800}, + {"above ceiling", `{"id":"1001","max_width":5000}`, screenPreviewMaxWidth}, + } + rig := newScreenRig(t, "0004") + payload := solidJPEG(t, 2, 2, color.RGBA{0, 0, 255, 255}) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ch := rig.call(tc.args) + reqID, got := rig.nextRequest() + if got != tc.want { + t.Fatalf("maxWidth = %d, want %d", got, tc.want) + } + rig.s.OnPacket(rig.main, previewPacket(reqID, screenPreviewStatusOK, screenPreviewFmtJPEG, 2, 2, payload)) + if c := rig.wait(ch); c.err != nil { + t.Fatalf("call error: %+v", c.err) + } + }) + } +} + +func TestScreenshotRejectsRegionParams(t *testing.T) { + cases := []struct { + name string + args string + code int + }{ + {"mode region", `{"id":"1001","mode":"region"}`, CodeUnsupported}, + {"mode window", `{"id":"1001","mode":"window"}`, CodeUnsupported}, + {"mode monitor", `{"id":"1001","mode":"monitor"}`, CodeUnsupported}, + {"hwnd", `{"id":"1001","hwnd":123}`, CodeUnsupported}, + {"hwnd string", `{"id":"1001","hwnd":"0x1F"}`, CodeUnsupported}, + {"x", `{"id":"1001","x":5}`, CodeUnsupported}, + {"y", `{"id":"1001","y":5}`, CodeUnsupported}, + {"width", `{"id":"1001","width":100}`, CodeUnsupported}, + {"height", `{"id":"1001","height":100}`, CodeUnsupported}, + {"monitor", `{"id":"1001","monitor":0}`, CodeUnsupported}, + {"invalid mode", `{"id":"1001","mode":"bogus"}`, CodeInvalidParams}, + } + rig := newScreenRig(t, "0004") + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := rig.wait(rig.call(tc.args)) + if c.err == nil || c.err.Code != tc.code { + t.Fatalf("error = %+v, want code %d", c.err, tc.code) + } + }) + } + if len(rig.writes) != 0 { + t.Fatal("a request was sent for a rejected capture mode") + } +} + +func TestScreenshotRejectsBadArgs(t *testing.T) { + rig := newScreenRig(t, "0004") + cases := []struct { + name string + args string + code int + }{ + {"missing id", `{}`, CodeInvalidParams}, + {"non-decimal id", `{"id":"abc"}`, CodeInvalidParams}, + {"max_width type", `{"id":"1001","max_width":"wide"}`, CodeInvalidParams}, + {"offline", `{"id":"9999"}`, CodeOffline}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := rig.wait(rig.call(tc.args)) + if c.err == nil || c.err.Code != tc.code { + t.Fatalf("error = %+v, want code %d", c.err, tc.code) + } + }) + } +} + +func TestScreenshotWithoutHub(t *testing.T) { + s := New(Config{Token: testToken}) + defer s.Close() + res, rerr := s.callGetScreenshot(context.Background(), []byte(`{"id":"1"}`)) + if res != nil || rerr == nil || rerr.Code != CodeInternal { + t.Fatalf("res=%v err=%+v, want -32000 without a hub", res, rerr) + } +} + +func TestParseScreenPreviewRsp(t *testing.T) { + valid := previewPacket(7, screenPreviewStatusOK, screenPreviewFmtJPEG, 2, 3, []byte{1, 2, 3, 4}) + if rsp, err := parseScreenPreviewRsp(valid); err != nil || rsp.reqID != 7 || len(rsp.reply.payload) != 4 { + t.Fatalf("valid packet: rsp=%+v err=%v", rsp, err) + } + + // A metadata tail after the declared payload is legal and ignored. + withMeta := append(append([]byte(nil), valid...), make([]byte, 24)...) + rsp, err := parseScreenPreviewRsp(withMeta) + if err != nil || len(rsp.reply.payload) != 4 { + t.Fatalf("packet with meta tail: rsp=%+v err=%v", rsp, err) + } + + short := valid[:screenPreviewRspHeaderLen-1] + if _, err := parseScreenPreviewRsp(short); err == nil { + t.Fatal("accepted a packet shorter than the header") + } + wrongToken := append([]byte(nil), valid...) + wrongToken[0] = protocol.TokenNextScreen + if _, err := parseScreenPreviewRsp(wrongToken); err == nil { + t.Fatal("accepted a packet with the wrong token") + } + overrun := append([]byte(nil), valid...) + binary.LittleEndian.PutUint32(overrun[screenPreviewOffBytes:], 99) + if _, err := parseScreenPreviewRsp(overrun); err == nil { + t.Fatal("accepted a packet whose byte count overruns the packet") + } +} + +func TestBGRXToJPEGRejectsMalformed(t *testing.T) { + cases := []struct { + name string + src []byte + w, h int + }{ + {"zero width", make([]byte, 16), 0, 4}, + {"short frame", make([]byte, 15), 2, 2}, + {"long frame", make([]byte, 17), 2, 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := bgrxToJPEG(tc.src, tc.w, tc.h); err == nil { + t.Fatal("converted a malformed BGRX frame") + } + }) + } +} + +func TestOnPacketIgnoresOtherTokens(t *testing.T) { + rig := newScreenRig(t, "0004") + if rig.s.OnPacket(rig.main, nil) { + t.Fatal("consumed an empty packet") + } + if rig.s.OnPacket(rig.main, []byte{protocol.TokenNextScreen, 0}) { + t.Fatal("consumed a screen frame") + } + if !rig.s.OnPacket(rig.main, []byte{protocol.TokenScreenPreviewRsp}) { + t.Fatal("did not consume (and log) a truncated preview response") + } +} diff --git a/server/go/mcp/server.go b/server/go/mcp/server.go index b5049c2..99ce3af 100644 --- a/server/go/mcp/server.go +++ b/server/go/mcp/server.go @@ -76,6 +76,7 @@ type Server struct { locks *deviceLocks sem chan struct{} sessions *sessionTable + previews *previewTable // in-flight get_screenshot requests, keyed by device // stop ends the idle reaper and reaperDone reports that it returned; // closeOnce makes Close idempotent (tests call it both explicitly and from @@ -114,6 +115,7 @@ func New(cfg Config) *Server { handlers: make(map[string]toolFunc), locks: newDeviceLocks(), sem: make(chan struct{}, cfg.MaxConcurrency), + previews: newPreviewTable(), stop: make(chan struct{}), } if cfg.Hub != nil { @@ -122,6 +124,7 @@ func New(cfg Config) *Server { s.sessions = newSessionTable(nil) } s.registerLocalTools() + s.registerScreenTools() s.registerTerminalTools() if cfg.Hub != nil { // No hub means no shell can ever be opened, so there is nothing to diff --git a/server/go/mcp/tools_local.go b/server/go/mcp/tools_local.go index 6f4c9b3..2d0177a 100644 --- a/server/go/mcp/tools_local.go +++ b/server/go/mcp/tools_local.go @@ -37,9 +37,14 @@ type toolResult struct { IsError bool `json:"isError"` } +// contentItem is one MCP content block. Text tools fill Text; get_screenshot +// emits an image block (Data is base64, MimeType image/jpeg), so both the text +// and image fields are optional on the wire. type contentItem struct { - Type string `json:"type"` - Text string `json:"text"` + Type string `json:"type"` + Text string `json:"text,omitempty"` + Data string `json:"data,omitempty"` + MimeType string `json:"mimeType,omitempty"` } func newToolResult(text string, structured any) toolResult { diff --git a/server/go/protocol/commands.go b/server/go/protocol/commands.go index b8fec62..cc6b799 100644 --- a/server/go/protocol/commands.go +++ b/server/go/protocol/commands.go @@ -141,8 +141,14 @@ const ( TokenTerminalStart byte = 232 // TOKEN_TERMINAL_START - modern PTY shell sub-conn open TokenTerminalClose byte = 233 // TOKEN_TERMINAL_CLOSE - shell exited / close ack TokenConnAuth byte = 246 // TOKEN_CONN_AUTH - sub-connection identity handshake - CmdCursorImage byte = 93 // CMD_CURSOR_IMAGE - custom cursor bitmap (Phase 5+ feature) - CmdSockDllLoader byte = 210 // SOCKET_DLLLOADER - shellcode/memory-DLL loader requests the full DLL payload + // COMMAND_SCREEN_PREVIEW_REQ / TOKEN_SCREEN_PREVIEW_RSP are a main-connection + // RPC pair: the client captures, scales and JPEG-encodes a frame itself. It + // opens no sub-connection and holds no screen-session slot, so it can run + // alongside a live screen viewer (design §7.1). + CommandScreenPreviewReq byte = 247 // COMMAND_SCREEN_PREVIEW_REQ - [cmd:1][reqId:2][maxWidth:2][quality:1][reserved:2] + TokenScreenPreviewRsp byte = 248 // TOKEN_SCREEN_PREVIEW_RSP - [token:1][reqId:2][status:1][format:1][w:2][h:2][bytes:4][reserved:3][data] + CmdCursorImage byte = 93 // CMD_CURSOR_IMAGE - custom cursor bitmap (Phase 5+ feature) + CmdSockDllLoader byte = 210 // SOCKET_DLLLOADER - shellcode/memory-DLL loader requests the full DLL payload ) // HeartbeatACK (CMD_HEARTBEAT_ACK) response layout — mirrors the HeartbeatACK