COMMAND_SCREEN_PREVIEW_REQ (247) / TOKEN_SCREEN_PREVIEW_RSP (248) is a request/reply pair on the device's main connection: the client captures, scales and encodes the frame itself, so no sub-connection is opened and no screen-session slot is taken. get_screenshot can therefore run while a human is watching the same machine. The server keeps one pending request per device in a small table, matches replies by reqId and drops foreign or late ones, and bounds a round trip with an 8s timer. A missing CLIENT_CAP_SCREEN_PREVIEW bit is refused with -32005 before anything is sent; region/window/monitor parameters are refused with the same code because the v2 CLIENT_CAP_SCREEN_REGION extension is not implemented in this build. Status 4 maps to -32602 because the caller can fix the target, while capture/encode failures stay -32000. Windows clients return JPEG directly. Linux clients have no JPEG encoder and return top-down BGRX, which the server converts (R/B swap, opaque alpha) before wrapping the frame as an MCP image item. The image item is new to the content envelope; text tools keep text-only items. Verified live against x64\Debug\ghost.exe (JPEG FF D8 FF, decoded size matches metadata, max_width clamped to 64) and against linux/ghost under WSL (BGRX path, valid JPEG, black frame from the known Xwayland capture limitation). gofmt/vet clean, unit and -race tests green. Co-Authored-By: deepseek-flash
425 lines
14 KiB
Go
425 lines
14 KiB
Go
package mcp
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
"image/jpeg"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/yuanyuanxiang/SimpleRemoter/server/go/connection"
|
|
"github.com/yuanyuanxiang/SimpleRemoter/server/go/hub"
|
|
"github.com/yuanyuanxiang/SimpleRemoter/server/go/protocol"
|
|
)
|
|
|
|
// get_screenshot uses the main-connection preview RPC
|
|
// (COMMAND_SCREEN_PREVIEW_REQ → TOKEN_SCREEN_PREVIEW_RSP): the client captures,
|
|
// scales and encodes the frame itself, so the request opens no sub-connection
|
|
// and competes with nobody for the per-device screen slot (design §7.1).
|
|
const (
|
|
screenPreviewDefaultWidth = 1024
|
|
screenPreviewMinWidth = 64
|
|
screenPreviewMaxWidth = 1920
|
|
screenPreviewJPEGQuality = 80
|
|
|
|
// screenPreviewRspHeaderLen is sizeof(ScreenPreviewRspHeader) with
|
|
// #pragma pack(1) (common/commands.h); the image bytes follow directly.
|
|
screenPreviewRspHeaderLen = 16
|
|
screenPreviewOffReqID = 1
|
|
screenPreviewOffStatus = 3
|
|
screenPreviewOffFormat = 4
|
|
screenPreviewOffWidth = 5
|
|
screenPreviewOffHeight = 7
|
|
screenPreviewOffBytes = 9
|
|
)
|
|
|
|
// ScreenPreviewStatus / ScreenPreviewFormat values (common/commands.h). Only
|
|
// two formats exist on the wire: Windows clients send JPEG, Linux clients send
|
|
// BGRX because they carry no JPEG encoder — the server encodes it here.
|
|
const (
|
|
screenPreviewStatusOK byte = 0
|
|
screenPreviewStatusCaptureFailed byte = 1
|
|
screenPreviewStatusEncodeFailed byte = 2
|
|
screenPreviewStatusNotSupported byte = 3
|
|
screenPreviewStatusBadTarget byte = 4
|
|
|
|
screenPreviewFmtJPEG byte = 0
|
|
screenPreviewFmtBGRX byte = 3
|
|
)
|
|
|
|
// screenPreviewTimeout bounds one round trip. It is a variable rather than a
|
|
// constant only so tests can shorten it; production code never changes it.
|
|
// There is deliberately no cancellation on timeout: the request rides the
|
|
// device's main connection, where MFC likewise refuses CancelIO (§7.4) — the
|
|
// pending entry is dropped and a late reply fails the reqId match.
|
|
var screenPreviewTimeout = 8 * time.Second
|
|
|
|
// screenshotArgs is the published inputSchema as Go. Pointer fields separate
|
|
// "absent" from an explicit value: mode=screen must still reject the
|
|
// region/window/monitor parameters instead of silently ignoring them.
|
|
type screenshotArgs struct {
|
|
ID string `json:"id"`
|
|
MaxWidth *int `json:"max_width"`
|
|
Mode string `json:"mode"`
|
|
HWND *json.RawMessage `json:"hwnd"`
|
|
X *int `json:"x"`
|
|
Y *int `json:"y"`
|
|
Width *int `json:"width"`
|
|
Height *int `json:"height"`
|
|
Monitor *int `json:"monitor"`
|
|
}
|
|
|
|
// screenshotImage / screenshotCapture are the structuredContent fields the
|
|
// shared outputSchema defines. Fields the v1 RPC cannot report are omitted
|
|
// rather than faked (design §3.2).
|
|
type screenshotImage struct {
|
|
MimeType string `json:"mimeType"`
|
|
Width int `json:"width"`
|
|
Height int `json:"height"`
|
|
Bytes int `json:"bytes"`
|
|
}
|
|
|
|
type screenshotCapture struct {
|
|
Mode string `json:"mode"`
|
|
}
|
|
|
|
type screenshotOutput struct {
|
|
Image screenshotImage `json:"image"`
|
|
Capture screenshotCapture `json:"capture"`
|
|
}
|
|
|
|
// previewReply is one decoded TOKEN_SCREEN_PREVIEW_RSP.
|
|
type previewReply struct {
|
|
status byte
|
|
format byte
|
|
width uint16
|
|
height uint16
|
|
payload []byte
|
|
}
|
|
|
|
// previewRequest is one in-flight screenshot. ch is buffered so delivery never
|
|
// blocks the TCP read loop.
|
|
type previewRequest struct {
|
|
reqID uint16
|
|
ch chan previewReply
|
|
}
|
|
|
|
// previewTable matches a client reply to the call waiting for it. The
|
|
// per-device call lock already serialises MCP screenshots, so at most one
|
|
// entry exists per device; responses are still keyed by device because the
|
|
// reply itself carries no device id — only the connection does.
|
|
type previewTable struct {
|
|
mu sync.Mutex
|
|
byDevice map[string]*previewRequest
|
|
nextID atomic.Uint32
|
|
}
|
|
|
|
func newPreviewTable() *previewTable {
|
|
return &previewTable{byDevice: make(map[string]*previewRequest)}
|
|
}
|
|
|
|
// nextReqID mirrors MFC's NextPreviewReqId: increment and skip 0, which would
|
|
// otherwise match an "unset" reqId on the client side.
|
|
func (t *previewTable) nextReqID() uint16 {
|
|
for {
|
|
if v := uint16(t.nextID.Add(1)); v != 0 {
|
|
return v
|
|
}
|
|
}
|
|
}
|
|
|
|
// begin reserves the device's single preview slot. nil means another request
|
|
// is already pending (only possible if the call lock was bypassed).
|
|
func (t *previewTable) begin(deviceID string) *previewRequest {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
if _, ok := t.byDevice[deviceID]; ok {
|
|
return nil
|
|
}
|
|
r := &previewRequest{reqID: t.nextReqID(), ch: make(chan previewReply, 1)}
|
|
t.byDevice[deviceID] = r
|
|
return r
|
|
}
|
|
|
|
// cancel drops a pending slot unless it was already delivered. The pointer
|
|
// check keeps a slow caller from deleting a newer request for the same device.
|
|
func (t *previewTable) cancel(deviceID string, r *previewRequest) {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
if t.byDevice[deviceID] == r {
|
|
delete(t.byDevice, deviceID)
|
|
}
|
|
}
|
|
|
|
// deliver hands a matching reply to its waiter. A reply whose reqId does not
|
|
// match — an expired request or one from another caller — is dropped, exactly
|
|
// like MFC's TakePreviewResponse.
|
|
func (t *previewTable) deliver(deviceID string, reqID uint16, reply previewReply) bool {
|
|
t.mu.Lock()
|
|
r, ok := t.byDevice[deviceID]
|
|
if !ok || r.reqID != reqID {
|
|
t.mu.Unlock()
|
|
return false
|
|
}
|
|
delete(t.byDevice, deviceID)
|
|
t.mu.Unlock()
|
|
r.ch <- reply // buffered, and the entry is gone: nobody else can send
|
|
return true
|
|
}
|
|
|
|
func (s *Server) registerScreenTools() {
|
|
s.handlers["get_screenshot"] = s.callGetScreenshot
|
|
}
|
|
|
|
// OnPacket feeds one packet from a device connection to the MCP layer. It
|
|
// reports whether MCP consumed the packet; anything MCP does not own returns
|
|
// false and falls through to cmd's normal command dispatch.
|
|
func (s *Server) OnPacket(ctx *connection.Context, data []byte) bool {
|
|
if s == nil || len(data) == 0 || data[0] != protocol.TokenScreenPreviewRsp {
|
|
return false
|
|
}
|
|
deviceID := ctx.GetInfo().ClientID
|
|
rsp, err := parseScreenPreviewRsp(data)
|
|
if err != nil {
|
|
s.log.Warn("malformed screen preview response from conn %d (device=%q): %v",
|
|
ctx.ID, deviceID, err)
|
|
return true
|
|
}
|
|
if deviceID != "" {
|
|
s.previews.deliver(deviceID, rsp.reqID, rsp.reply)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// parseScreenPreviewRsp validates the 16-byte header and copies the image
|
|
// bytes out of the TCP read buffer. A declared length that overruns the packet
|
|
// marks the whole response malformed: the client would otherwise hand us an
|
|
// under-filled image.
|
|
func parseScreenPreviewRsp(data []byte) (struct {
|
|
reqID uint16
|
|
reply previewReply
|
|
}, error) {
|
|
var rsp struct {
|
|
reqID uint16
|
|
reply previewReply
|
|
}
|
|
if len(data) < screenPreviewRspHeaderLen {
|
|
return rsp, fmt.Errorf("packet is %d bytes, want at least %d",
|
|
len(data), screenPreviewRspHeaderLen)
|
|
}
|
|
if data[0] != protocol.TokenScreenPreviewRsp {
|
|
return rsp, fmt.Errorf("token %d, want %d", data[0], protocol.TokenScreenPreviewRsp)
|
|
}
|
|
rsp.reqID = binary.LittleEndian.Uint16(data[screenPreviewOffReqID:])
|
|
rsp.reply.status = data[screenPreviewOffStatus]
|
|
rsp.reply.format = data[screenPreviewOffFormat]
|
|
rsp.reply.width = binary.LittleEndian.Uint16(data[screenPreviewOffWidth:])
|
|
rsp.reply.height = binary.LittleEndian.Uint16(data[screenPreviewOffHeight:])
|
|
n := binary.LittleEndian.Uint32(data[screenPreviewOffBytes:])
|
|
if avail := len(data) - screenPreviewRspHeaderLen; n > uint32(avail) {
|
|
return rsp, fmt.Errorf("declares %d image bytes but only %d follow", n, avail)
|
|
}
|
|
rsp.reply.payload = append([]byte(nil), data[screenPreviewRspHeaderLen:screenPreviewRspHeaderLen+int(n)]...)
|
|
return rsp, nil
|
|
}
|
|
|
|
// buildScreenPreviewReq renders the 8-byte v1 request (whole virtual desktop).
|
|
// v2 (region/window/monitor) is gated behind CLIENT_CAP_SCREEN_REGION and is
|
|
// not implemented by this server build.
|
|
func buildScreenPreviewReq(reqID, maxWidth uint16, quality byte) []byte {
|
|
buf := make([]byte, 8)
|
|
buf[0] = protocol.CommandScreenPreviewReq
|
|
binary.LittleEndian.PutUint16(buf[1:3], reqID)
|
|
binary.LittleEndian.PutUint16(buf[3:5], maxWidth)
|
|
buf[5] = quality
|
|
return buf
|
|
}
|
|
|
|
// callGetScreenshot captures one frame through the main-connection RPC.
|
|
func (s *Server) callGetScreenshot(ctx context.Context, raw json.RawMessage) (any, *rpcError) {
|
|
if s.cfg.Hub == nil {
|
|
return nil, errf(CodeInternal, "server device registry is not configured")
|
|
}
|
|
var args screenshotArgs
|
|
if rerr := decodeArgs(raw, &args); rerr != nil {
|
|
return nil, rerr
|
|
}
|
|
if args.ID == "" {
|
|
return nil, errf(CodeInvalidParams, "Missing required parameter: id")
|
|
}
|
|
if !isDecimalID(args.ID) {
|
|
return nil, errf(CodeInvalidParams, "Invalid id: expected a decimal host id string")
|
|
}
|
|
if s.cfg.Hub.MainConn(args.ID) == nil {
|
|
return nil, errf(CodeOffline, "Host not found or offline: %s", args.ID)
|
|
}
|
|
if !protocol.SupportsCap(s.cfg.Hub.Capability(args.ID), protocol.ClientCapScreenPreview) {
|
|
return nil, errf(CodeUnsupported,
|
|
"Host does not support screen preview: the client does not declare CLIENT_CAP_SCREEN_PREVIEW (upgrade the client)")
|
|
}
|
|
|
|
mode := strings.TrimSpace(args.Mode)
|
|
if mode == "" {
|
|
mode = "screen"
|
|
}
|
|
if mode != "screen" {
|
|
switch mode {
|
|
case "region", "window", "monitor":
|
|
return nil, errf(CodeUnsupported,
|
|
"mode=%s needs the CLIENT_CAP_SCREEN_REGION extension, which this server build does not implement; use mode=screen", mode)
|
|
default:
|
|
return nil, errf(CodeInvalidParams,
|
|
"Invalid mode: %s (expect screen|region|window|monitor)", mode)
|
|
}
|
|
}
|
|
if name := firstRegionParam(&args); name != "" {
|
|
return nil, errf(CodeUnsupported,
|
|
"parameter %q needs the CLIENT_CAP_SCREEN_REGION extension, which this server build does not implement; use mode=screen with only id and max_width", name)
|
|
}
|
|
|
|
maxWidth := screenPreviewDefaultWidth
|
|
if args.MaxWidth != nil && *args.MaxWidth > 0 {
|
|
maxWidth = min(max(*args.MaxWidth, screenPreviewMinWidth), screenPreviewMaxWidth)
|
|
}
|
|
|
|
req := s.previews.begin(args.ID)
|
|
if req == nil {
|
|
return nil, errf(CodeBusy,
|
|
"Device busy: another screenshot request is already pending for host %s", args.ID)
|
|
}
|
|
defer s.previews.cancel(args.ID, req)
|
|
|
|
if err := s.cfg.Hub.SendToDevice(args.ID,
|
|
buildScreenPreviewReq(req.reqID, uint16(maxWidth), screenPreviewJPEGQuality)); err != nil {
|
|
if errors.Is(err, hub.ErrDeviceOffline) {
|
|
return nil, errf(CodeOffline, "Host not found or offline: %s", args.ID)
|
|
}
|
|
return nil, errf(CodeSendFailed, "Failed to send screenshot request to host: %v", err)
|
|
}
|
|
|
|
timer := time.NewTimer(screenPreviewTimeout)
|
|
defer timer.Stop()
|
|
select {
|
|
case reply := <-req.ch:
|
|
return screenshotResult(reply, mode)
|
|
case <-timer.C:
|
|
return nil, errf(CodeTimeout,
|
|
"Timeout waiting for the screenshot (the host did not answer in %s; a client without screen-preview support cannot be asked)", screenPreviewTimeout)
|
|
case <-ctx.Done():
|
|
return nil, errf(CodeTimeout, "Request cancelled while waiting for the screenshot")
|
|
}
|
|
}
|
|
|
|
// firstRegionParam names the first v2 capture parameter present in the call.
|
|
// Checked in schema order so the error message is deterministic.
|
|
func firstRegionParam(args *screenshotArgs) string {
|
|
switch {
|
|
case args.HWND != nil:
|
|
return "hwnd"
|
|
case args.X != nil:
|
|
return "x"
|
|
case args.Y != nil:
|
|
return "y"
|
|
case args.Width != nil:
|
|
return "width"
|
|
case args.Height != nil:
|
|
return "height"
|
|
case args.Monitor != nil:
|
|
return "monitor"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// screenshotResult maps the device status/format onto the MCP result. Status 4
|
|
// is a caller-visible parameter problem (target invalid), so it keeps -32602;
|
|
// capture/encode failures are -32000 because retrying the same call cannot fix
|
|
// them.
|
|
func screenshotResult(reply previewReply, mode string) (any, *rpcError) {
|
|
switch reply.status {
|
|
case screenPreviewStatusOK:
|
|
case screenPreviewStatusCaptureFailed:
|
|
return nil, errf(CodeInternal, "Screenshot capture failed on the host (status 1)")
|
|
case screenPreviewStatusEncodeFailed:
|
|
return nil, errf(CodeInternal, "Screenshot encoding failed on the host (status 2)")
|
|
case screenPreviewStatusNotSupported:
|
|
return nil, errf(CodeUnsupported, "Host does not support screen capture (status 3)")
|
|
case screenPreviewStatusBadTarget:
|
|
return nil, errf(CodeInvalidParams, "Screenshot target is invalid (status 4)")
|
|
default:
|
|
return nil, errf(CodeInternal, "Screenshot failed with unknown status %d", reply.status)
|
|
}
|
|
|
|
w, h := int(reply.width), int(reply.height)
|
|
switch reply.format {
|
|
case screenPreviewFmtJPEG:
|
|
if len(reply.payload) == 0 {
|
|
return nil, errf(CodeInternal, "Screenshot response carries no image data")
|
|
}
|
|
return imageResult(reply.payload, w, h, mode), nil
|
|
case screenPreviewFmtBGRX:
|
|
jpg, err := bgrxToJPEG(reply.payload, w, h)
|
|
if err != nil {
|
|
return nil, errf(CodeInternal, "Malformed BGRX screenshot from host: %v", err)
|
|
}
|
|
// The reported width/height describe the pixels, which do not change in
|
|
// the JPEG encode; bytes describes what is actually returned.
|
|
return imageResult(jpg, w, h, mode), nil
|
|
default:
|
|
return nil, errf(CodeInternal, "Unsupported screenshot format %d", reply.format)
|
|
}
|
|
}
|
|
|
|
// bgrxToJPEG converts a top-down 4-byte-per-pixel B,G,R,X frame (Linux
|
|
// clients) into a JPEG. There is no row padding, and X is forced opaque.
|
|
func bgrxToJPEG(src []byte, w, h int) ([]byte, error) {
|
|
if w <= 0 || h <= 0 {
|
|
return nil, fmt.Errorf("invalid dimensions %dx%d", w, h)
|
|
}
|
|
if len(src) != w*h*4 {
|
|
return nil, fmt.Errorf("got %d bytes, want %d for %dx%d BGRX", len(src), w*h*4, w, h)
|
|
}
|
|
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
|
for i := 0; i < w*h; i++ {
|
|
img.Pix[i*4+0] = src[i*4+2] // R ← B
|
|
img.Pix[i*4+1] = src[i*4+1] // G
|
|
img.Pix[i*4+2] = src[i*4+0] // B ← R
|
|
img.Pix[i*4+3] = 0xFF
|
|
}
|
|
var buf bytes.Buffer
|
|
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: screenPreviewJPEGQuality}); err != nil {
|
|
return nil, err
|
|
}
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
// imageResult wraps the encoded JPEG in the MCP envelope: one image content
|
|
// item plus the metadata the outputSchema declares.
|
|
func imageResult(jpg []byte, w, h int, mode string) toolResult {
|
|
return toolResult{
|
|
Content: []contentItem{{
|
|
Type: "image",
|
|
Data: base64.StdEncoding.EncodeToString(jpg),
|
|
MimeType: "image/jpeg",
|
|
}},
|
|
StructuredContent: screenshotOutput{
|
|
Image: screenshotImage{
|
|
MimeType: "image/jpeg",
|
|
Width: w,
|
|
Height: h,
|
|
Bytes: len(jpg),
|
|
},
|
|
// v1 sends no ScreenCaptureMeta, so the source rect and scale are
|
|
// unknown; report the mode only rather than inventing values.
|
|
Capture: screenshotCapture{Mode: mode},
|
|
},
|
|
}
|
|
}
|