"""校验 tools.json 与 MCP 服务端的 tools/list 是否一致。 三种模式: python verify_tools.py --offline mcp/tools.json tools_raw.json 对比 tools.json 与已保存的黄金样本(不联网) python verify_tools.py --live mcp/tools.json [token] [port] 抓取运行中服务端的 tools/list 再对比(回归测试用) token 省略时依次取环境变量 YAMA_MCP_TOKEN、注册表 McpToken; port 省略时自动探测 6545(明文后端)/ 6544(TLS 前端) python verify_tools.py --subset mcp/tools.json [token] [port] [--readonly] 校验 Go 服务端(server/go)的 MCP 端点:名字集合必须恰好是它实现的那 9 个工具(只读模式 7 个),且 description / inputSchema / outputSchema 与 tools.json 里的同名工具逐字段一致。Go 端把 tools_minimal.json 内嵌 进二进制,这一模式检查的就是线上真正发布的字节。 比对内容:name / description / inputSchema / outputSchema,逐字段深度比较, 外加工具顺序。gate / write / manage / impl 是 tools.json 独有的元信息,不参与比对; 例外是 gate == "!machine" 的降级工具——它们只在 McpMachine=0 时出现,不进本次比对。 """ import http.client import json import os import ssl import sys # Go 最小服务端实现的 9 个工具(docs/Mcp_Go_Minimal_Design.md §3.1)。 # tools/list 恰好是这些,多一个少一个都算失败。 GO_MINIMAL_TOOLS = ( "list_hosts", "get_host_detail", "get_screenshot", "list_files", "get_audit_log", "get_server_status", "terminal_exec", "terminal_close", "read_file", ) # YAMA_MCP_READONLY=1 时 Go 端隐藏的写工具(terminal_* 会创建/关闭会话)。 GO_READONLY_HIDDEN = ("terminal_exec", "terminal_close") FIELDS = ("description", "inputSchema", "outputSchema") def normalize(obj): """规范化:键排序后序列化,消除 JSON object 的键序差异。""" return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":")) def resolve_token(argv_token=""): if argv_token: return argv_token env = os.environ.get("YAMA_MCP_TOKEN", "") if env: return env try: import winreg key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\YAMA\settings") val, _ = winreg.QueryValueEx(key, "McpToken") return val except Exception: return "" def fetch_live(token, port=None): body = json.dumps({ "jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}, }, ensure_ascii=False).encode("utf-8") if port is not None: candidates = [(port, port == 6544)] else: candidates = [(6545, False), (6544, True)] last = None for p, tls in candidates: try: if tls: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = http.client.HTTPSConnection("127.0.0.1", p, timeout=30, context=ctx) else: conn = http.client.HTTPConnection("127.0.0.1", p, timeout=30) conn.request("POST", "/mcp", body, { "Authorization": "Bearer " + token, "Content-Type": "application/json", "Accept": "application/json, text/event-stream", }) resp = conn.getresponse() raw = resp.read() conn.close() if resp.status == 200: print(f"连接 127.0.0.1:{p} ({'TLS' if tls else 'plain'})") obj = json.loads(raw.decode("utf-8")) tools = obj["result"]["tools"] stats = { "response_bytes": len(raw), "tools_array_bytes": len( json.dumps(tools, ensure_ascii=False, separators=(",", ":")).encode("utf-8")), "description_bytes": sum(len(t.get("description", "").encode("utf-8")) for t in tools), "input_schema_bytes": sum( len(json.dumps(t.get("inputSchema", {}), ensure_ascii=False, separators=(",", ":")).encode("utf-8")) for t in tools), "output_schema_bytes": sum( len(json.dumps(t.get("outputSchema", {}), ensure_ascii=False, separators=(",", ":")).encode("utf-8")) for t in tools), } return tools, stats last = f"{p}: HTTP {resp.status} {raw[:200].decode('utf-8', 'replace')}" except Exception as e: last = f"{p}: {e}" sys.exit(f"连接失败 -> {last}") def report_field_diffs(live_by_name, golden_by_name, names): """逐字段比较同名工具,返回差异处数;一个工具最多报一处。""" diffs = 0 for name in names: a, b = live_by_name[name], golden_by_name[name] for f in FIELDS: va, vb = normalize(a.get(f)), normalize(b.get(f)) if va != vb: diffs += 1 print(f"!! {name}.{f} 不一致") for i, (ca, cb) in enumerate(zip(va, vb)): if ca != cb: print(f" @{i}") print(f" 服务端: ...{va[max(0,i-60):i+80]}...") print(f" json : ...{vb[max(0,i-60):i+80]}...") break else: print(f" 长度不同: {len(va)} vs {len(vb)}") break return diffs def verify_subset(path, token, port, readonly): """Go 最小服务端:名字集合恰好是 GO_MINIMAL_TOOLS,schema 与 tools.json 一致。""" expected = [n for n in GO_MINIMAL_TOOLS if not (readonly and n in GO_READONLY_HIDDEN)] tools_live, live_stats = fetch_live(token, port) print(f"tools.json: {path}") print(f"模式: {'只读(隐藏 ' + ' / '.join(GO_READONLY_HIDDEN) + ')' if readonly else '读写'}") print(f"期望: {len(expected)} 个工具 / 服务端: {len(tools_live)} 个") live_names = [t["name"] for t in tools_live] expected_set, live_set = set(expected), set(live_names) diffs = 0 extra = sorted(live_set - expected_set) missing = [n for n in expected if n not in live_set] if extra: diffs += len(extra) print(f"!! 服务端多出: {extra}") if missing: diffs += len(missing) print(f"!! 服务端缺少: {missing}") dupes = sorted({n for n in live_names if live_names.count(n) > 1}) if dupes: diffs += len(dupes) print(f"!! 服务端重复: {dupes}") # 顺序一致性:tools/list 的数组顺序会影响模型的选择倾向 if not extra and not missing and not dupes: order = [n for n in live_names if n in expected_set] if order != expected: diffs += 1 print("!! 工具顺序与 tools.json 不一致") for i, (x, y) in enumerate(zip(order, expected)): if x != y: print(f" 首个差异 @{i}: 服务端={x} 期望={y}") break else: print("工具顺序: 一致") doc = json.load(open(path, encoding="utf-8")) tools_json = doc["tools"] if isinstance(doc, dict) else doc golden_by_name = {t["name"]: t for t in tools_json} live_by_name = {t["name"]: t for t in tools_live} comparable = [] for name in expected: if name not in golden_by_name: diffs += 1 print(f"!! tools.json 缺少工具 {name}") elif name in live_by_name: comparable.append(name) diffs += report_field_diffs(live_by_name, golden_by_name, comparable) print(f" 体积: 响应 {live_stats['response_bytes']} B / tools 数组 " f"{live_stats['tools_array_bytes']} B") if diffs == 0: print(f"\nPASS: Go 服务端 {len(expected)} 个工具与 tools.json 完全一致") return 0 print(f"\nFAIL: {diffs} 处差异") return 1 def main(): args = sys.argv[1:] if not args: sys.exit(__doc__) if args[0] == "--subset": argv = [a for a in args[1:]] readonly = "--readonly" in argv argv = [a for a in argv if a != "--readonly"] if not argv: sys.exit("用法: python verify_tools.py --subset [token] [port] [--readonly]") path = argv[0] token = resolve_token(argv[1] if len(argv) > 1 else "") port = int(argv[2]) if len(argv) > 2 else None if not token: sys.exit("未找到 McpToken(参数 / YAMA_MCP_TOKEN / 注册表 均无)") return verify_subset(path, token, port, readonly) if args[0] == "--offline": if len(args) < 3: sys.exit("用法: python verify_tools.py --offline <黄金样本.json>") path, sample = args[1], args[2] tools_live = json.load(open(sample, encoding="utf-8"))["result"]["tools"] print(f"黄金样本: {sample}") elif args[0] == "--live": if len(args) < 2: sys.exit("用法: python verify_tools.py --live [token] [port]") path = args[1] token = resolve_token(args[2] if len(args) > 2 else "") port = int(args[3]) if len(args) > 3 else None if not token: sys.exit("未找到 McpToken(参数 / YAMA_MCP_TOKEN / 注册表 均无)") tools_live, live_stats = fetch_live(token, port) print(f"活服务: 127.0.0.1:{port if port else 'auto'}") print(f"体积: 响应 {live_stats['response_bytes']} B / tools 数组 {live_stats['tools_array_bytes']} B" f"(description {live_stats['description_bytes']} / inputSchema {live_stats['input_schema_bytes']}" f" / outputSchema {live_stats['output_schema_bytes']})") else: sys.exit(__doc__) doc = json.load(open(path, encoding="utf-8")) tools_json = doc["tools"] # gate == "!machine" 的工具只在 McpMachine=0 时出现,而黄金样本/活服务都是开关 # 全开的抓取,按约定它们不该在这里出现(tools.json 是两份抓取的并集)。 off_gated = {t["name"] for t in tools_json if t.get("gate") == "!machine"} tools_cmp = [t for t in tools_json if t["name"] not in off_gated] print(f"tools.json: {path} ({len(tools_json)} 个工具)") if off_gated: print(f" 其中 {len(off_gated)} 个 !machine 降级工具不计入本次比对: {sorted(off_gated)}") print(f"服务端: {len(tools_live)} 个工具") if len(tools_cmp) != len(tools_live): print(f"!! 数量不一致: {len(tools_cmp)} vs {len(tools_live)}") live_by_name = {t["name"]: t for t in tools_live} json_by_name = {t["name"]: t for t in tools_cmp} only_live = sorted(set(live_by_name) - set(json_by_name)) only_json = sorted(set(json_by_name) - set(live_by_name)) if only_live: print(f"!! 仅服务端有: {only_live}") if only_json: print(f"!! 仅 json 有: {only_json}") diffs = 0 FIELDS_SHARED = sorted(set(live_by_name) & set(json_by_name)) diffs += report_field_diffs(live_by_name, json_by_name, FIELDS_SHARED) # 顺序一致性:tools/list 的数组顺序会影响模型的选择倾向 order_live = [t["name"] for t in tools_live] order_json = [t["name"] for t in tools_cmp] if order_live != order_json: print("!! 工具顺序不一致") for i, (x, y) in enumerate(zip(order_live, order_json)): if x != y: print(f" 首个差异 @{i}: 服务端={x} json={y}") break diffs += 1 else: print("工具顺序: 一致") if diffs == 0 and not only_live and not only_json: print(f"\nPASS: {len(tools_cmp)} 个工具完全一致") return 0 print(f"\nFAIL: {diffs} 处差异") return 1 if __name__ == "__main__": sys.exit(main())