"""校验 tools.json 与 MCP 服务端的 tools/list 是否一致。 两种模式: python verify_tools.py --offline mcp/tools.json tools_raw.json 对比 tools.json 与已保存的黄金样本(不联网) python verify_tools.py --live mcp/tools.json [token] [port] 抓取运行中服务端的 tools/list 再对比(回归测试用) token 省略时依次取环境变量 YAMA_MCP_TOKEN、注册表 McpToken; port 省略时自动探测 6545(明文后端)/ 6544(TLS 前端) 比对内容:name / description / inputSchema / outputSchema,逐字段深度比较, 外加工具顺序。gate / write / manage / impl 是 tools.json 独有的元信息,不参与比对。 """ import http.client import json import os import ssl import sys def normalize(obj): """规范化:键排序后序列化,消除 JSON object 的键序差异。""" return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":")) def resolve_token(argv_token=""): if argv_token: return argv_token env = os.environ.get("YAMA_MCP_TOKEN", "") if env: return env try: import winreg key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\YAMA\settings") val, _ = winreg.QueryValueEx(key, "McpToken") return val except Exception: return "" def fetch_live(token, port=None): body = json.dumps({ "jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}, }, ensure_ascii=False).encode("utf-8") if port is not None: candidates = [(port, port == 6544)] else: candidates = [(6545, False), (6544, True)] last = None for p, tls in candidates: try: if tls: ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = http.client.HTTPSConnection("127.0.0.1", p, timeout=30, context=ctx) else: conn = http.client.HTTPConnection("127.0.0.1", p, timeout=30) conn.request("POST", "/mcp", body, { "Authorization": "Bearer " + token, "Content-Type": "application/json", "Accept": "application/json, text/event-stream", }) resp = conn.getresponse() raw = resp.read() conn.close() if resp.status == 200: print(f"连接 127.0.0.1:{p} ({'TLS' if tls else 'plain'})") return json.loads(raw.decode("utf-8"))["result"]["tools"] last = f"{p}: HTTP {resp.status} {raw[:200].decode('utf-8', 'replace')}" except Exception as e: last = f"{p}: {e}" sys.exit(f"连接失败 -> {last}") def main(): args = sys.argv[1:] if not args: sys.exit(__doc__) if args[0] == "--offline": if len(args) < 3: sys.exit("用法: python verify_tools.py --offline <黄金样本.json>") path, sample = args[1], args[2] tools_live = json.load(open(sample, encoding="utf-8"))["result"]["tools"] print(f"黄金样本: {sample}") elif args[0] == "--live": if len(args) < 2: sys.exit("用法: python verify_tools.py --live [token] [port]") path = args[1] token = resolve_token(args[2] if len(args) > 2 else "") port = int(args[3]) if len(args) > 3 else None if not token: sys.exit("未找到 McpToken(参数 / YAMA_MCP_TOKEN / 注册表 均无)") tools_live = fetch_live(token, port) print(f"活服务: 127.0.0.1:{port if port else 'auto'}") else: sys.exit(__doc__) doc = json.load(open(path, encoding="utf-8")) tools_json = doc["tools"] print(f"tools.json: {path} ({len(tools_json)} 个工具)") print(f"服务端: {len(tools_live)} 个工具") if len(tools_json) != len(tools_live): print(f"!! 数量不一致: {len(tools_json)} vs {len(tools_live)}") live_by_name = {t["name"]: t for t in tools_live} json_by_name = {t["name"]: t for t in tools_json} only_live = sorted(set(live_by_name) - set(json_by_name)) only_json = sorted(set(json_by_name) - set(live_by_name)) if only_live: print(f"!! 仅服务端有: {only_live}") if only_json: print(f"!! 仅 json 有: {only_json}") FIELDS = ("description", "inputSchema", "outputSchema") diffs = 0 for name in sorted(set(live_by_name) & set(json_by_name)): a, b = live_by_name[name], json_by_name[name] for f in FIELDS: va, vb = normalize(a.get(f)), normalize(b.get(f)) if va != vb: diffs += 1 print(f"!! {name}.{f} 不一致") for i, (ca, cb) in enumerate(zip(va, vb)): if ca != cb: print(f" @{i}") print(f" 服务端: ...{va[max(0,i-60):i+80]}...") print(f" json : ...{vb[max(0,i-60):i+80]}...") break else: print(f" 长度不同: {len(va)} vs {len(vb)}") break # 一个工具报一次即可 # 顺序一致性:tools/list 的数组顺序会影响模型的选择倾向 order_live = [t["name"] for t in tools_live] order_json = [t["name"] for t in tools_json] if order_live != order_json: print("!! 工具顺序不一致") for i, (x, y) in enumerate(zip(order_live, order_json)): if x != y: print(f" 首个差异 @{i}: 服务端={x} json={y}") break diffs += 1 else: print("工具顺序: 一致") if diffs == 0 and not only_live and not only_json: print(f"\nPASS: {len(tools_json)} 个工具完全一致") return 0 print(f"\nFAIL: {diffs} 处差异") return 1 if __name__ == "__main__": sys.exit(main())