Files
SimpleRemoter/tools/mcp_tools/verify_tools.py
T
yuanyuanxianganddeepseek-flash d4f10db1f2 Feature(Go): verify_tools --subset and MCP deployment docs
Add a third mode to verify_tools.py that validates the Go server end to
end: it fetches /mcp tools/list and requires the name set to equal the
nine implemented tools exactly (seven in readonly mode), compares
description / inputSchema / outputSchema field by field against
mcp/tools.json, and also checks the published order. Unlike --live,
extra or missing tools are failures, not notes.

The field-comparison loop is factored into report_field_diffs so
--offline / --live and --subset share one comparison, diff printer and
FIELDS tuple.

Document and deploy the endpoint: the systemd env example gains
YAMA_MCP_TOKEN / READONLY / TERM_IDLE / MAX_CONCURRENCY / READ_MAX,
the design doc moves to implemented and its section 10 now matches the
code (embedded tools_minimal.json, the three packet dispatch groups).
The tools README explains gen_go_tools.py --check and --subset.

Verified against a freshly built debug Go server on 16543/18090:
read-write mode passes with 9 tools, YAMA_MCP_READONLY=1 passes with 7,
and both mismatch directions (extra tools, missing tools) fail with
exit 1. gen_go_tools.py --check stays clean; no Go code changed.

Co-Authored-By: deepseek-flash
2026-11-12 23:09:33 +01:00

291 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""校验 tools.json 与 MCP 服务端的 tools/list 是否一致。
三种模式:
python verify_tools.py --offline mcp/tools.json tools_raw.json
对比 tools.json 与已保存的黄金样本(不联网)
python verify_tools.py --live mcp/tools.json [token] [port]
抓取运行中服务端的 tools/list 再对比(回归测试用)
token 省略时依次取环境变量 YAMA_MCP_TOKEN、注册表 McpToken;
port 省略时自动探测 6545(明文后端)/ 6544(TLS 前端)
python verify_tools.py --subset mcp/tools.json [token] [port] [--readonly]
校验 Go 服务端(server/go)的 MCP 端点:名字集合必须恰好是它实现的那
9 个工具(只读模式 7 个),且 description / inputSchema / outputSchema
与 tools.json 里的同名工具逐字段一致。Go 端把 tools_minimal.json 内嵌
进二进制,这一模式检查的就是线上真正发布的字节。
比对内容:name / description / inputSchema / outputSchema,逐字段深度比较,
外加工具顺序。gate / write / manage / impl 是 tools.json 独有的元信息,不参与比对;
例外是 gate == "!machine" 的降级工具——它们只在 McpMachine=0 时出现,不进本次比对。
"""
import http.client
import json
import os
import ssl
import sys
# Go 最小服务端实现的 9 个工具(docs/Mcp_Go_Minimal_Design.md §3.1)。
# tools/list 恰好是这些,多一个少一个都算失败。
GO_MINIMAL_TOOLS = (
"list_hosts",
"get_host_detail",
"get_screenshot",
"list_files",
"get_audit_log",
"get_server_status",
"terminal_exec",
"terminal_close",
"read_file",
)
# YAMA_MCP_READONLY=1 时 Go 端隐藏的写工具(terminal_* 会创建/关闭会话)。
GO_READONLY_HIDDEN = ("terminal_exec", "terminal_close")
FIELDS = ("description", "inputSchema", "outputSchema")
def normalize(obj):
"""规范化:键排序后序列化,消除 JSON object 的键序差异。"""
return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def resolve_token(argv_token=""):
if argv_token:
return argv_token
env = os.environ.get("YAMA_MCP_TOKEN", "")
if env:
return env
try:
import winreg
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r"Software\YAMA\settings")
val, _ = winreg.QueryValueEx(key, "McpToken")
return val
except Exception:
return ""
def fetch_live(token, port=None):
body = json.dumps({
"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {},
}, ensure_ascii=False).encode("utf-8")
if port is not None:
candidates = [(port, port == 6544)]
else:
candidates = [(6545, False), (6544, True)]
last = None
for p, tls in candidates:
try:
if tls:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
conn = http.client.HTTPSConnection("127.0.0.1", p, timeout=30, context=ctx)
else:
conn = http.client.HTTPConnection("127.0.0.1", p, timeout=30)
conn.request("POST", "/mcp", body, {
"Authorization": "Bearer " + token,
"Content-Type": "application/json",
"Accept": "application/json, text/event-stream",
})
resp = conn.getresponse()
raw = resp.read()
conn.close()
if resp.status == 200:
print(f"连接 127.0.0.1:{p} ({'TLS' if tls else 'plain'})")
obj = json.loads(raw.decode("utf-8"))
tools = obj["result"]["tools"]
stats = {
"response_bytes": len(raw),
"tools_array_bytes": len(
json.dumps(tools, ensure_ascii=False, separators=(",", ":")).encode("utf-8")),
"description_bytes": sum(len(t.get("description", "").encode("utf-8")) for t in tools),
"input_schema_bytes": sum(
len(json.dumps(t.get("inputSchema", {}), ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
for t in tools),
"output_schema_bytes": sum(
len(json.dumps(t.get("outputSchema", {}), ensure_ascii=False, separators=(",", ":")).encode("utf-8"))
for t in tools),
}
return tools, stats
last = f"{p}: HTTP {resp.status} {raw[:200].decode('utf-8', 'replace')}"
except Exception as e:
last = f"{p}: {e}"
sys.exit(f"连接失败 -> {last}")
def report_field_diffs(live_by_name, golden_by_name, names):
"""逐字段比较同名工具,返回差异处数;一个工具最多报一处。"""
diffs = 0
for name in names:
a, b = live_by_name[name], golden_by_name[name]
for f in FIELDS:
va, vb = normalize(a.get(f)), normalize(b.get(f))
if va != vb:
diffs += 1
print(f"!! {name}.{f} 不一致")
for i, (ca, cb) in enumerate(zip(va, vb)):
if ca != cb:
print(f" @{i}")
print(f" 服务端: ...{va[max(0,i-60):i+80]}...")
print(f" json : ...{vb[max(0,i-60):i+80]}...")
break
else:
print(f" 长度不同: {len(va)} vs {len(vb)}")
break
return diffs
def verify_subset(path, token, port, readonly):
"""Go 最小服务端:名字集合恰好是 GO_MINIMAL_TOOLS,schema 与 tools.json 一致。"""
expected = [n for n in GO_MINIMAL_TOOLS
if not (readonly and n in GO_READONLY_HIDDEN)]
tools_live, live_stats = fetch_live(token, port)
print(f"tools.json: {path}")
print(f"模式: {'只读(隐藏 ' + ' / '.join(GO_READONLY_HIDDEN) + ')' if readonly else '读写'}")
print(f"期望: {len(expected)} 个工具 / 服务端: {len(tools_live)} 个")
live_names = [t["name"] for t in tools_live]
expected_set, live_set = set(expected), set(live_names)
diffs = 0
extra = sorted(live_set - expected_set)
missing = [n for n in expected if n not in live_set]
if extra:
diffs += len(extra)
print(f"!! 服务端多出: {extra}")
if missing:
diffs += len(missing)
print(f"!! 服务端缺少: {missing}")
dupes = sorted({n for n in live_names if live_names.count(n) > 1})
if dupes:
diffs += len(dupes)
print(f"!! 服务端重复: {dupes}")
# 顺序一致性:tools/list 的数组顺序会影响模型的选择倾向
if not extra and not missing and not dupes:
order = [n for n in live_names if n in expected_set]
if order != expected:
diffs += 1
print("!! 工具顺序与 tools.json 不一致")
for i, (x, y) in enumerate(zip(order, expected)):
if x != y:
print(f" 首个差异 @{i}: 服务端={x} 期望={y}")
break
else:
print("工具顺序: 一致")
doc = json.load(open(path, encoding="utf-8"))
tools_json = doc["tools"] if isinstance(doc, dict) else doc
golden_by_name = {t["name"]: t for t in tools_json}
live_by_name = {t["name"]: t for t in tools_live}
comparable = []
for name in expected:
if name not in golden_by_name:
diffs += 1
print(f"!! tools.json 缺少工具 {name}")
elif name in live_by_name:
comparable.append(name)
diffs += report_field_diffs(live_by_name, golden_by_name, comparable)
print(f" 体积: 响应 {live_stats['response_bytes']} B / tools 数组 "
f"{live_stats['tools_array_bytes']} B")
if diffs == 0:
print(f"\nPASS: Go 服务端 {len(expected)} 个工具与 tools.json 完全一致")
return 0
print(f"\nFAIL: {diffs} 处差异")
return 1
def main():
args = sys.argv[1:]
if not args:
sys.exit(__doc__)
if args[0] == "--subset":
argv = [a for a in args[1:]]
readonly = "--readonly" in argv
argv = [a for a in argv if a != "--readonly"]
if not argv:
sys.exit("用法: python verify_tools.py --subset <tools.json> [token] [port] [--readonly]")
path = argv[0]
token = resolve_token(argv[1] if len(argv) > 1 else "")
port = int(argv[2]) if len(argv) > 2 else None
if not token:
sys.exit("未找到 McpToken(参数 / YAMA_MCP_TOKEN / 注册表 均无)")
return verify_subset(path, token, port, readonly)
if args[0] == "--offline":
if len(args) < 3:
sys.exit("用法: python verify_tools.py --offline <tools.json> <黄金样本.json>")
path, sample = args[1], args[2]
tools_live = json.load(open(sample, encoding="utf-8"))["result"]["tools"]
print(f"黄金样本: {sample}")
elif args[0] == "--live":
if len(args) < 2:
sys.exit("用法: python verify_tools.py --live <tools.json> [token] [port]")
path = args[1]
token = resolve_token(args[2] if len(args) > 2 else "")
port = int(args[3]) if len(args) > 3 else None
if not token:
sys.exit("未找到 McpToken(参数 / YAMA_MCP_TOKEN / 注册表 均无)")
tools_live, live_stats = fetch_live(token, port)
print(f"活服务: 127.0.0.1:{port if port else 'auto'}")
print(f"体积: 响应 {live_stats['response_bytes']} B / tools 数组 {live_stats['tools_array_bytes']} B"
f"(description {live_stats['description_bytes']} / inputSchema {live_stats['input_schema_bytes']}"
f" / outputSchema {live_stats['output_schema_bytes']})")
else:
sys.exit(__doc__)
doc = json.load(open(path, encoding="utf-8"))
tools_json = doc["tools"]
# gate == "!machine" 的工具只在 McpMachine=0 时出现,而黄金样本/活服务都是开关
# 全开的抓取,按约定它们不该在这里出现(tools.json 是两份抓取的并集)。
off_gated = {t["name"] for t in tools_json if t.get("gate") == "!machine"}
tools_cmp = [t for t in tools_json if t["name"] not in off_gated]
print(f"tools.json: {path} ({len(tools_json)} 个工具)")
if off_gated:
print(f" 其中 {len(off_gated)} 个 !machine 降级工具不计入本次比对: {sorted(off_gated)}")
print(f"服务端: {len(tools_live)} 个工具")
if len(tools_cmp) != len(tools_live):
print(f"!! 数量不一致: {len(tools_cmp)} vs {len(tools_live)}")
live_by_name = {t["name"]: t for t in tools_live}
json_by_name = {t["name"]: t for t in tools_cmp}
only_live = sorted(set(live_by_name) - set(json_by_name))
only_json = sorted(set(json_by_name) - set(live_by_name))
if only_live:
print(f"!! 仅服务端有: {only_live}")
if only_json:
print(f"!! 仅 json 有: {only_json}")
diffs = 0
FIELDS_SHARED = sorted(set(live_by_name) & set(json_by_name))
diffs += report_field_diffs(live_by_name, json_by_name, FIELDS_SHARED)
# 顺序一致性:tools/list 的数组顺序会影响模型的选择倾向
order_live = [t["name"] for t in tools_live]
order_json = [t["name"] for t in tools_cmp]
if order_live != order_json:
print("!! 工具顺序不一致")
for i, (x, y) in enumerate(zip(order_live, order_json)):
if x != y:
print(f" 首个差异 @{i}: 服务端={x} json={y}")
break
diffs += 1
else:
print("工具顺序: 一致")
if diffs == 0 and not only_live and not only_json:
print(f"\nPASS: {len(tools_cmp)} 个工具完全一致")
return 0
print(f"\nFAIL: {diffs} 处差异")
return 1
if __name__ == "__main__":
sys.exit(main())