Fix(mcp): harden terminal teardown and fill annotation gaps
OnTerminalClosed now erases the shell route and nulls the session pointer before OfflineProc returns that context to the pool, and WaitTerminalReady re-looks-up the session on every wake, failing with cleanup when it is gone, closed, or has no sub-connection. Before, a waiter could CancelIO a recycled context, or a late cleanup could unroute a connection that reused it. terminal_close only set closed and expected a waiter to clean up, so after a caller timeout the remote shell kept running and the slot stayed busy for an hour. A waiters counter now separates the cases: with a waiter the close hands off, without one it erases the session and cancels IO at once. WaitTermCommand's closed branch keeps a truncated tail pollable instead of dropping it, cancelling IO immediately, and the drained poll cleans up. The write side had the same hazard. exec_command, terminal_exec and http_probe wrote the command line after the prompt wait without rechecking the sub-connection; IsTermSubCtxLive now skips the write when the shell has died, and WaitTerminalDone reports closed on every failure path so callers never CancelIO a pointer that may be back in the pool. list_hosts, read_file and http_probe were missing from kToolHints and got destructive defaults; the first two are now read-only/idempotent, http_probe stays writable because its method can mutate the probed service. Verified on the isolated Debug instance: the truncated tail drains without gaps (12514 lines), a waiterless close frees the slot in 0.1s, and the terminal, exec_command, http_probe, 8a/8b/8c and verify_tools checks pass. Co-Authored-By: deepseek-flash
This commit is contained in:
1 parent
5841f27df9
commit
425d737ca0
3 files changed
+111
-22
No files matched your search
@@ -4135,16 +4135,20 @@ std::string BuildExecCommand(const Json::Value& id, const Json::Value& args, CMy
|
||||
// 哨兵形态见 BuildSentinelLines:POSIX PTY / ConPTY 下命令与哨兵分两行下发(吞行命令
|
||||
// 不再超时),老 ShellManager 保持单行 && / || 形式(其客户端按写入长度跳回显)。
|
||||
std::string cmdLine, sentLine;
|
||||
SendSentinelLines(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
// 等待提示符期间 shell 可能已退出:OnTerminalClosed 会把子连接指针置空、OfflineProc 随后把它
|
||||
// 归还连接池(可能被新连接复用)。确认仍是本会话的子连接才写,否则直接交给 WaitTerminalDone
|
||||
// 收尾(会话已 closed,立刻返回已收集的输出),避免往别人的连接里灌命令。
|
||||
if (mcp.IsTermSubCtxLive(devId, std::string(), subCtx))
|
||||
SendSentinelLines(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
|
||||
std::vector<BYTE> raw;
|
||||
int exitCode = -1;
|
||||
bool closed = false;
|
||||
if (!mcp.WaitTerminalDone(devId, raw, exitCode, closed, timeoutMs)) {
|
||||
subCtx->CancelIO();
|
||||
if (!closed) subCtx->CancelIO();
|
||||
return BuildError(id, -32001, "Timeout waiting for command output");
|
||||
}
|
||||
subCtx->CancelIO(); // 关子链接,结束 shell 进程
|
||||
if (!closed) subCtx->CancelIO(); // 关子链接,结束 shell 进程;shell 已退出时指针可能已回收
|
||||
|
||||
// 清洗:raw → UTF-8 → 剥 ANSI → CRLF 归一 → 去 ConPTY 回显命令行 → TrimRight。
|
||||
// raw 已截断到哨兵前。
|
||||
@@ -5126,7 +5130,10 @@ std::string BuildTerminalExec(const Json::Value& id, const Json::Value& args, CM
|
||||
if (posix && isPty)
|
||||
mcp.WaitTerminalBootstrap(devId, timeoutMs < 2000 ? timeoutMs : 2000);
|
||||
|
||||
SendTerminalCommandLine(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
// 等待期间 shell 可能已退出(子连接断开后指针交还连接池、可能被复用):确认仍是本会话
|
||||
// 的子连接才写;不写则下面 WaitTermCommand 立即按 closed 收尾(会话结束、需重新 open)。
|
||||
if (mcp.IsTermSubCtxLive(devId, sessionId, subCtx))
|
||||
SendTerminalCommandLine(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
|
||||
// 审计:命令执行(不可关闭)。
|
||||
if (parent) {
|
||||
@@ -5152,7 +5159,8 @@ std::string BuildTerminalExec(const Json::Value& id, const Json::Value& args, CM
|
||||
int elapsedMs = (int)std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now() - started).count();
|
||||
|
||||
// shell 退出时 WaitTermCommand 已清理会话与路由;子连接本身已由客户端关闭,无需 CancelIO。
|
||||
// shell 退出时 WaitTermCommand 会清理会话与路由(本次被 max_output_bytes 截断时先保留,
|
||||
// 等后续轮询把尾巴取完再清理);子连接本身已由客户端关闭,无需 CancelIO。
|
||||
|
||||
// 轮询块没有命令回显可剥(回显只在首块);命令路径带 cmdLine/sentLine 走完整清洗。
|
||||
// 片段交付(命令未结束或本次被 max_output_bytes 截断)保留块尾空白,拼接才不粘连。
|
||||
@@ -8606,7 +8614,9 @@ std::string BuildHttpProbe(const Json::Value& id, const Json::Value& args, CMy20
|
||||
mcp.WaitTerminalBootstrap(devId, 2000);
|
||||
|
||||
std::string cmdLine, sentLine;
|
||||
SendTerminalCommandLine(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
// 同 terminal_exec:等提示符期间 shell 可能已退出,子连接指针可能已被连接池复用。
|
||||
if (mcp.IsTermSubCtxLive(devId, sessionId, subCtx))
|
||||
SendTerminalCommandLine(subCtx, isPty, posix, command, nonce, cmdLine, sentLine);
|
||||
|
||||
// 审计:请求的目标(不可关闭;不带 body/headers,避免泄露凭据)。
|
||||
if (parent) {
|
||||
@@ -8648,6 +8658,14 @@ std::string BuildHttpProbe(const Json::Value& id, const Json::Value& args, CMy20
|
||||
"Timeout waiting for http_probe result; the request is still running in terminal session " +
|
||||
sessionId + " (call terminal_exec with only session_id to collect the tail)");
|
||||
|
||||
// 写前校验发现子连接已断(shell 退出)时会跳过下发,这里必然没有输出:报会话结束的墓碑文案,
|
||||
// 比让解析器报「host 输出异常」清楚。
|
||||
if (closed && all.empty()) {
|
||||
std::string destroyed = mcp.DescribeTermDestroyed(devId, sessionId);
|
||||
return BuildError(id, -32002, destroyed.empty()
|
||||
? "Terminal session ended before the probe could run; call terminal_open again" : destroyed);
|
||||
}
|
||||
|
||||
// 本工具隐式创建的会话用完即关:留着会让后续 exec_command 被 -32003「终端会话已存在」
|
||||
// 挡到空闲回收为止。命令未结束时(上面的超时分支)保持会话,便于调用方轮询。
|
||||
if (!reused)
|
||||
@@ -12261,6 +12279,18 @@ bool CMcpServer::IsTerminalContext(context* subCtx) {
|
||||
return m_TermContextToDevice.find(subCtx) != m_TermContextToDevice.end();
|
||||
}
|
||||
|
||||
bool CMcpServer::IsTermSubCtxLive(uint64_t device_id, const std::string& sessionId, context* subCtx) {
|
||||
if (!subCtx) return false;
|
||||
std::lock_guard<std::mutex> lk(m_TermMutex);
|
||||
auto it = m_TermSessions.find(device_id);
|
||||
if (it == m_TermSessions.end()) return false;
|
||||
const TermSession& s = it->second;
|
||||
// OnTerminalClosed 置 closed 的同时把 subCtx 置空:closed 即「这个指针已交还连接池」。
|
||||
if (s.closed || s.subCtx != subCtx) return false;
|
||||
if (!sessionId.empty() && (!s.persistent || s.sessionId != sessionId)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
void CMcpServer::OnTerminalData(context* subCtx, const BYTE* data, ULONG len) {
|
||||
std::lock_guard<std::mutex> lk(m_TermMutex);
|
||||
auto it = m_TermContextToDevice.find(subCtx);
|
||||
@@ -12295,13 +12325,17 @@ void CMcpServer::OnTerminalClosed(context* subCtx) {
|
||||
auto sit = m_TermSessions.find(it->second);
|
||||
if (sit == m_TermSessions.end()) return;
|
||||
TermSession& s = sit->second;
|
||||
// 子连接已断,OfflineProc 随后会把它归还连接池:会话里不能再留这个指针,否则等待线程
|
||||
// 醒来(或后续轮询)会对已回收、甚至已被新连接复用的 context 误 CancelIO;路由也必须
|
||||
// 立刻摘掉,否则迟到的清理会按旧指针误删新连接的表项。
|
||||
m_TermContextToDevice.erase(it);
|
||||
s.subCtx = nullptr;
|
||||
s.closed = true;
|
||||
if (s.persistent && !s.busy && s.started) {
|
||||
// 空闲持久会话:shell 已退、无等待线程会清理,直接擦路由+会话(无需 CancelIO,已死)。
|
||||
m_TermContextToDevice.erase(subCtx);
|
||||
// 空闲持久会话:shell 已退、无等待线程会清理,直接擦会话(子连接已死,无需 CancelIO)。
|
||||
m_TermSessions.erase(sit);
|
||||
}
|
||||
m_TermCv.notify_all(); // busy 场景唤醒等待线程由其清理;一次性 exec 路径不变
|
||||
m_TermCv.notify_all(); // busy 场景唤醒等待线程交付剩余输出后由其清理;一次性 exec 路径不变
|
||||
}
|
||||
|
||||
bool CMcpServer::BeginTermPending(uint64_t device_id, const std::string& command, const std::string& nonce) {
|
||||
@@ -12316,15 +12350,26 @@ bool CMcpServer::BeginTermPending(uint64_t device_id, const std::string& command
|
||||
|
||||
bool CMcpServer::WaitTerminalReady(uint64_t device_id, context*& subCtx, bool& isPty, int timeoutMs) {
|
||||
std::unique_lock<std::mutex> lk(m_TermMutex);
|
||||
auto it = m_TermSessions.find(device_id);
|
||||
if (it == m_TermSessions.end()) return false;
|
||||
// 谓词每次重新查表:等待期间 OnTerminalClosed(子连接断开)可能擦会话或置 closed,
|
||||
// 缓存的迭代器会悬空。会话消失也算「结束等待」,由下面统一判失败。
|
||||
auto ready = [&] {
|
||||
auto jt = m_TermSessions.find(device_id);
|
||||
return jt == m_TermSessions.end() || jt->second.started || jt->second.closed;
|
||||
};
|
||||
bool signaled = WaitWithCancel(lk, m_TermCv, DeadlineAfter(timeoutMs), ready);
|
||||
|
||||
bool signaled = WaitWithCancel(lk, m_TermCv, DeadlineAfter(timeoutMs),
|
||||
[&] { return it->second.started; });
|
||||
if (!signaled) {
|
||||
auto it = m_TermSessions.find(device_id);
|
||||
if (!signaled || it == m_TermSessions.end() || !it->second.started || it->second.closed ||
|
||||
!it->second.subCtx) {
|
||||
// 放弃:迟到的 shell 子连接就绪包(TOKEN_SHELL_START/TOKEN_TERMINAL_START)不能弹终端窗口。
|
||||
// 失败原因可能是超时,也可能是子连接刚建立就断开(OnTerminalClosed 已置空 subCtx):
|
||||
// 绝不能把空指针或已回收的 context 交给调用方。
|
||||
MarkLateFirstPacket(device_id, { (BYTE)TOKEN_SHELL_START, (BYTE)TOKEN_TERMINAL_START });
|
||||
m_TermSessions.erase(it); // 超时/取消 → 清理
|
||||
if (it != m_TermSessions.end()) {
|
||||
// 会话要么未就绪、要么子连接已断,都不可再用;清理掉避免占着该主机的终端槽。
|
||||
if (it->second.subCtx) m_TermContextToDevice.erase(it->second.subCtx);
|
||||
m_TermSessions.erase(it);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
subCtx = it->second.subCtx;
|
||||
@@ -12352,13 +12397,17 @@ bool CMcpServer::WaitTerminalDone(uint64_t device_id, std::vector<BYTE>& out,
|
||||
int& exitCode, bool& closed, int timeoutMs) {
|
||||
std::unique_lock<std::mutex> lk(m_TermMutex);
|
||||
auto it = m_TermSessions.find(device_id);
|
||||
if (it == m_TermSessions.end()) return false;
|
||||
if (it == m_TermSessions.end()) {
|
||||
closed = true; // 会话已不在(被取消/回收):子连接去向未知,调用方不能再去碰这个指针
|
||||
return false;
|
||||
}
|
||||
|
||||
bool signaled = WaitWithCancel(lk, m_TermCv, DeadlineAfter(timeoutMs),
|
||||
[&] { return it->second.done || it->second.closed; });
|
||||
|
||||
if (!signaled) { // 超时/取消 → 清理(含路由表)
|
||||
m_TermContextToDevice.erase(it->second.subCtx);
|
||||
closed = it->second.closed; // 照实回报:closed 时子连接指针已置空、可能已被连接池复用
|
||||
if (it->second.subCtx) m_TermContextToDevice.erase(it->second.subCtx);
|
||||
m_TermSessions.erase(it);
|
||||
return false;
|
||||
}
|
||||
@@ -12530,6 +12579,7 @@ int CMcpServer::WaitTermCommand(uint64_t device_id, const std::string& sessionId
|
||||
return 7; // 没有在跑的命令:轮询只会空等到超时,直接告诉调用方用错
|
||||
isPty = it->second.isPty;
|
||||
nonce = it->second.nonce;
|
||||
++it->second.waiters; // terminal_close 靠它判断 closed 之后有没有人负责收尾
|
||||
|
||||
// 谓词每次都重新查表:等待期间会话可能被 terminal_close / busy 兜底回收擦掉。
|
||||
const std::string sid = sessionId;
|
||||
@@ -12581,6 +12631,23 @@ int CMcpServer::WaitTermCommand(uint64_t device_id, const std::string& sessionId
|
||||
s.delivered = deliverEnd;
|
||||
|
||||
if (s.closed) { // shell 进程退出(或运行中被 terminal_close)→ 交付剩余输出后清理会话+路由
|
||||
if (s.delivered < end) {
|
||||
// 本块被 maxOutputBytes 截断:会话与缓冲都保留(已 closed,pollMode 仍允许继续调用),
|
||||
// 由后续轮询把尾巴取完再清理。若在这里直接擦会话,未交付的输出会静默丢失。
|
||||
// 但远端 shell 必须现在就结束(terminal_close 场景),否则它会一直跑到 1 小时兜底;
|
||||
// 缓冲已收在 s.data 里,CancelIO 不影响后续轮询。
|
||||
context* subCtx = s.subCtx;
|
||||
if (subCtx) {
|
||||
m_TermContextToDevice.erase(subCtx);
|
||||
s.subCtx = nullptr;
|
||||
}
|
||||
--s.waiters;
|
||||
if (subCtx) {
|
||||
lk.unlock();
|
||||
subCtx->CancelIO();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
context* subCtx = s.subCtx;
|
||||
exitCode = -1;
|
||||
MarkTermDestroyedLocked(device_id, s.sessionId,
|
||||
@@ -12600,11 +12667,13 @@ int CMcpServer::WaitTermCommand(uint64_t device_id, const std::string& sessionId
|
||||
s.busy = false;
|
||||
s.lastActiveAt = time(nullptr); // idle 语义是「距上次活动」,避免刚完成就被回收
|
||||
}
|
||||
--s.waiters;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 超时:命令仍在远端运行。会话保持 busy、不销毁,调用方继续轮询(P0-2)。
|
||||
s.lastActiveAt = time(nullptr);
|
||||
--s.waiters;
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -12616,17 +12685,22 @@ int CMcpServer::CloseTermSession(uint64_t device_id, const std::string& sessionI
|
||||
if (it == m_TermSessions.end()) return 1; // 不存在(幂等)
|
||||
TermSession& s = it->second;
|
||||
if (!s.persistent || s.sessionId != sessionId) return 2; // token 不匹配
|
||||
if (s.busy) {
|
||||
// 有命令在飞:置 closed 唤醒等待线程,由其清理(避免双重擦除)。
|
||||
if (s.busy && s.waiters > 0) {
|
||||
// 有等待线程:置 closed 唤醒它,由其交付剩余输出后清理(避免双重擦除)。
|
||||
s.closed = true;
|
||||
MarkTermDestroyedLocked(device_id, sessionId,
|
||||
"Terminal session was closed by terminal_close while a command was running");
|
||||
m_TermCv.notify_all();
|
||||
return 0;
|
||||
}
|
||||
// 空闲,或 busy 但已无等待线程(调用方超时走人,只可能靠轮询):立刻收尾。
|
||||
// 后者若沿用「置 closed 等别人清理」,远端 shell 会一直跑到 1 小时 busy 兜底,
|
||||
// 期间该主机的终端全部返回 busy。
|
||||
subCtx = s.subCtx;
|
||||
MarkTermDestroyedLocked(device_id, sessionId, "Terminal session was closed by terminal_close");
|
||||
m_TermContextToDevice.erase(s.subCtx);
|
||||
MarkTermDestroyedLocked(device_id, sessionId,
|
||||
s.busy ? "Terminal session was closed by terminal_close while a command was running"
|
||||
: "Terminal session was closed by terminal_close");
|
||||
if (subCtx) m_TermContextToDevice.erase(subCtx);
|
||||
m_TermSessions.erase(it);
|
||||
}
|
||||
if (subCtx) subCtx->CancelIO(); // 锁外取消 IO,触发客户端 shell 退出
|
||||
|
||||
Reference in new issue
Block a user