This commit is contained in:
薇薇安 2026-01-21 22:38:24 +08:00
parent 2d17406799
commit 414607d566
5 changed files with 134 additions and 7 deletions

View File

@ -21,6 +21,7 @@ from api.supervisor_account import (
program_name_for_account,
tail_supervisor,
tail_supervisord_log,
tail_trading_log_files,
)
logger = logging.getLogger(__name__)
@ -235,14 +236,21 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
running, pid, state = parse_supervisor_status(raw)
# 仅 owner/admin 可看 tail便于自助排障
stderr_tail = ""
stdout_tail = ""
stderr_tail_error = ""
supervisord_tail = ""
logfile_tail: Dict[str, Any] = {}
try:
is_admin = (user.get("role") or "user") == "admin"
role = UserAccountMembership.get_role(int(user["id"]), int(account_id)) if not is_admin else "admin"
if is_admin or role == "owner":
if state in {"FATAL", "EXITED", "BACKOFF"}:
stderr_tail = tail_supervisor(program, "stderr", 120)
stdout_tail = tail_supervisor(program, "stdout", 200)
try:
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
except Exception:
logfile_tail = {}
if not stderr_tail:
try:
supervisord_tail = tail_supervisord_log(80)
@ -251,6 +259,7 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
except Exception as te:
stderr_tail_error = str(te)
stderr_tail = ""
stdout_tail = ""
# spawn error 时 program stderr 可能为空,尝试给 supervisord 主日志做兜底
try:
supervisord_tail = tail_supervisord_log(80)
@ -260,6 +269,10 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
resp = {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}
if stderr_tail:
resp["stderr_tail"] = stderr_tail
if stdout_tail:
resp["stdout_tail"] = stdout_tail
if logfile_tail:
resp["logfile_tail"] = logfile_tail
if stderr_tail_error:
resp["stderr_tail_error"] = stderr_tail_error
if supervisord_tail:
@ -304,14 +317,24 @@ async def trading_start_for_account(account_id: int, user: Dict[str, Any] = Depe
return {"message": "已启动", "output": out, "status": {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}}
except Exception as e:
tail = ""
out_tail = ""
tail_err = ""
status_raw = ""
supervisord_tail = ""
logfile_tail: Dict[str, Any] = {}
try:
tail = tail_supervisor(program, "stderr", 120)
except Exception as te:
tail_err = str(te)
tail = ""
try:
out_tail = tail_supervisor(program, "stdout", 200)
except Exception:
out_tail = ""
try:
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
except Exception:
logfile_tail = {}
try:
status_raw = run_supervisorctl(["status", program])
except Exception:
@ -328,6 +351,8 @@ async def trading_start_for_account(account_id: int, user: Dict[str, Any] = Depe
"program": program,
"hint": "如果是 spawn error重点看 stderr_tail常见python 路径不可执行/依赖缺失/权限/工作目录不存在)",
"stderr_tail": tail,
"stdout_tail": out_tail,
"logfile_tail": logfile_tail,
"stderr_tail_error": tail_err,
"status_raw": status_raw,
"supervisord_tail": supervisord_tail,
@ -364,14 +389,24 @@ async def trading_restart_for_account(account_id: int, user: Dict[str, Any] = De
return {"message": "已重启", "output": out, "status": {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}}
except Exception as e:
tail = ""
out_tail = ""
tail_err = ""
status_raw = ""
supervisord_tail = ""
logfile_tail: Dict[str, Any] = {}
try:
tail = tail_supervisor(program, "stderr", 120)
except Exception as te:
tail_err = str(te)
tail = ""
try:
out_tail = tail_supervisor(program, "stdout", 200)
except Exception:
out_tail = ""
try:
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
except Exception:
logfile_tail = {}
try:
status_raw = run_supervisorctl(["status", program])
except Exception:
@ -388,6 +423,8 @@ async def trading_restart_for_account(account_id: int, user: Dict[str, Any] = De
"program": program,
"hint": "如果是 spawn error重点看 stderr_tail常见python 路径不可执行/依赖缺失/权限/工作目录不存在)",
"stderr_tail": tail,
"stdout_tail": out_tail,
"logfile_tail": logfile_tail,
"stderr_tail_error": tail_err,
"status_raw": status_raw,
"supervisord_tail": supervisord_tail,

View File

@ -29,6 +29,21 @@ DEFAULT_CANDIDATE_CONFS = [
"/etc/supervisord.conf",
]
# 常见 supervisord 主日志路径候选(不同发行版/面板插件差异很大)
DEFAULT_SUPERVISORD_LOG_CANDIDATES = [
# aaPanel / 宝塔 supervisor 插件常见
"/www/server/panel/plugin/supervisor/log/supervisord.log",
"/www/server/panel/plugin/supervisor/log/supervisor.log",
"/www/server/panel/plugin/supervisor/supervisord.log",
"/www/server/panel/plugin/supervisor/supervisor.log",
# 系统 supervisor 常见
"/var/log/supervisor/supervisord.log",
"/var/log/supervisor/supervisor.log",
"/var/log/supervisord.log",
"/var/log/supervisord/supervisord.log",
"/tmp/supervisord.log",
]
def _get_project_root() -> Path:
# backend/api/supervisor_account.py -> api -> backend -> project_root
@ -156,7 +171,7 @@ def render_program_ini(account_id: int, program_name: str) -> str:
python_bin = sys.executable
# 日志目录可通过环境变量覆盖
log_dir = Path(os.getenv("TRADING_LOG_DIR", str(project_root / "logs"))).expanduser()
log_dir, out_log, err_log = expected_trading_log_paths(project_root, int(account_id))
# supervisor 在 reread/update 时会校验 logfile 目录是否存在;这里提前创建避免 CANT_REREAD
try:
log_dir.mkdir(parents=True, exist_ok=True)
@ -177,8 +192,11 @@ def render_program_ini(account_id: int, program_name: str) -> str:
f"directory={project_root}",
f"command={python_bin} -m trading_system.main",
"autostart=" + ("true" if autostart else "false"),
"autorestart=true",
"startsecs=3",
# 更合理仅在“非0退出”时重启0 退出视为“正常结束”,不进入 FATAL 反复拉起
"autorestart=unexpected",
# 兼容0/2 都视为“预期退出”(例如配置不完整/前置检查失败时主动退出)
"exitcodes=0,2",
"startsecs=0",
"stopasgroup=true",
"killasgroup=true",
"stopsignal=TERM",
@ -298,6 +316,27 @@ def _tail_text_file(path: Path, lines: int = 200, max_bytes: int = 64 * 1024) ->
if n > 500:
n = 500
return "\n".join(parts[-n:]).strip()
except Exception:
# 兜底:若启用 sudo通常 backend 自己无权读 root 日志),尝试 sudo tail
try:
use_sudo = (os.getenv("SUPERVISOR_USE_SUDO", "false") or "false").lower() == "true"
if not use_sudo:
return ""
n = int(lines or 200)
if n < 20:
n = 20
if n > 500:
n = 500
res = subprocess.run(
["sudo", "-n", "tail", "-n", str(n), str(path)],
capture_output=True,
text=True,
timeout=5,
)
out = (res.stdout or "").strip()
err = (res.stderr or "").strip()
# 不强行报错:宁可空,也不要影响主流程
return (out or err or "").strip()
except Exception:
return ""
@ -348,9 +387,46 @@ def tail_supervisord_log(lines: int = 200) -> str:
lp = _parse_supervisord_logfile_from_conf(conf)
if lp:
return _tail_text_file(lp, lines=lines)
# 最后兜底:尝试常见路径
for cand in DEFAULT_SUPERVISORD_LOG_CANDIDATES:
try:
p = Path(cand)
if p.exists():
text = _tail_text_file(p, lines=lines)
if text:
return text
except Exception:
continue
return ""
def expected_trading_log_paths(project_root: Path, account_id: int) -> Tuple[Path, Path, Path]:
"""
计算 trading program stdout/stderr logfile 路径需与 render_program_ini 保持一致
返回 (log_dir, out_log, err_log)
"""
log_dir = Path(os.getenv("TRADING_LOG_DIR", str(project_root / "logs"))).expanduser()
out_log = log_dir / f"trading_{int(account_id)}.out.log"
err_log = log_dir / f"trading_{int(account_id)}.err.log"
return log_dir, out_log, err_log
def tail_trading_log_files(account_id: int, lines: int = 200) -> Dict[str, Any]:
"""
直接读取该账号 trading 进程的 stdout/stderr logfile 尾部不依赖 supervisorctl tail
返回 {out_log, err_log, stdout_tail, stderr_tail}
"""
project_root = _get_project_root()
log_dir, out_log, err_log = expected_trading_log_paths(project_root, int(account_id))
return {
"log_dir": str(log_dir),
"out_log": str(out_log),
"err_log": str(err_log),
"stdout_tail_file": _tail_text_file(out_log, lines=lines),
"stderr_tail_file": _tail_text_file(err_log, lines=lines),
}
@dataclass
class EnsureProgramResult:
ok: bool

View File

@ -873,6 +873,12 @@ const ConfigPanel = ({ currentUser }) => {
<pre style={{ whiteSpace: 'pre-wrap', marginTop: '8px' }}>{String(accountTradingStatus.stderr_tail || '')}</pre>
</details>
) : null}
{accountTradingStatus?.stdout_tail ? (
<details style={{ marginTop: '8px' }}>
<summary style={{ cursor: 'pointer' }}>查看最近输出日志stdout常见原因在这里</summary>
<pre style={{ whiteSpace: 'pre-wrap', marginTop: '8px' }}>{String(accountTradingStatus.stdout_tail || '')}</pre>
</details>
) : null}
{accountTradingStatus?.stderr_tail_error ? (
<details style={{ marginTop: '8px' }}>
<summary style={{ cursor: 'pointer' }}>读取 stderr 失败原因用于排障</summary>

View File

@ -317,6 +317,7 @@ async def main():
logger.info("收到停止信号,正在关闭...")
except Exception as e:
logger.error(f"程序运行出错: {e}", exc_info=True)
raise
finally:
# 清理资源
if client:
@ -331,3 +332,4 @@ if __name__ == '__main__':
logger.info("程序被用户中断")
except Exception as e:
logger.error(f"程序异常退出: {e}", exc_info=True)
raise

View File

@ -171,5 +171,11 @@ async def main():
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
pass
except Exception as e:
logging.getLogger(__name__).error(f"推荐进程异常退出: {e}", exc_info=True)
raise