a
This commit is contained in:
parent
2d17406799
commit
414607d566
|
|
@ -21,6 +21,7 @@ from api.supervisor_account import (
|
|||
program_name_for_account,
|
||||
tail_supervisor,
|
||||
tail_supervisord_log,
|
||||
tail_trading_log_files,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -235,14 +236,21 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
|
|||
running, pid, state = parse_supervisor_status(raw)
|
||||
# 仅 owner/admin 可看 tail(便于自助排障)
|
||||
stderr_tail = ""
|
||||
stdout_tail = ""
|
||||
stderr_tail_error = ""
|
||||
supervisord_tail = ""
|
||||
logfile_tail: Dict[str, Any] = {}
|
||||
try:
|
||||
is_admin = (user.get("role") or "user") == "admin"
|
||||
role = UserAccountMembership.get_role(int(user["id"]), int(account_id)) if not is_admin else "admin"
|
||||
if is_admin or role == "owner":
|
||||
if state in {"FATAL", "EXITED", "BACKOFF"}:
|
||||
stderr_tail = tail_supervisor(program, "stderr", 120)
|
||||
stdout_tail = tail_supervisor(program, "stdout", 200)
|
||||
try:
|
||||
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
|
||||
except Exception:
|
||||
logfile_tail = {}
|
||||
if not stderr_tail:
|
||||
try:
|
||||
supervisord_tail = tail_supervisord_log(80)
|
||||
|
|
@ -251,6 +259,7 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
|
|||
except Exception as te:
|
||||
stderr_tail_error = str(te)
|
||||
stderr_tail = ""
|
||||
stdout_tail = ""
|
||||
# spawn error 时 program stderr 可能为空,尝试给 supervisord 主日志做兜底
|
||||
try:
|
||||
supervisord_tail = tail_supervisord_log(80)
|
||||
|
|
@ -260,6 +269,10 @@ async def trading_status_for_account(account_id: int, user: Dict[str, Any] = Dep
|
|||
resp = {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}
|
||||
if stderr_tail:
|
||||
resp["stderr_tail"] = stderr_tail
|
||||
if stdout_tail:
|
||||
resp["stdout_tail"] = stdout_tail
|
||||
if logfile_tail:
|
||||
resp["logfile_tail"] = logfile_tail
|
||||
if stderr_tail_error:
|
||||
resp["stderr_tail_error"] = stderr_tail_error
|
||||
if supervisord_tail:
|
||||
|
|
@ -304,14 +317,24 @@ async def trading_start_for_account(account_id: int, user: Dict[str, Any] = Depe
|
|||
return {"message": "已启动", "output": out, "status": {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}}
|
||||
except Exception as e:
|
||||
tail = ""
|
||||
out_tail = ""
|
||||
tail_err = ""
|
||||
status_raw = ""
|
||||
supervisord_tail = ""
|
||||
logfile_tail: Dict[str, Any] = {}
|
||||
try:
|
||||
tail = tail_supervisor(program, "stderr", 120)
|
||||
except Exception as te:
|
||||
tail_err = str(te)
|
||||
tail = ""
|
||||
try:
|
||||
out_tail = tail_supervisor(program, "stdout", 200)
|
||||
except Exception:
|
||||
out_tail = ""
|
||||
try:
|
||||
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
|
||||
except Exception:
|
||||
logfile_tail = {}
|
||||
try:
|
||||
status_raw = run_supervisorctl(["status", program])
|
||||
except Exception:
|
||||
|
|
@ -328,6 +351,8 @@ async def trading_start_for_account(account_id: int, user: Dict[str, Any] = Depe
|
|||
"program": program,
|
||||
"hint": "如果是 spawn error,重点看 stderr_tail(常见:python 路径不可执行/依赖缺失/权限/工作目录不存在)",
|
||||
"stderr_tail": tail,
|
||||
"stdout_tail": out_tail,
|
||||
"logfile_tail": logfile_tail,
|
||||
"stderr_tail_error": tail_err,
|
||||
"status_raw": status_raw,
|
||||
"supervisord_tail": supervisord_tail,
|
||||
|
|
@ -364,14 +389,24 @@ async def trading_restart_for_account(account_id: int, user: Dict[str, Any] = De
|
|||
return {"message": "已重启", "output": out, "status": {"program": program, "running": running, "pid": pid, "state": state, "raw": raw}}
|
||||
except Exception as e:
|
||||
tail = ""
|
||||
out_tail = ""
|
||||
tail_err = ""
|
||||
status_raw = ""
|
||||
supervisord_tail = ""
|
||||
logfile_tail: Dict[str, Any] = {}
|
||||
try:
|
||||
tail = tail_supervisor(program, "stderr", 120)
|
||||
except Exception as te:
|
||||
tail_err = str(te)
|
||||
tail = ""
|
||||
try:
|
||||
out_tail = tail_supervisor(program, "stdout", 200)
|
||||
except Exception:
|
||||
out_tail = ""
|
||||
try:
|
||||
logfile_tail = tail_trading_log_files(int(account_id), lines=200)
|
||||
except Exception:
|
||||
logfile_tail = {}
|
||||
try:
|
||||
status_raw = run_supervisorctl(["status", program])
|
||||
except Exception:
|
||||
|
|
@ -388,6 +423,8 @@ async def trading_restart_for_account(account_id: int, user: Dict[str, Any] = De
|
|||
"program": program,
|
||||
"hint": "如果是 spawn error,重点看 stderr_tail(常见:python 路径不可执行/依赖缺失/权限/工作目录不存在)",
|
||||
"stderr_tail": tail,
|
||||
"stdout_tail": out_tail,
|
||||
"logfile_tail": logfile_tail,
|
||||
"stderr_tail_error": tail_err,
|
||||
"status_raw": status_raw,
|
||||
"supervisord_tail": supervisord_tail,
|
||||
|
|
|
|||
|
|
@ -29,6 +29,21 @@ DEFAULT_CANDIDATE_CONFS = [
|
|||
"/etc/supervisord.conf",
|
||||
]
|
||||
|
||||
# 常见 supervisord 主日志路径候选(不同发行版/面板插件差异很大)
|
||||
DEFAULT_SUPERVISORD_LOG_CANDIDATES = [
|
||||
# aaPanel / 宝塔 supervisor 插件常见
|
||||
"/www/server/panel/plugin/supervisor/log/supervisord.log",
|
||||
"/www/server/panel/plugin/supervisor/log/supervisor.log",
|
||||
"/www/server/panel/plugin/supervisor/supervisord.log",
|
||||
"/www/server/panel/plugin/supervisor/supervisor.log",
|
||||
# 系统 supervisor 常见
|
||||
"/var/log/supervisor/supervisord.log",
|
||||
"/var/log/supervisor/supervisor.log",
|
||||
"/var/log/supervisord.log",
|
||||
"/var/log/supervisord/supervisord.log",
|
||||
"/tmp/supervisord.log",
|
||||
]
|
||||
|
||||
|
||||
def _get_project_root() -> Path:
|
||||
# backend/api/supervisor_account.py -> api -> backend -> project_root
|
||||
|
|
@ -156,7 +171,7 @@ def render_program_ini(account_id: int, program_name: str) -> str:
|
|||
python_bin = sys.executable
|
||||
|
||||
# 日志目录可通过环境变量覆盖
|
||||
log_dir = Path(os.getenv("TRADING_LOG_DIR", str(project_root / "logs"))).expanduser()
|
||||
log_dir, out_log, err_log = expected_trading_log_paths(project_root, int(account_id))
|
||||
# supervisor 在 reread/update 时会校验 logfile 目录是否存在;这里提前创建避免 CANT_REREAD
|
||||
try:
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -177,8 +192,11 @@ def render_program_ini(account_id: int, program_name: str) -> str:
|
|||
f"directory={project_root}",
|
||||
f"command={python_bin} -m trading_system.main",
|
||||
"autostart=" + ("true" if autostart else "false"),
|
||||
"autorestart=true",
|
||||
"startsecs=3",
|
||||
# 更合理:仅在“非0退出”时重启;0 退出视为“正常结束”,不进入 FATAL 反复拉起
|
||||
"autorestart=unexpected",
|
||||
# 兼容:0/2 都视为“预期退出”(例如配置不完整/前置检查失败时主动退出)
|
||||
"exitcodes=0,2",
|
||||
"startsecs=0",
|
||||
"stopasgroup=true",
|
||||
"killasgroup=true",
|
||||
"stopsignal=TERM",
|
||||
|
|
@ -298,6 +316,27 @@ def _tail_text_file(path: Path, lines: int = 200, max_bytes: int = 64 * 1024) ->
|
|||
if n > 500:
|
||||
n = 500
|
||||
return "\n".join(parts[-n:]).strip()
|
||||
except Exception:
|
||||
# 兜底:若启用 sudo(通常 backend 自己无权读 root 日志),尝试 sudo tail
|
||||
try:
|
||||
use_sudo = (os.getenv("SUPERVISOR_USE_SUDO", "false") or "false").lower() == "true"
|
||||
if not use_sudo:
|
||||
return ""
|
||||
n = int(lines or 200)
|
||||
if n < 20:
|
||||
n = 20
|
||||
if n > 500:
|
||||
n = 500
|
||||
res = subprocess.run(
|
||||
["sudo", "-n", "tail", "-n", str(n), str(path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
out = (res.stdout or "").strip()
|
||||
err = (res.stderr or "").strip()
|
||||
# 不强行报错:宁可空,也不要影响主流程
|
||||
return (out or err or "").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
|
@ -348,9 +387,46 @@ def tail_supervisord_log(lines: int = 200) -> str:
|
|||
lp = _parse_supervisord_logfile_from_conf(conf)
|
||||
if lp:
|
||||
return _tail_text_file(lp, lines=lines)
|
||||
# 最后兜底:尝试常见路径
|
||||
for cand in DEFAULT_SUPERVISORD_LOG_CANDIDATES:
|
||||
try:
|
||||
p = Path(cand)
|
||||
if p.exists():
|
||||
text = _tail_text_file(p, lines=lines)
|
||||
if text:
|
||||
return text
|
||||
except Exception:
|
||||
continue
|
||||
return ""
|
||||
|
||||
|
||||
def expected_trading_log_paths(project_root: Path, account_id: int) -> Tuple[Path, Path, Path]:
|
||||
"""
|
||||
计算 trading program 的 stdout/stderr logfile 路径(需与 render_program_ini 保持一致)。
|
||||
返回 (log_dir, out_log, err_log)
|
||||
"""
|
||||
log_dir = Path(os.getenv("TRADING_LOG_DIR", str(project_root / "logs"))).expanduser()
|
||||
out_log = log_dir / f"trading_{int(account_id)}.out.log"
|
||||
err_log = log_dir / f"trading_{int(account_id)}.err.log"
|
||||
return log_dir, out_log, err_log
|
||||
|
||||
|
||||
def tail_trading_log_files(account_id: int, lines: int = 200) -> Dict[str, Any]:
|
||||
"""
|
||||
直接读取该账号 trading 进程的 stdout/stderr logfile 尾部(不依赖 supervisorctl tail)。
|
||||
返回 {out_log, err_log, stdout_tail, stderr_tail}
|
||||
"""
|
||||
project_root = _get_project_root()
|
||||
log_dir, out_log, err_log = expected_trading_log_paths(project_root, int(account_id))
|
||||
return {
|
||||
"log_dir": str(log_dir),
|
||||
"out_log": str(out_log),
|
||||
"err_log": str(err_log),
|
||||
"stdout_tail_file": _tail_text_file(out_log, lines=lines),
|
||||
"stderr_tail_file": _tail_text_file(err_log, lines=lines),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnsureProgramResult:
|
||||
ok: bool
|
||||
|
|
|
|||
|
|
@ -873,6 +873,12 @@ const ConfigPanel = ({ currentUser }) => {
|
|||
<pre style={{ whiteSpace: 'pre-wrap', marginTop: '8px' }}>{String(accountTradingStatus.stderr_tail || '')}</pre>
|
||||
</details>
|
||||
) : null}
|
||||
{accountTradingStatus?.stdout_tail ? (
|
||||
<details style={{ marginTop: '8px' }}>
|
||||
<summary style={{ cursor: 'pointer' }}>查看最近输出日志(stdout,常见原因在这里)</summary>
|
||||
<pre style={{ whiteSpace: 'pre-wrap', marginTop: '8px' }}>{String(accountTradingStatus.stdout_tail || '')}</pre>
|
||||
</details>
|
||||
) : null}
|
||||
{accountTradingStatus?.stderr_tail_error ? (
|
||||
<details style={{ marginTop: '8px' }}>
|
||||
<summary style={{ cursor: 'pointer' }}>读取 stderr 失败原因(用于排障)</summary>
|
||||
|
|
|
|||
|
|
@ -317,6 +317,7 @@ async def main():
|
|||
logger.info("收到停止信号,正在关闭...")
|
||||
except Exception as e:
|
||||
logger.error(f"程序运行出错: {e}", exc_info=True)
|
||||
raise
|
||||
finally:
|
||||
# 清理资源
|
||||
if client:
|
||||
|
|
@ -331,3 +332,4 @@ if __name__ == '__main__':
|
|||
logger.info("程序被用户中断")
|
||||
except Exception as e:
|
||||
logger.error(f"程序异常退出: {e}", exc_info=True)
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -171,5 +171,11 @@ async def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).error(f"推荐进程异常退出: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user