机架管理前后端（rack/）：FastAPI + SQLite + Vue3，经统一门户上线
- 后端 prrack：MQTT（phonerail-backend @ 127.0.0.1:1884）↔ 遥测入库 / 事件增量同步 / WebSocket 推送； 命令白名单 + 重传同 seq；审计记门户 X-Auth-User（只绑 127.0.0.1，main.py 拒绝非回环）。 - 自动恢复：只看电流（平线 / 过低，服务器够不到手机 adb），先长按电源再断电重启， 每小时限次后停手；默认逐台关闭。adb 探活接口 recovery.Prober 预留。 - 前端：总览 / 板子页（五路操作、电流电压曲线）/ 事件 / 操作记录 / 自动恢复；手机宽度可用。 - 部署 deploy/deploy.sh：服务器 Python 3.8 上先跑测试再替换；systemd phonerail-rack，每日备份。 - 门户应用 id 16、key rack（phonerail 已被需求文档站占用），授权 admin + staff。 测试 26 例（本机 3.14 / 服务器 3.8 均过）。线上板 b81f3fc9c8bc 在线，WS 经门户 101 + 1 条/s 遥测。 Co-Authored-By: Claude Opus 5.5 <[email protected]>
这个提交包含在：
zhou yang
@@ -0,0 +1,221 @@
|
||||
"""自动崩溃恢复（需求 §4.1 两级重启：先软后硬），2026-09-27 定为「先只看电流」。
|
||||
|
||||
服务器够不到手机的 adb（手机在机架那边的 NAT 后面），所以这一版的判据只来自板子遥测：
|
||||
· FLAT：窗口内电流几乎不变（max − min < flat_ua）且不低 —— 卡死的手机常见的样子；
|
||||
· LOW ：窗口内电流一直低于 low_ua —— 关机 / 掉到待机，却本该在跑。
|
||||
adb 探活留了接口（`Prober`），手机能从服务器够到（比如经 frp）时再接上。
|
||||
|
||||
⚠️ 这两条阈值**还没有实测依据**（刀片还没插、手机还没上架），默认值是按常识填的。
|
||||
所以自动恢复**默认关闭、逐台打开**，而且：
|
||||
· 只在「这台手机开了自动恢复 + 板子在线 + 刀片在位 + 这一路有电 + 执行器空闲」时才判定，
|
||||
任何一条不满足就把积累的历史清掉 —— 失联不动作（§10.2）；
|
||||
· 每小时最多 `max_per_hour` 次，超了就 GAVE_UP 停手、告警，要人来复位 ——
|
||||
一个误判的阈值最坏的后果是「把一台正常的手机每 5 分钟重启一次」，这条闸就是防它的。
|
||||
|
||||
状态机：IDLE ──判定──▶ SOFT（长按电源 10 s）──等 settle──▶ 观察一个窗口
|
||||
▲ │ 仍然判定 │ 没判定
|
||||
└──── 恢复 ◀──────────┼──────────────────────────────┘
|
||||
▼
|
||||
HARD（断电重启）──仍然判定──▶ GAVE_UP（停手，等人）
|
||||
"""
|
||||
import asyncio
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Callable, Deque, Dict, List, Optional, Tuple
|
||||
|
||||
log = logging.getLogger("prrack.recovery")
|
||||
|
||||
IDLE, SOFT, HARD, GAVE_UP = "idle", "soft", "hard", "gave_up"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Params:
|
||||
window_s: int = 300 # 判定窗口
|
||||
flat_ua: int = 5000 # 窗口内电流摆幅小于它 = 平线（5 mA）
|
||||
low_ua: int = 30000 # 窗口内电流一直低于它 = 像关机（30 mA）
|
||||
settle_s: int = 120 # 动作之后先等它重启完，这段不看
|
||||
max_per_hour: int = 3 # 每台每小时最多动作几次
|
||||
soft_ms: int = 10000 # 一级：长按电源键时长（需求 §4.1 ≈10 s）
|
||||
|
||||
@staticmethod
|
||||
def load(text: str) -> "Params":
|
||||
p = Params()
|
||||
try:
|
||||
for k, v in json.loads(text or "{}").items():
|
||||
if hasattr(p, k) and isinstance(v, int) and v > 0:
|
||||
setattr(p, k, v)
|
||||
except ValueError:
|
||||
pass
|
||||
return p
|
||||
|
||||
|
||||
class Detector:
|
||||
"""一台手机的电流窗口。样本 1 条/s；窗口没攒满（覆盖不到 window_s 的 90 %）一律不下结论。"""
|
||||
|
||||
def __init__(self, p: Params):
|
||||
self.p = p
|
||||
self.s: Deque[Tuple[int, int]] = collections.deque()
|
||||
|
||||
def reset(self):
|
||||
self.s.clear()
|
||||
|
||||
def add(self, ts: int, ua: int):
|
||||
if self.s and ts <= self.s[-1][0]:
|
||||
return # 时间倒退 / 重复：丢掉（一条迟到的遥测不能把窗口搅乱）
|
||||
self.s.append((ts, ua))
|
||||
horizon = ts - self.p.window_s * 1000
|
||||
while self.s and self.s[0][0] < horizon:
|
||||
self.s.popleft()
|
||||
|
||||
def full(self, now: int) -> bool:
|
||||
if not self.s:
|
||||
return False
|
||||
covered = self.s[-1][0] - self.s[0][0]
|
||||
return covered >= self.p.window_s * 900 and len(self.s) >= self.p.window_s * 0.8
|
||||
|
||||
def verdict(self, now: int) -> Optional[str]:
|
||||
if not self.full(now):
|
||||
return None
|
||||
vals = [u for _, u in self.s]
|
||||
if max(vals) < self.p.low_ua:
|
||||
return "low"
|
||||
if max(vals) - min(vals) < self.p.flat_ua:
|
||||
return "flat"
|
||||
return None
|
||||
|
||||
|
||||
Action = Tuple[str, dict] # (原因, 要发的命令)
|
||||
|
||||
|
||||
class PhoneRecovery:
|
||||
"""一台手机的状态机。纯逻辑：喂样本、吐动作，时钟由调用方给。"""
|
||||
|
||||
def __init__(self, p: Params):
|
||||
self.p = p
|
||||
self.det = Detector(p)
|
||||
self.state = IDLE
|
||||
self.since = 0 # 进入当前状态的时刻
|
||||
self.reason = ""
|
||||
self.attempts: List[int] = []
|
||||
|
||||
def eligible_reset(self):
|
||||
self.det.reset()
|
||||
|
||||
def manual_reset(self):
|
||||
self.state, self.reason, self.attempts = IDLE, "", []
|
||||
self.det.reset()
|
||||
|
||||
def feed(self, now: int, ua: int, rail: int) -> Tuple[Optional[str], Optional[Action]]:
|
||||
"""返回 (新状态或 None, 动作或 None)。"""
|
||||
if self.state == GAVE_UP:
|
||||
return None, None
|
||||
if self.state in (SOFT, HARD) and now - self.since < self.p.settle_s * 1000:
|
||||
return None, None # 还在重启过程中，这段电流不代表任何东西
|
||||
self.det.add(now, ua)
|
||||
v = self.det.verdict(now)
|
||||
if v is None:
|
||||
if self.state in (SOFT, HARD) and self.det.full(now):
|
||||
self.state, self.reason = IDLE, "recovered"
|
||||
return IDLE, None
|
||||
return None, None
|
||||
self.attempts = [t for t in self.attempts if now - t < 3600 * 1000]
|
||||
if len(self.attempts) >= self.p.max_per_hour or self.state == HARD:
|
||||
self.state, self.reason = GAVE_UP, ("rate_limit" if self.state != HARD else v)
|
||||
return GAVE_UP, None
|
||||
nxt = SOFT if self.state == IDLE else HARD
|
||||
self.state, self.since, self.reason = nxt, now, v
|
||||
self.attempts.append(now)
|
||||
self.det.reset()
|
||||
if nxt == SOFT:
|
||||
return SOFT, (v, {"cmd": "key", "rail": rail, "keys": "pwr", "ms": self.p.soft_ms})
|
||||
return HARD, (v, {"cmd": "power_cycle", "rail": rail})
|
||||
|
||||
|
||||
class Prober:
|
||||
"""adb 探活的接口（预留）。返回 True 活着 / False 死了 / None 不知道（没配、够不着）。"""
|
||||
|
||||
async def probe(self, addr: str) -> Optional[bool]:
|
||||
return None
|
||||
|
||||
|
||||
class RecoveryManager:
|
||||
"""把状态机接到 Hub 上：订阅遥测，判定，经 hub.call 下发动作，记录并推送。"""
|
||||
|
||||
def __init__(self, hub, clock: Callable[[], int]):
|
||||
self.hub = hub
|
||||
self.db = hub.db
|
||||
self.clock = clock
|
||||
self.params = Params.load(self.db.setting("recovery", "{}"))
|
||||
self.phones: Dict[Tuple[str, int], PhoneRecovery] = {}
|
||||
self.enabled: Dict[Tuple[str, int], bool] = {}
|
||||
self.reload()
|
||||
hub.tele_observers.append(self.on_tele)
|
||||
|
||||
def reload(self):
|
||||
for r in self.db.q("SELECT board_id, rail, auto_recover FROM phones"):
|
||||
key = (r["board_id"], r["rail"])
|
||||
was = self.enabled.get(key)
|
||||
self.enabled[key] = bool(r["auto_recover"])
|
||||
if was and not self.enabled[key] and key in self.phones:
|
||||
self.phones[key].manual_reset() # 关掉 = 清状态（含 GAVE_UP）
|
||||
|
||||
def set_params(self, p: Params):
|
||||
self.params = p
|
||||
self.db.set_setting("recovery", json.dumps(asdict(p)))
|
||||
for ph in self.phones.values():
|
||||
ph.p = p
|
||||
ph.det.p = p
|
||||
|
||||
def phone(self, board: str, rail: int) -> PhoneRecovery:
|
||||
key = (board, rail)
|
||||
if key not in self.phones:
|
||||
self.phones[key] = PhoneRecovery(self.params)
|
||||
return self.phones[key]
|
||||
|
||||
def states(self) -> List[dict]:
|
||||
return [{"board": b, "rail": r, "state": ph.state, "reason": ph.reason, "since": ph.since,
|
||||
"attempts_1h": len([t for t in ph.attempts if self.clock() - t < 3600 * 1000]),
|
||||
"enabled": self.enabled.get((b, r), False)}
|
||||
for (b, r), ph in sorted(self.phones.items())]
|
||||
|
||||
def reset(self, board: str, rail: int):
|
||||
self.phone(board, rail).manual_reset()
|
||||
self._log(board, rail, IDLE, "manual_reset")
|
||||
|
||||
def on_tele(self, board: str, ts: int, st: dict):
|
||||
now = self.clock()
|
||||
for r in st.get("rails", []):
|
||||
rail = r.get("r")
|
||||
key = (board, rail)
|
||||
if not self.enabled.get(key):
|
||||
continue
|
||||
ph = self.phone(board, rail)
|
||||
ok = r.get("pr") and r.get("en") and r.get("st") == "idle" and isinstance(r.get("ua"), int)
|
||||
if not ok:
|
||||
ph.eligible_reset()
|
||||
continue
|
||||
state, action = ph.feed(now, r["ua"], rail)
|
||||
if state is not None:
|
||||
self._log(board, rail, state, ph.reason, action[1] if action else None)
|
||||
if action is not None:
|
||||
asyncio.ensure_future(self._act(board, rail, action[1]))
|
||||
|
||||
async def _act(self, board: str, rail: int, cmd: dict):
|
||||
try:
|
||||
rsp = await self.hub.call(board, cmd, "auto-recovery")
|
||||
if not rsp.get("ok"):
|
||||
log.warning("自动恢复 %s#%d 的 %s 被板子拒了：%s", board, rail, cmd["cmd"], rsp)
|
||||
except Exception as e: # 超时也只记下来：失联不再加码
|
||||
log.warning("自动恢复 %s#%d 下发失败：%s", board, rail, e)
|
||||
|
||||
def _log(self, board: str, rail: int, state: str, reason: str, cmd: Optional[dict] = None):
|
||||
detail = json.dumps(cmd) if cmd else ""
|
||||
now = self.clock()
|
||||
rid = self.db.x("INSERT INTO recovery_log (ts, board_id, rail, state, reason, detail) "
|
||||
"VALUES (?,?,?,?,?,?)", (now, board, rail, state, reason, detail))
|
||||
self.hub.broadcast({"t": "recovery", "row": {"id": rid, "ts": now, "board_id": board,
|
||||
"rail": rail, "state": state, "reason": reason,
|
||||
"detail": detail}})
|
||||
|
||||
在新工单中引用
屏蔽一个用户