"""自动崩溃恢复（需求 §4.1 两级重启：先软后硬），2026-09-27 定为「先只看电流」。 服务器够不到手机的 adb（手机在机架那边的 NAT 后面），所以这一版的判据只来自板子遥测： · FLAT：窗口内电流几乎不变（max − min < flat_ua）且不低 —— 卡死的手机常见的样子； · LOW ：窗口内电流一直低于 low_ua —— 关机 / 掉到待机，却本该在跑。 adb 探活留了接口（`Prober`），手机能从服务器够到（比如经 frp）时再接上。 ⚠️ 这两条阈值**还没有实测依据**（刀片还没插、手机还没上架），默认值是按常识填的。 所以自动恢复**默认关闭、逐台打开**，而且： · 只在「这台手机开了自动恢复 + 板子在线 + 刀片在位 + 这一路有电 + 执行器空闲」时才判定， 任何一条不满足就把积累的历史清掉 —— 失联不动作（§10.2）； · 每小时最多 `max_per_hour` 次，超了就 GAVE_UP 停手、告警，要人来复位 —— 一个误判的阈值最坏的后果是「把一台正常的手机每 5 分钟重启一次」，这条闸就是防它的。 状态机：IDLE ──判定──▶ SOFT（长按电源 10 s）──等 settle──▶ 观察一个窗口 ▲ │ 仍然判定 │ 没判定 └──── 恢复 ◀──────────┼──────────────────────────────┘ ▼ HARD（断电重启）──仍然判定──▶ GAVE_UP（停手，等人） """ import asyncio import collections import json import logging from dataclasses import asdict, dataclass from typing import Callable, Deque, Dict, List, Optional, Tuple log = logging.getLogger("prrack.recovery") IDLE, SOFT, HARD, GAVE_UP = "idle", "soft", "hard", "gave_up" @dataclass class Params: window_s: int = 300 # 判定窗口 flat_ua: int = 5000 # 窗口内电流摆幅小于它 = 平线（5 mA） low_ua: int = 30000 # 窗口内电流一直低于它 = 像关机（30 mA） settle_s: int = 120 # 动作之后先等它重启完，这段不看 max_per_hour: int = 3 # 每台每小时最多动作几次 soft_ms: int = 10000 # 一级：长按电源键时长（需求 §4.1 ≈10 s） @staticmethod def load(text: str) -> "Params": p = Params() try: for k, v in json.loads(text or "{}").items(): if hasattr(p, k) and isinstance(v, int) and v > 0: setattr(p, k, v) except ValueError: pass return p class Detector: """一台手机的电流窗口。样本 1 条/s；窗口没攒满（覆盖不到 window_s 的 90 %）一律不下结论。""" def __init__(self, p: Params): self.p = p self.s: Deque[Tuple[int, int]] = collections.deque() def reset(self): self.s.clear() def add(self, ts: int, ua: int): if self.s and ts <= self.s[-1][0]: return # 时间倒退 / 重复：丢掉（一条迟到的遥测不能把窗口搅乱） self.s.append((ts, ua)) horizon = ts - self.p.window_s * 1000 while self.s and self.s[0][0] < horizon: self.s.popleft() def full(self, now: int) -> bool: if not self.s: return False covered = self.s[-1][0] - self.s[0][0] return covered >= self.p.window_s * 900 and len(self.s) >= self.p.window_s * 0.8 def verdict(self, now: int) -> Optional[str]: if not self.full(now): return None vals = [u for _, u in self.s] if max(vals) < self.p.low_ua: return "low" if max(vals) - min(vals) < self.p.flat_ua: return "flat" return None Action = Tuple[str, dict] # (原因, 要发的命令) class PhoneRecovery: """一台手机的状态机。纯逻辑：喂样本、吐动作，时钟由调用方给。""" def __init__(self, p: Params): self.p = p self.det = Detector(p) self.state = IDLE self.since = 0 # 进入当前状态的时刻 self.reason = "" self.attempts: List[int] = [] def eligible_reset(self): self.det.reset() def manual_reset(self): self.state, self.reason, self.attempts = IDLE, "", [] self.det.reset() def feed(self, now: int, ua: int, rail: int) -> Tuple[Optional[str], Optional[Action]]: """返回 (新状态或 None, 动作或 None)。""" if self.state == GAVE_UP: return None, None if self.state in (SOFT, HARD) and now - self.since < self.p.settle_s * 1000: return None, None # 还在重启过程中，这段电流不代表任何东西 self.det.add(now, ua) v = self.det.verdict(now) if v is None: if self.state in (SOFT, HARD) and self.det.full(now): self.state, self.reason = IDLE, "recovered" return IDLE, None return None, None self.attempts = [t for t in self.attempts if now - t < 3600 * 1000] if len(self.attempts) >= self.p.max_per_hour or self.state == HARD: self.state, self.reason = GAVE_UP, ("rate_limit" if self.state != HARD else v) return GAVE_UP, None nxt = SOFT if self.state == IDLE else HARD self.state, self.since, self.reason = nxt, now, v self.attempts.append(now) self.det.reset() if nxt == SOFT: return SOFT, (v, {"cmd": "key", "rail": rail, "keys": "pwr", "ms": self.p.soft_ms}) return HARD, (v, {"cmd": "power_cycle", "rail": rail}) class Prober: """adb 探活的接口（预留）。返回 True 活着 / False 死了 / None 不知道（没配、够不着）。""" async def probe(self, addr: str) -> Optional[bool]: return None class RecoveryManager: """把状态机接到 Hub 上：订阅遥测，判定，经 hub.call 下发动作，记录并推送。""" def __init__(self, hub, clock: Callable[[], int]): self.hub = hub self.db = hub.db self.clock = clock self.params = Params.load(self.db.setting("recovery", "{}")) self.phones: Dict[Tuple[str, int], PhoneRecovery] = {} self.enabled: Dict[Tuple[str, int], bool] = {} self.reload() hub.tele_observers.append(self.on_tele) def reload(self): for r in self.db.q("SELECT board_id, rail, auto_recover FROM phones"): key = (r["board_id"], r["rail"]) was = self.enabled.get(key) self.enabled[key] = bool(r["auto_recover"]) if was and not self.enabled[key] and key in self.phones: self.phones[key].manual_reset() # 关掉 = 清状态（含 GAVE_UP） def set_params(self, p: Params): self.params = p self.db.set_setting("recovery", json.dumps(asdict(p))) for ph in self.phones.values(): ph.p = p ph.det.p = p def phone(self, board: str, rail: int) -> PhoneRecovery: key = (board, rail) if key not in self.phones: self.phones[key] = PhoneRecovery(self.params) return self.phones[key] def states(self) -> List[dict]: return [{"board": b, "rail": r, "state": ph.state, "reason": ph.reason, "since": ph.since, "attempts_1h": len([t for t in ph.attempts if self.clock() - t < 3600 * 1000]), "enabled": self.enabled.get((b, r), False)} for (b, r), ph in sorted(self.phones.items())] def reset(self, board: str, rail: int): self.phone(board, rail).manual_reset() self._log(board, rail, IDLE, "manual_reset") def on_tele(self, board: str, ts: int, st: dict): now = self.clock() for r in st.get("rails", []): rail = r.get("r") key = (board, rail) if not self.enabled.get(key): continue ph = self.phone(board, rail) ok = r.get("pr") and r.get("en") and r.get("st") == "idle" and isinstance(r.get("ua"), int) if not ok: ph.eligible_reset() continue state, action = ph.feed(now, r["ua"], rail) if state is not None: self._log(board, rail, state, ph.reason, action[1] if action else None) if action is not None: asyncio.ensure_future(self._act(board, rail, action[1])) async def _act(self, board: str, rail: int, cmd: dict): try: rsp = await self.hub.call(board, cmd, "auto-recovery") if not rsp.get("ok"): log.warning("自动恢复 %s#%d 的 %s 被板子拒了：%s", board, rail, cmd["cmd"], rsp) except Exception as e: # 超时也只记下来：失联不再加码 log.warning("自动恢复 %s#%d 下发失败：%s", board, rail, e) def _log(self, board: str, rail: int, state: str, reason: str, cmd: Optional[dict] = None): detail = json.dumps(cmd) if cmd else "" now = self.clock() rid = self.db.x("INSERT INTO recovery_log (ts, board_id, rail, state, reason, detail) " "VALUES (?,?,?,?,?,?)", (now, board, rail, state, reason, detail)) self.hub.broadcast({"t": "recovery", "row": {"id": rid, "ts": now, "board_id": board, "rail": rail, "state": state, "reason": reason, "detail": detail}})